cmssw: Segfault in ReferenceTrajectory::addMaterialEffectsLocalGbl() in 1001.3

Workflow 1001.3 step 3 segfaulted in both MULTIARCHS and SKYLAKE IBs. Here is the stack trace from CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300

Thread 5 (Thread 0x150bcd7ff640 (LWP 3193665) "cmsRun"):
#3  0x0000150c196fca30 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x0000150c1ff6c8b1 in int TStreamerInfo::ReadBufferArtificial<char**>(TBuffer&, char** const&, TStreamerElement*, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#6  0x0000150c2002f3de in int TStreamerInfo::ReadBuffer<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#7  0x0000150c1feb268d in TStreamerInfoActions::GenericReadAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#8  0x0000150c1fdebe6e in TBufferFile::ReadClassBuffer(TClass const*, void*, TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#9  0x0000150c20030b0c in int TStreamerInfo::ReadBuffer<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#10 0x0000150c1feb268d in TStreamerInfoActions::GenericReadAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#11 0x0000150c1fdebe6e in TBufferFile::ReadClassBuffer(TClass const*, void*, TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#12 0x0000150c20030b0c in int TStreamerInfo::ReadBuffer<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#13 0x0000150c1feb268d in TStreamerInfoActions::GenericReadAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#14 0x0000150c1fdebe6e in TBufferFile::ReadClassBuffer(TClass const*, void*, TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#15 0x0000150c1fb352ec in TStreamerBase::ReadBuffer(TBuffer&, char*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libCore.so
#16 0x0000150c200309ec in int TStreamerInfo::ReadBuffer<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#17 0x0000150c1feb268d in TStreamerInfoActions::GenericReadAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#18 0x0000150c1fdebe6e in TBufferFile::ReadClassBuffer(TClass const*, void*, TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#19 0x0000150c1fdec9d9 in TBufferFile::ReadObjectAny(TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#20 0x0000150c1fe6628e in TGenCollectionStreamer::ReadObjects(int, TBuffer&, TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#21 0x0000150c1fe66d17 in TGenCollectionStreamer::ReadBufferGeneric(TBuffer&, void*, TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#22 0x0000150c1fde4dc9 in TBufferFile::ReadFastArray(void*, TClass const*, int, TMemberStreamer*, TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#23 0x0000150c1ff32ee5 in int TStreamerInfoActions::ReadSTL<&TStreamerInfoActions::ReadSTLMemberWiseSameClass, &TStreamerInfoActions::ReadSTLObjectWiseFastArray>(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#24 0x0000150c1fdebe6e in TBufferFile::ReadClassBuffer(TClass const*, void*, TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#25 0x0000150c20030b0c in int TStreamerInfo::ReadBuffer<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#26 0x0000150c1feb268d in TStreamerInfoActions::GenericReadAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#27 0x0000150c1fde3b75 in TBufferFile::ApplySequence(TStreamerInfoActions::TActionSequence const&, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libRIO.so
#28 0x0000150c20544aa7 in TBranchElement::ReadLeavesMember(TBuffer&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libTree.so
#29 0x0000150c2053d349 in TBranch::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libTree.so
#30 0x0000150c2054fc64 in TBranchElement::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libTree.so
#31 0x0000150c2054fc1d in TBranchElement::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/external/el9_amd64_gcc12/lib/libTree.so
#32 0x0000150bccdd029c in edm::RootTree::getEntry(TBranch*, long long) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/pluginIOPoolInput.so
#33 0x0000150bccdb0e3c in edm::RootDelayedReader::getProduct_(edm::BranchID const&, edm::EDProductGetter const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/pluginIOPoolInput.so
#34 0x0000150c20a9420f in edm::DelayedReader::getProduct(edm::BranchID const&, edm::EDProductGetter const*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#35 0x0000150c20b3fa2b in edm::DelayedReaderInputProductResolver::prefetchAsync_(edm::WaitingTaskHolder, edm::Principal const&, bool, edm::ServiceToken const&, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const::{lambda()#1}::operator()() const::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#36 0x0000150c20b40ecc in edm::DelayedReaderInputProductResolver::prefetchAsync_(edm::WaitingTaskHolder, edm::Principal const&, bool, edm::ServiceToken const&, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#37 0x0000150c20b40f38 in edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::DelayedReaderInputProductResolver::prefetchAsync_(edm::WaitingTaskHolder, edm::Principal const&, bool, edm::ServiceToken const&, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::DelayedReaderInputProductResolver::prefetchAsync_(edm::WaitingTaskHolder, edm::Principal const&, bool, edm::ServiceToken const&, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const::{lambda()#1}&)::{lambda()#1}>::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so

Thread 4 (Thread 0x150bce647640 (LWP 3193664) "cmsRun"):
#3  0x0000150c196fca30 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x0000150c1eb036bb in sched_yield () from /lib64/libc.so.6
#6  0x0000150c1f0cac43 in __gthread_yield () at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_0_0_pre2_SKYLAKEAVX512-el9_amd64_gcc12/build/CMSSW_14_0_0_pre2_SKYLAKEAVX512-build/el9_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/include/c++/12.3.1/x86_64-redhat-linux-gnu/bits/gthr-default.h:693

Thread 3 (Thread 0x150bcf048640 (LWP 3193663) "cmsRun"):
#3  0x0000150c197004c0 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x0000150bca7b0b5c in gbl::GblMeasurement* std::__do_uninit_copy<__gnu_cxx::__normal_iterator<gbl::GblMeasurement const*, std::vector<gbl::GblMeasurement, std::allocator<gbl::GblMeasurement> > >, gbl::GblMeasurement*>(__gnu_cxx::__normal_iterator<gbl::GblMeasurement const*, std::vector<gbl::GblMeasurement, std::allocator<gbl::GblMeasurement> > >, __gnu_cxx::__normal_iterator<gbl::GblMeasurement const*, std::vector<gbl::GblMeasurement, std::allocator<gbl::GblMeasurement> > >, gbl::GblMeasurement*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libAlignmentReferenceTrajectories.so
#6  0x0000150bca7c0985 in ReferenceTrajectory::addMaterialEffectsLocalGbl(std::vector<CLHEP::HepMatrix, std::allocator<CLHEP::HepMatrix> > const&, std::vector<CLHEP::HepMatrix, std::allocator<CLHEP::HepMatrix> > const&, std::vector<CLHEP::HepSymMatrix, std::allocator<CLHEP::HepSymMatrix> > const&, std::vector<CLHEP::HepSymMatrix, std::allocator<CLHEP::HepSymMatrix> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libAlignmentReferenceTrajectories.so
#7  0x0000150bca7b5b9d in ReferenceTrajectory::construct(TrajectoryStateOnSurface const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, MagneticField const*, reco::BeamSpot const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libAlignmentReferenceTrajectories.so
#8  0x0000150bca7b6ea8 in ReferenceTrajectory::ReferenceTrajectory(TrajectoryStateOnSurface const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, MagneticField const*, reco::BeamSpot const&, ReferenceTrajectoryBase::Config const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libAlignmentReferenceTrajectories.so
#9  0x0000150bca7cc977 in TwoBodyDecayTrajectory::construct(TwoBodyDecayTrajectoryState const&, std::pair<std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > > const&, MagneticField const*, reco::BeamSpot const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libAlignmentReferenceTrajectories.so
#10 0x0000150bca7cdeb8 in TwoBodyDecayTrajectory::TwoBodyDecayTrajectory(TwoBodyDecayTrajectoryState const&, std::pair<std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > > const&, MagneticField const*, reco::BeamSpot const&, ReferenceTrajectoryBase::Config const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libAlignmentReferenceTrajectories.so
#11 0x0000150bbebeea3b in TwoBodyDecayTrajectoryFactory::constructTrajectories(std::vector<std::pair<Trajectory const*, reco::Track const*>, std::allocator<std::pair<Trajectory const*, reco::Track const*> > > const&, TwoBodyDecay const&, MagneticField const*, reco::BeamSpot const&, bool) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/pluginAlignmentReferenceTrajectoriesPlugins.so
#12 0x0000150bbebec05f in TwoBodyDecayTrajectoryFactory::trajectories(edm::EventSetup const&, std::vector<std::pair<Trajectory const*, reco::Track const*>, std::allocator<std::pair<Trajectory const*, reco::Track const*> > > const&, reco::BeamSpot const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/pluginAlignmentReferenceTrajectoriesPlugins.so
#13 0x0000150bbecac72e in MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so
#14 0x0000150bcac98382 in AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libAlignmentCommonAlignmentProducer.so
#15 0x0000150c20bacb6c in edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#16 0x0000150c20b9590c in edm::WorkerT<edm::one::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#17 0x0000150c20b25c79 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#18 0x0000150c20b26d5f in edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so

Thread 1 (Thread 0x150c1e4c8640 (LWP 3193616) "cmsRun"):
#3  0x0000150c196fca30 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x0000150c1eb61560 in __lll_lock_wait () from /lib64/libc.so.6
#6  0x0000150c1eb67c7d in pthread_mutex_lock@@GLIBC_2.2.5 () from /lib64/libc.so.6
#7  0x0000150c20b3f6dd in edm::DelayedReaderInputProductResolver::resolveProduct_(edm::Principal const&, bool, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#8  0x0000150c20b40d9c in edm::DelayedReaderInputProductResolver::resolveProduct_(edm::Principal const&, bool, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#9  0x0000150c20aa7156 in edm::EventPrincipal::getByProductID(edm::ProductID const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#10 0x0000150c20aa73c6 in edm::EventPrincipal::getIt(edm::ProductID const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#11 0x0000150c2086d7e7 in edm::RefCore::tryToGetProductPtr(std::type_info const&, edm::EDProductGetter const*) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libDataFormatsCommon.so
#12 0x0000150bcac89a56 in edm::refitem::GetRefPtrImpl<std::vector<reco::TrackExtra, std::allocator<reco::TrackExtra> >, reco::TrackExtra, edm::refhelper::FindUsingAdvance<std::vector<reco::TrackExtra, std::allocator<reco::TrackExtra> >, reco::TrackExtra>, unsigned int>::getRefPtr_(edm::RefCore const&, unsigned int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libAlignmentCommonAlignmentProducer.so
#13 0x0000150bcac89b13 in reco::Track::recHits() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libAlignmentCommonAlignmentProducer.so
#14 0x0000150bcacb31f1 in AlignmentTrackSelector::detailedHitsCheck(reco::Track const*, edm::Event const&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libAlignmentCommonAlignmentProducer.so
#15 0x0000150bcacb4dec in AlignmentTrackSelector::basicCuts(std::vector<reco::Track const*, std::allocator<reco::Track const*> > const&, edm::Event const&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libAlignmentCommonAlignmentProducer.so
#16 0x0000150bcacb5e19 in AlignmentTrackSelector::select(std::vector<reco::Track const*, std::allocator<reco::Track const*> > const&, edm::Event const&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libAlignmentCommonAlignmentProducer.so
#17 0x0000150bcad177ad in ObjectSelectorBase<TrackConfigSelector, std::vector<reco::Track, std::allocator<reco::Track> >, NonNullNumberSelector, helper::NullPostProcessor<std::vector<reco::Track, std::allocator<reco::Track> > >, helper::TrackCollectionStoreManager, helper::TrackSelectorBase, reco::modules::NoEventSetupInit<TrackConfigSelector> >::filter(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/pluginCommonAlignmentProducerSelectors.so
#18 0x0000150c20bb76ff in edm::stream::EDFilterAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#19 0x0000150c20b9c98c in edm::WorkerT<edm::stream::EDFilterAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#20 0x0000150c20b25c79 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so
#21 0x0000150c20b261e4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02821/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/lib/el9_amd64_gcc12/scram_x86-64-v3/libFWCoreFramework.so

Module: AlignmentProducerAsAnalyzer:SiPixelAliMilleAlignmentProducerHGDimuon (crashed)
Module: none
Module: none
Module: AlignmentTrackSelectorModule:ALCARECOPixelLACalibrationTracks

https://cmssdt.cern.ch/SDT/cgi-bin/logreader/el9_amd64_gcc12/CMSSW_14_0_MULTIARCHS_X_2024-01-25-2300/pyRelValMatrixLogs/run/1001.3_RunSingleMuon2022B/step3_RunSingleMuon2022B.log#/