cmssw: Assertion failures in edm::service::Timing

Many workflows (but not all) in CMSSW_11_3_X_2021-03-08-1800 are failing along

cmsRun: /data/cmsbld/jenkins/workspace/build-any-ib/w/tmp/BUILDROOT/6d143f7bbaf77910d1f5a061987aa3a1/opt/cmssw/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/src/FWCore/Services/plugins/Timing.cc:192: double edm::service::popStack(): Assertion `!modStack.empty()' failed.

Thread 10 (Thread 0x2ac651000700 (LWP 3130)):
#2  0x00002ac622d1cfd0 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00002ac617f24d84 in _dl_fixup () from /lib64/ld-linux-x86-64.so.2
#5  0x00002ac617f2c9ea in _dl_runtime_resolve_xsave () from /lib64/ld-linux-x86-64.so.2
#6  0x00002ac66192cb5f in EcalTrigPrimProducer::beginRun(edm::Run const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginSimCalorimetryEcalTrigPrimProducersPlugins.so
#7  0x00002ac61842fd8d in edm::stream::ProducingModuleAdaptorBase<edm::stream::EDProducerBase>::doStreamBeginRun(edm::StreamID, edm::RunTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#8  0x00002ac6184031a0 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDoStreamBegin(edm::StreamID, edm::RunTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#9  0x00002ac618304df2 in decltype ({parm#1}()) edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#10 0x00002ac61830503f in bool edm::Worker::runModule<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#11 0x00002ac6183051c0 in edm::Worker::doWorkNoPrefetchingAsync<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::WaitingTaskHolder, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::ServiceToken const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#12 0x00002ac61830529d in tbb::internal::function_task<edm::Worker::doWorkNoPrefetchingAsync<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::WaitingTaskHolder, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::ServiceToken const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so

Thread 9 (Thread 0x2ac650001700 (LWP 3129)):
#3  0x00002ac622d1eed2 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x00002ac61ab69387 in raise () from /lib64/libc.so.6
#6  0x00002ac61ab6aa78 in abort () from /lib64/libc.so.6
#7  0x00002ac61ab621a6 in __assert_fail_base () from /lib64/libc.so.6
#8  0x00002ac61ab62252 in __assert_fail () from /lib64/libc.so.6
#9  0x00002ac622d48dcb in edm::service::Timing::postCommon() const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginFWCoreServicesPlugins.so
#10 0x00002ac618304e23 in decltype ({parm#1}()) edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#11 0x00002ac61830503f in bool edm::Worker::runModule<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#12 0x00002ac6183051c0 in edm::Worker::doWorkNoPrefetchingAsync<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::WaitingTaskHolder, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::ServiceToken const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#13 0x00002ac61830529d in tbb::internal::function_task<edm::Worker::doWorkNoPrefetchingAsync<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::WaitingTaskHolder, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::ServiceToken const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so

Thread 8 (Thread 0x2ac64f600700 (LWP 3128)):
#2  0x00002ac622d1cfd0 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00002ac617f1f7c5 in do_lookup_x () from /lib64/ld-linux-x86-64.so.2
#5  0x00002ac617f2009f in _dl_lookup_symbol_x () from /lib64/ld-linux-x86-64.so.2
#6  0x00002ac617f24dee in _dl_fixup () from /lib64/ld-linux-x86-64.so.2
#7  0x00002ac617f2c9ea in _dl_runtime_resolve_xsave () from /lib64/ld-linux-x86-64.so.2
#8  0x00002ac66192cbd4 in EcalTrigPrimProducer::beginRun(edm::Run const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginSimCalorimetryEcalTrigPrimProducersPlugins.so
#9  0x00002ac61842fd8d in edm::stream::ProducingModuleAdaptorBase<edm::stream::EDProducerBase>::doStreamBeginRun(edm::StreamID, edm::RunTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#10 0x00002ac6184031a0 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDoStreamBegin(edm::StreamID, edm::RunTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#11 0x00002ac618304df2 in decltype ({parm#1}()) edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#12 0x00002ac61830503f in bool edm::Worker::runModule<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#13 0x00002ac6183051c0 in edm::Worker::doWorkNoPrefetchingAsync<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::WaitingTaskHolder, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::ServiceToken const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so
#14 0x00002ac61830529d in tbb::internal::function_task<edm::Worker::doWorkNoPrefetchingAsync<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1> >(edm::WaitingTaskHolder, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::ServiceToken const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreFramework.so

Thread 1 (Thread 0x2ac61c82f240 (LWP 2190)):
#2  0x00002ac622d1cfd0 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00002ac61ac26c3d in poll () from /lib64/libc.so.6
#5  0x00002ac646e3e50a in frontier_read () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/libfrontier_client.so.2
#6  0x00002ac646e3b7a9 in http_read () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/libfrontier_client.so.2
#7  0x00002ac646e3b7f3 in read_char () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/libfrontier_client.so.2
#8  0x00002ac646e3b8d1 in read_line () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/libfrontier_client.so.2
#9  0x00002ac646e3c1e1 in read_connection () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/libfrontier_client.so.2
#10 0x00002ac646e3c62f in frontierHttpClnt_post () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/libfrontier_client.so.2
#11 0x00002ac646e3c53d in frontierHttpClnt_get () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/libfrontier_client.so.2
#12 0x00002ac646e48a44 in get_data () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/libfrontier_client.so.2
#13 0x00002ac646e4926e in frontier_postRawData () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/libfrontier_client.so.2
#14 0x00002ac646e48edc in frontier_getRawData () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/libfrontier_client.so.2
#15 0x00002ac646e40a15 in frontier::Session::getData(std::vector<frontier::Request const*, std::allocator<frontier::Request const*> > const&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/libfrontier_client.so.2
#16 0x00002ac646ddbca4 in coral::FrontierAccess::Statement::execute(coral::AttributeList const&, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/liblcg_FrontierAccess.so
#17 0x00002ac646dce4b9 in coral::FrontierAccess::Query::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/external/slc7_amd64_gcc900/lib/liblcg_FrontierAccess.so
#18 0x00002ac63e31d535 in cond::persistency::PAYLOAD::Table::select(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, cond::Binary&, cond::Binary&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libCondCoreCondDB.so
#19 0x00002ac63e3322db in cond::persistency::KeyList::loadFromDB[abi:cxx11](unsigned long long) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libCondCoreCondDB.so
#20 0x00002ac640db5768 in DTKeyedConfigCache::get(cond::persistency::KeyList const&, int, DTKeyedConfig const*&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libCondToolsDT.so
#21 0x00002ac640db5da8 in DTKeyedConfigCache::getData(cond::persistency::KeyList const&, int, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libCondToolsDT.so
#22 0x00002ac640d1d7b4 in DTConfigDBProducer::readDTCCBConfig(DTConfigManagerRcd const&, DTConfigManager&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginL1TriggerConfigDTTPGConfigProducers.so
#23 0x00002ac640d21afe in DTConfigDBProducer::produce(DTConfigManagerRcd const&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginL1TriggerConfigDTTPGConfigProducers.so
#24 0x00002ac640d2f9d5 in decltype ({parm#1}()) edm::convertException::wrap<edm::eventsetup::Callback<DTConfigDBProducer, std::unique_ptr<DTConfigManager, std::default_delete<DTConfigManager> >, DTConfigManagerRcd, edm::eventsetup::CallbackSimpleDecorator<DTConfigManagerRcd> >::runProducerAsync(tbb::task_group*, std::__exception_ptr::exception_ptr const*, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, edm::ServiceToken const&)::{lambda()#1}::operator()() const::{lambda()#1}>(edm::eventsetup::Callback<DTConfigDBProducer, std::unique_ptr<DTConfigManager, std::default_delete<DTConfigManager> >, DTConfigManagerRcd, edm::eventsetup::CallbackSimpleDecorator<DTConfigManagerRcd> >::runProducerAsync(tbb::task_group*, std::__exception_ptr::exception_ptr const*, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, edm::ServiceToken const&)::{lambda()#1}::operator()() const::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginL1TriggerConfigDTTPGConfigProducers.so
#25 0x00002ac640d2fce6 in edm::eventsetup::Callback<DTConfigDBProducer, std::unique_ptr<DTConfigManager, std::default_delete<DTConfigManager> >, DTConfigManagerRcd, edm::eventsetup::CallbackSimpleDecorator<DTConfigManagerRcd> >::runProducerAsync(tbb::task_group*, std::__exception_ptr::exception_ptr const*, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, edm::ServiceToken const&)::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginL1TriggerConfigDTTPGConfigProducers.so
#26 0x00002ac640d30adf in void edm::SerialTaskQueueChain::actionToRun<edm::eventsetup::Callback<DTConfigDBProducer, std::unique_ptr<DTConfigManager, std::default_delete<DTConfigManager> >, DTConfigManagerRcd, edm::eventsetup::CallbackSimpleDecorator<DTConfigManagerRcd> >::runProducerAsync(tbb::task_group*, std::__exception_ptr::exception_ptr const*, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, edm::ServiceToken const&)::{lambda()#1}&>(edm::eventsetup::Callback<DTConfigDBProducer, std::unique_ptr<DTConfigManager, std::default_delete<DTConfigManager> >, DTConfigManagerRcd, edm::eventsetup::CallbackSimpleDecorator<DTConfigManagerRcd> >::runProducerAsync(tbb::task_group*, std::__exception_ptr::exception_ptr const*, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, edm::ServiceToken const&)::{lambda()#1}&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginL1TriggerConfigDTTPGConfigProducers.so
#27 0x00002ac640d30b51 in edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::eventsetup::Callback<DTConfigDBProducer, std::unique_ptr<DTConfigManager, std::default_delete<DTConfigManager> >, DTConfigManagerRcd, edm::eventsetup::CallbackSimpleDecorator<DTConfigManagerRcd> >::runProducerAsync(tbb::task_group*, std::__exception_ptr::exception_ptr const*, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, edm::ServiceToken const&)::{lambda()#1}>(tbb::task_group&, edm::eventsetup::Callback<DTConfigDBProducer, std::unique_ptr<DTConfigManager, std::default_delete<DTConfigManager> >, DTConfigManagerRcd, edm::eventsetup::CallbackSimpleDecorator<DTConfigManagerRcd> >::runProducerAsync(tbb::task_group*, std::__exception_ptr::exception_ptr const*, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, edm::ServiceToken const&)::{lambda()#1}&&)::{lambda()#1}>::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/pluginL1TriggerConfigDTTPGConfigProducers.so
#28 0x00002ac6180a4f29 in tbb::internal::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc900/cms/cmssw-patch/CMSSW_11_3_X_2021-03-08-1800/lib/slc7_amd64_gcc900/libFWCoreConcurrency.so

Current Modules:
Module: none (crashed)
Module: L1TCaloLayer1:simCaloStage2Layer1Digis
Module: EcalTrigPrimProducer:DMEcalTriggerPrimitiveDigis
Module: EcalTrigPrimProducer:DMEcalTriggerPrimitiveDigis

https://cmssdt.cern.ch/SDT/cgi-bin/logreader/slc7_amd64_gcc900/CMSSW_11_3_X_2021-03-08-1800/pyRelValMatrixLogs/run/250202.0_TTbar_13+TTbar_13INPUT+DIGIPRMXUP15_PU25+RECOPRMXUP15_PU25+HARVESTUP15_PU25/step2_TTbar_13+TTbar_13INPUT+DIGIPRMXUP15_PU25+RECOPRMXUP15_PU25+HARVESTUP15_PU25.log#/80-80