ClickHouse: Segfault in libunwind

https://s3.amazonaws.com/clickhouse-test-reports/0/a9f389d546359055f3180b04faf0ee8474adc371/stress_test__address__actions_.html

Finally we have caught the stacktrace:

2022-01-11 13:26:34 Thread 818 "QueryPipelineEx" received signal SIGSEGV, Segmentation fault.
2022-01-11 13:26:34 [Switching to Thread 0x7fb10f38a700 (LWP 6322)]
2022-01-11 13:26:34 libunwind::DwarfInstructions<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::getSavedRegister (addressSpace=..., registers=..., cfa=cfa@entry=4294967313, savedReg=...) at ../contrib/libunwind/src/DwarfInstructions.hpp:100
2022-01-11 13:26:34 #0  libunwind::DwarfInstructions<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::getSavedRegister (addressSpace=..., registers=..., cfa=cfa@entry=4294967313, savedReg=...) at ../contrib/libunwind/src/DwarfInstructions.hpp:100
2022-01-11 13:26:34 No locals.
2022-01-11 13:26:34 #1  0x0000000037ea986f in libunwind::DwarfInstructions<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::stepWithDwarf (addressSpace=..., pc=<optimized out>, fdeStart=<optimized out>, registers=...) at ../contrib/libunwind/src/DwarfInstructions.hpp:215
2022-01-11 13:26:34         i = 3
2022-01-11 13:26:34         newRegisters = {_registers = {__rax = <optimized out>, __rbx = <optimized out>, __rcx = <optimized out>, __rdx = <optimized out>, __rdi = <optimized out>, __rsi = <optimized out>, __rbp = <optimized out>, __rsp = <optimized out>, __r8 = <optimized out>, __r9 = <optimized out>, __r10 = <optimized out>, __r11 = <optimized out>, __r12 = <optimized out>, __r13 = <optimized out>, __r14 = <optimized out>, __r15 = <optimized out>, __rip = <optimized out>, __rflags = 140398441257872, __cs = 17549805157236, __fs = 10, __gs = 105828009323536}}
2022-01-11 13:26:34         cfa = 140398441256912
2022-01-11 13:26:34         lastReg = 32
2022-01-11 13:26:34         returnAddress = 0
2022-01-11 13:26:34         prolog = {cfaRegister = 6, cfaRegisterOffset = 16, cfaExpression = 0, spExtraArgSize = 0, codeOffsetAtStackDecrement = 1, registersInOtherRegisters = false, sameValueUsed = false, savedRegisters = {{location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterInCFA, value = -48}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterInCFA, value = -16}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterInCFA, value = -40}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterInCFA, value = -32}, {location = libunwind::kRegisterInCFA, value = -24}, {location = libunwind::kRegisterInCFA, value = -8}, {location = libunwind::kRegisterUnused, value = 0} <repeats 16 times>}}
2022-01-11 13:26:34         fdeInfo = {fdeStart = 174863768, fdeLength = 64, fdeInstructions = 174863785, pcStart = 202615776, pcEnd = 202616495, lsda = 0}
2022-01-11 13:26:34         cieInfo = {cieStart = 174806504, cieLength = 24, cieInstructions = 174806521, pointerEncoding = 27 '\033', lsdaEncoding = 255 '\377', personalityEncoding = 0 '\000', personalityOffsetInCIE = 0 '\000', personality = 0, codeAlignFactor = 1, dataAlignFactor = -8, isSignalFrame = false, fdesHaveAugmentationData = true, returnAddressRegister = 16 '\020'}
2022-01-11 13:26:34 #2  0x0000000037ea93c1 in libunwind::UnwindCursor<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::stepWithDwarfFDE (this=0x7fb10f37d7c0) at ../contrib/libunwind/src/UnwindCursor.hpp:933
2022-01-11 13:26:34 No locals.
2022-01-11 13:26:34 #3  libunwind::UnwindCursor<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::step (this=0x7fb10f37d7c0) at ../contrib/libunwind/src/UnwindCursor.hpp:1987
2022-01-11 13:26:35         result = <optimized out>
2022-01-11 13:26:35 #4  0x0000000037ea9008 in unw_backtrace (buffer=0x7fb10f37de50, size=45) at ../contrib/libunwind/src/libunwind.cpp:297
2022-01-11 13:26:35         context = {data = {140398441258560, 140398441258576, 140398441258880, 140398441258880, 140398441257160, 45, 140398441257376, 140398441256896, 5, 4, 32, 17551952608247, 17940362863843014904, 17549805157184, 140398441258576, 45, 938119046, 140398441257872, 17549805157236, 10, 105828009323536}}
2022-01-11 13:26:35         cursor = {data = {159365944, 967060832, 140398441264864, 140402494883840, 17549805158105, 32, 140398441264832, 32, 4294967297, 140398441262544, 0, 6984974018888649, 0, 83576157, 20484, 140398441264704, 140402494881792, 140398441262624, 202616187, 140398441257872, 17549805157236, 10, 105828009323536, 202615776, 202616495, 0, 0, 0, 0, 274945015808, 174863768, 202109440, 0}}
2022-01-11 13:26:35         current = 5
2022-01-11 13:26:35         ip = <optimized out>
2022-01-11 13:26:35 #5  0x000000000c2341cd in StackTrace::tryCapture (this=0x7fb10f37de40) at ../src/Common/StackTrace.cpp:305
2022-01-11 13:26:35 No locals.
2022-01-11 13:26:35 #6  StackTrace::StackTrace (this=0x7fb10f37de40, signal_context=...) at ../src/Common/StackTrace.cpp:271
2022-01-11 13:26:35         caller_address = <optimized out>
2022-01-11 13:26:35 #7  0x0000000024eb6706 in DB::(anonymous namespace)::signalHandler (info=<optimized out>, context=<optimized out>) at ../src/Storages/System/StorageSystemStackTrace.cpp:84
2022-01-11 13:26:35         signal_context = {uc_flags = 7, uc_link = 0x0, uc_stack = {ss_sp = 0x7fb2dc13d000, ss_flags = 0, ss_size = 32768}, uc_mcontext = {gregs = {0, 6984974018888649, 0, 83576157, 140398694187104, 140398441264704, 140398441264832, 140398694187104, 140398441264832, 32, 140398441264656, 32, 32, 140398441264864, 17549805158105, 140398441262536, 202616496, 514, 12103423998558259, 0, 0, 20484, 0}, fpregs = 0x7fb10f37e280, __reserved1 = {203851955, 140398441259840, 204852388, 1172321806, 83576150, 204851136, 2897561901077514607, 4720223906611369058}}, uc_sigmask = {__val = {20484, 34, 4294967295, 544, 2155, 0 <repeats 11 times>}}, __fpregs_mem = {cwd = 0, swd = 0, ftw = 0, fop = 0, rip = 4985484787500187648, rdp = 4985484787500187648, mxcsr = 35652479, mxcr_mask = 0, _st = {{significand = {37513, 12342, 0, 0}, exponent = 0, __glibc_reserved1 = {0, 0, 0}}, {significand = {8103, 0, 65535, 0}, exponent = 0, __glibc_reserved1 = {0, 0, 0}}, {significand = {0, 0, 0, 0}, exponent = 0, __glibc_reserved1 = {0, 0, 0}}, {significand = {0, 0, 0, 0}, exponent = 0, __glibc_reserved1 = {0, 0, 0}}, {significand = {0, 0, 0, 0}, exponent = 0, __glibc_reserved1 = {0, 0, 0}}, {significand = {0, 0, 0, 0}, exponent = 0, __glibc_reserved1 = {0, 0, 0}}, {significand = {0, 0, 0, 0}, exponent = 0, __glibc_reserved1 = {0, 0, 0}}, {significand = {0, 0, 0, 0}, exponent = 55050, __glibc_reserved1 = {28835, 2621, 41943}}}, _xmm = {{element = {16376, 0, 1889785610, 2748779069}}, {element = {16377, 0, 0, 0}}, {element = {0, 0, 0, 0}}, {element = {0, 0, 1651340654, 740913765}}, {element = {1836412448, 695362914, 1885959276, 1948809580}}, {element = {1953384815, 674641202, 1720771303, 1098993586}}, {element = {0, 0, 1720771303, 1095847858}}, {element = {0, 0, 0, 0}}, {element = {0, 0, 0, 0}}, {element = {0, 0, 0, 0}}, {element = {0, 0, 4, 0}}, {element = {4, 0, 3753902080, 3196848895}}, {element = {0, 0, 0, 1127219200}}, {element = {0, 1127219200, 0, 1160773632}}, {element = {0, 1160773632, 1048576, 1160773632}}, {element = {1048576, 1160773632, 0, 1005584384}}}, __glibc_reserved1 = {0, 1005584384, 0, 0, 0, 0, 255324296, 32689, 202616521, 0, 518, 0, 51, 2818048, 0, 0, 0, 0, 1179670611, 2700, 767, 0, 2696, 0}}}
2022-01-11 13:26:35         saved_errno = 0
2022-01-11 13:26:35         notification_num = <optimized out>
2022-01-11 13:26:35         expected = false
2022-01-11 13:26:35         query_id = <optimized out>
2022-01-11 13:26:35         res = <optimized out>
2022-01-11 13:26:35 #8  <signal handler called>
2022-01-11 13:26:35 No symbol table info available.
2022-01-11 13:26:35 #9  0x000000000c13aeb0 in __asan::QuickCheckForUnpoisonedRegion(unsigned long, unsigned long) ()
2022-01-11 13:26:35 No symbol table info available.
2022-01-11 13:26:35 #10 0x000000000c13ad7b in __asan_memcpy ()
2022-01-11 13:26:35 No symbol table info available.
2022-01-11 13:26:35 #11 0x0000000016359417 in void DB::impl_::BinaryOperation<wide::integer<256ul, int>, unsigned long, DB::MultiplyImpl<wide::integer<256ul, int>, unsigned long>, wide::integer<256ul, int> >::process<(DB::impl_::OpCase)0>(wide::integer<256ul, int> const*, unsigned long const*, wide::integer<256ul, int>*, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) ()
2022-01-11 13:26:35 No symbol table info available.
2022-01-11 13:26:35 #12 0x00000000165cc789 in COW<DB::IColumn>::immutable_ptr<DB::IColumn> DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeNumeric<DB::DataTypeNumber<wide::integer<256ul, int> >, DB::DataTypeNumber<unsigned long> >(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, DB::DataTypeNumber<wide::integer<256ul, int> > const&, DB::DataTypeNumber<unsigned long> const&, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const ()
2022-01-11 13:26:35 No symbol table info available.
2022-01-11 13:26:35 #13 0x00000000165c6eeb in bool DB::castTypeToEither<DB::DataTypeNumber<char8_t>, DB::DataTypeNumber<unsigned short>, DB::DataTypeNumber<unsigned int>, DB::DataTypeNumber<unsigned long>, DB::DataTypeNumber<wide::integer<128ul, unsigned int> >, DB::DataTypeNumber<wide::integer<256ul, unsigned int> >, DB::DataTypeNumber<signed char>, DB::DataTypeNumber<short>, DB::DataTypeNumber<int>, DB::DataTypeNumber<long>, DB::DataTypeNumber<wide::integer<128ul, int> >, DB::DataTypeNumber<wide::integer<256ul, int> >, DB::DataTypeDecimal<DB::Decimal<int> >, DB::DataTypeDecimal<DB::Decimal<long> >, DB::DataTypeDecimal<DB::Decimal<wide::integer<128ul, int> > >, DB::DataTypeDecimal<DB::Decimal<wide::integer<256ul, int> > >, DB::DataTypeDate, DB::DataTypeDateTime, DB::DataTypeFixedString, DB::DataTypeString, DB::DataTypeNumber<float>, DB::DataTypeNumber<double>, DB::IDataType, DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::castBothTypes<DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const::{lambda(auto:1 const&, auto:2 const&)#1}>(DB::IDataType const*, DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const::{lambda(auto:1 const&, auto:2 const&)#1}, DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const::{lambda(auto:1 const&, auto:2 const&)#1}&&)::{lambda(auto:1 const&)#1}::operator()<DB::DataTypeNumber<wide::integer<256ul, int> > >(DB::DataTypeNumber<wide::integer<256ul, int> > const&) const::{lambda(auto:1 const&)#1}>(TypeList<DB::DataTypeNumber<char8_t>, DB::DataTypeNumber<unsigned short>, DB::DataTypeNumber<unsigned int>, DB::DataTypeNumber<unsigned long>, DB::DataTypeNumber<wide::integer<128ul, unsigned int> >, DB::DataTypeNumber<wide::integer<256ul, unsigned int> >, DB::DataTypeNumber<signed char>, DB::DataTypeNumber<short>, DB::DataTypeNumber<int>, DB::DataTypeNumber<long>, DB::DataTypeNumber<wide::integer<128ul, int> >, DB::DataTypeNumber<wide::integer<256ul, int> >, DB::DataTypeDecimal<DB::Decimal<int> >, DB::DataTypeDecimal<DB::Decimal<long> >, DB::DataTypeDecimal<DB::Decimal<wide::integer<128ul, int> > >, DB::DataTypeDecimal<DB::Decimal<wide::integer<256ul, int> > >, DB::DataTypeDate, DB::DataTypeDateTime, DB::DataTypeFixedString, DB::DataTypeString, DB::DataTypeNumber<float>, DB::DataTypeNumber<double> >, DB::IDataType const*, DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::castBothTypes<DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const::{lambda(auto:1 const&, auto:2 const&)#1}>(DB::IDataType const*, DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const::{lambda(auto:1 const&, auto:2 const&)#1}, DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const::{lambda(auto:1 const&, auto:2 const&)#1}&&)::{lambda(auto:1 const&)#1}::operator()<DB::DataTypeNumber<wide::integer<256ul, int> > >(DB::DataTypeNumber<wide::integer<256ul, int> > const&) const::{lambda(auto:1 const&)#1}&&) ()
2022-01-11 13:26:35 No symbol table info available.
2022-01-11 13:26:35 #14 0x00000000164c0fab in bool DB::castTypeToEither<DB::DataTypeNumber<char8_t>, DB::DataTypeNumber<unsigned short>, DB::DataTypeNumber<unsigned int>, DB::DataTypeNumber<unsigned long>, DB::DataTypeNumber<wide::integer<128ul, unsigned int> >, DB::DataTypeNumber<wide::integer<256ul, unsigned int> >, DB::DataTypeNumber<signed char>, DB::DataTypeNumber<short>, DB::DataTypeNumber<int>, DB::DataTypeNumber<long>, DB::DataTypeNumber<wide::integer<128ul, int> >, DB::DataTypeNumber<wide::integer<256ul, int> >, DB::DataTypeDecimal<DB::Decimal<int> >, DB::DataTypeDecimal<DB::Decimal<long> >, DB::DataTypeDecimal<DB::Decimal<wide::integer<128ul, int> > >, DB::DataTypeDecimal<DB::Decimal<wide::integer<256ul, int> > >, DB::DataTypeDate, DB::DataTypeDateTime, DB::DataTypeFixedString, DB::DataTypeString, DB::DataTypeNumber<float>, DB::DataTypeNumber<double>, DB::IDataType, DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::castBothTypes<DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const::{lambda(auto:1 const&, auto:2 const&)#1}>(DB::IDataType const*, DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const::{lambda(auto:1 const&, auto:2 const&)#1}, DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const::{lambda(auto:1 const&, auto:2 const&)#1}&&)::{lambda(auto:1 const&)#1}>(TypeList<DB::DataTypeNumber<char8_t>, DB::DataTypeNumber<unsigned short>, DB::DataTypeNumber<unsigned int>, DB::DataTypeNumber<unsigned long>, DB::DataTypeNumber<wide::integer<128ul, unsigned int> >, DB::DataTypeNumber<wide::integer<256ul, unsigned int> >, DB::DataTypeNumber<signed char>, DB::DataTypeNumber<short>, DB::DataTypeNumber<int>, DB::DataTypeNumber<long>, DB::DataTypeNumber<wide::integer<128ul, int> >, DB::DataTypeNumber<wide::integer<256ul, int> >, DB::DataTypeDecimal<DB::Decimal<int> >, DB::DataTypeDecimal<DB::Decimal<long> >, DB::DataTypeDecimal<DB::Decimal<wide::integer<128ul, int> > >, DB::DataTypeDecimal<DB::Decimal<wide::integer<256ul, int> > >, DB::DataTypeDate, DB::DataTypeDateTime, DB::DataTypeFixedString, DB::DataTypeString, DB::DataTypeNumber<float>, DB::DataTypeNumber<double> >, DB::IDataType const*, DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::castBothTypes<DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const::{lambda(auto:1 const&, auto:2 const&)#1}>(DB::IDataType const*, DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const::{lambda(auto:1 const&, auto:2 const&)#1}, DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const::{lambda(auto:1 const&, auto:2 const&)#1}&&)::{lambda(auto:1 const&)#1}&&) ()
2022-01-11 13:26:35 No symbol table info available.
2022-01-11 13:26:35 #15 0x00000000164bf5e5 in DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl2(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long, DB::PODArray<char8_t, 4096ul, Allocator<false, false>, 15ul, 16ul> const*) const ()
2022-01-11 13:26:35 No symbol table info available.
2022-01-11 13:26:35 #16 0x00000000164bc021 in DB::FunctionBinaryArithmetic<DB::MultiplyImpl, DB::NameMultiply, true, true, false>::executeImpl(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long) const ()
2022-01-11 13:26:35 No symbol table info available.
2022-01-11 13:26:35 #17 0x00000000128d10ec in DB::FunctionToExecutableFunctionAdaptor::executeImpl(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> > const&, std::__1::shared_ptr<DB::IDataType const> const&, unsigned long) const ()
2022-01-11 13:26:36 No symbol table info available.
2022-01-11 13:26:36 #18 0x0000000025203303 in DB::IExecutableFunction::executeWithoutLowCardinalityColumns (this=0x603000dcb7c0, args=..., result_type=..., input_rows_count=65505, dry_run=<optimized out>) at ../src/Functions/IFunction.cpp:216
2022-01-11 13:26:36         res = {<boost::intrusive_ptr<DB::IColumn const>> = {px = 0x0}, <No data fields>}
2022-01-11 13:26:36 #19 0x0000000025204225 in DB::IExecutableFunction::executeWithoutSparseColumns (this=<optimized out>, arguments=..., result_type=..., input_rows_count=<optimized out>, dry_run=<optimized out>) at ../src/Functions/IFunction.cpp:264
2022-01-11 13:26:36         res_low_cardinality_type = <optimized out>
2022-01-11 13:26:36         columns_without_low_cardinality = {<std::__1::__vector_base<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> >> = {<std::__1::__vector_base_common<true>> = {<No data fields>}, __begin_ = 0x608001782420, __end_ = 0x608001782480, __end_cap_ = {<std::__1::__compressed_pair_elem<DB::ColumnWithTypeAndName*, 0, false>> = {__value_ = 0x608001782480}, <std::__1::__compressed_pair_elem<std::__1::allocator<DB::ColumnWithTypeAndName>, 1, true>> = {<std::__1::allocator<DB::ColumnWithTypeAndName>> = {<No data fields>}, <No data fields>}, <No data fields>}}, <No data fields>}
2022-01-11 13:26:36         result = <optimized out>
2022-01-11 13:26:36 #20 0x0000000025207fcf in DB::IExecutableFunction::execute (this=0x20, arguments=..., result_type=..., input_rows_count=<optimized out>, dry_run=<optimized out>) at ../src/Functions/IFunction.cpp:334
2022-01-11 13:26:36         columns_without_sparse = {<std::__1::__vector_base<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> >> = {<std::__1::__vector_base_common<true>> = {<No data fields>}, __begin_ = 0x608002f5d7a0, __end_ = 0x608002f5d800, __end_cap_ = {<std::__1::__compressed_pair_elem<DB::ColumnWithTypeAndName*, 0, false>> = {__value_ = 0x608002f5d800}, <std::__1::__compressed_pair_elem<std::__1::allocator<DB::ColumnWithTypeAndName>, 1, true>> = {<std::__1::allocator<DB::ColumnWithTypeAndName>> = {<No data fields>}, <No data fields>}, <No data fields>}}, <No data fields>}
2022-01-11 13:26:36         sparse_column_position = <optimized out>
2022-01-11 13:26:36         num_full_columns = <optimized out>
2022-01-11 13:26:36         num_sparse_columns = <optimized out>
2022-01-11 13:26:36 #21 0x00000000263ac887 in DB::executeAction (action=..., execution_context=..., dry_run=false) at ../src/Interpreters/ExpressionActions.cpp:601
2022-01-11 13:26:36         arguments = {<std::__1::__vector_base<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> >> = {<std::__1::__vector_base_common<true>> = {<No data fields>}, __begin_ = 0x60800402ee20, __end_ = 0x60800402ee80, __end_cap_ = {<std::__1::__compressed_pair_elem<DB::ColumnWithTypeAndName*, 0, false>> = {__value_ = 0x60800402ee80}, <std::__1::__compressed_pair_elem<std::__1::allocator<DB::ColumnWithTypeAndName>, 1, true>> = {<std::__1::allocator<DB::ColumnWithTypeAndName>> = {<No data fields>}, <No data fields>}, <No data fields>}}, <No data fields>}
2022-01-11 13:26:36         res_column = @0x61000127f170: {column = {<boost::intrusive_ptr<DB::IColumn const>> = {px = 0x0}, <No data fields>}, type = {__ptr_ = 0x6070022567b8, __cntrl_ = 0x6070022567a0}, name = {<std::__1::__basic_string_common<true>> = {<No data fields>}, static __short_mask = 128, static __long_mask = 9223372036854775808, __r_ = {<std::__1::__compressed_pair_elem<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >::__rep, 0, false>> = {__value_ = {{__l = {__data_ = 0x604000416e50 "multiply(toInt256(number), number)", __size_ = 34, __cap_ = 9223372036854775856}, __s = {__data_ = "PnA\000@`\000\000\"\000\000\000\000\000\000\000\060\000\000\000\000\000", {<std::__1::__padding<char, 1>> = {<No data fields>}, __size_ = 128 '\200'}}, __r = {__words = {105827998461520, 34, 9223372036854775856}}}}}, <std::__1::__compressed_pair_elem<std::__1::allocator<char>, 1, true>> = {<std::__1::allocator<char>> = {<No data fields>}, <No data fields>}, <No data fields>}, static npos = 18446744073709551615}}
2022-01-11 13:26:36         inputs = <optimized out>
2022-01-11 13:26:36         columns = @0x7fb10f381148: {<std::__1::__vector_base<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> >> = {<std::__1::__vector_base_common<true>> = {<No data fields>}, __begin_ = 0x61000127f140, __end_ = 0x61000127f200, __end_cap_ = {<std::__1::__compressed_pair_elem<DB::ColumnWithTypeAndName*, 0, false>> = {__value_ = 0x61000127f200}, <std::__1::__compressed_pair_elem<std::__1::allocator<DB::ColumnWithTypeAndName>, 1, true>> = {<std::__1::allocator<DB::ColumnWithTypeAndName>> = {<No data fields>}, <No data fields>}, <No data fields>}}, <No data fields>}
2022-01-11 13:26:36         num_rows = @0x7fb10f381178: 65505
2022-01-11 13:26:36         pos = <optimized out>
2022-01-11 13:26:36         arguments = <optimized out>
2022-01-11 13:26:36         res_column = <optimized out>
2022-01-11 13:26:36         i = <optimized out>
2022-01-11 13:26:36         arg = <optimized out>
2022-01-11 13:26:36         array_join_key = <optimized out>
2022-01-11 13:26:36         array_join_key_pos = <optimized out>
2022-01-11 13:26:36         array = <optimized out>
2022-01-11 13:26:36         res_column = <optimized out>
2022-01-11 13:26:36         __range3 = <optimized out>
2022-01-11 13:26:36         __begin3 = <optimized out>
2022-01-11 13:26:36         __end3 = <optimized out>
2022-01-11 13:26:36         column = <optimized out>
2022-01-11 13:26:36         __range3 = <optimized out>
2022-01-11 13:26:36         __begin3 = <optimized out>
2022-01-11 13:26:36         __end3 = <optimized out>
2022-01-11 13:26:36         column = <optimized out>
2022-01-11 13:26:36         res_column = <optimized out>
2022-01-11 13:26:36 #22 DB::ExpressionActions::execute (this=<optimized out>, block=..., num_rows=<optimized out>, dry_run=<optimized out>) at ../src/Interpreters/ExpressionActions.cpp:718
2022-01-11 13:26:36         e = <optimized out>
2022-01-11 13:26:36         action = <optimized out>
2022-01-11 13:26:36         __range1 = @0x6120025093e8: {<std::__1::__vector_base<DB::ExpressionActions::Action, std::__1::allocator<DB::ExpressionActions::Action> >> = {<std::__1::__vector_base_common<true>> = {<No data fields>}, __begin_ = 0x6130001b62c0, __end_ = 0x6130001b63e0, __end_cap_ = {<std::__1::__compressed_pair_elem<DB::ExpressionActions::Action*, 0, false>> = {__value_ = 0x6130001b6440}, <std::__1::__compressed_pair_elem<std::__1::allocator<DB::ExpressionActions::Action>, 1, true>> = {<std::__1::allocator<DB::ExpressionActions::Action>> = {<No data fields>}, <No data fields>}, <No data fields>}}, <No data fields>}
2022-01-11 13:26:36         __begin1 = <optimized out>
2022-01-11 13:26:36         __end1 = {__i = 0x6130001b63e0}
2022-01-11 13:26:36         execution_context = {inputs = @0x7fb10f381640, columns = {<std::__1::__vector_base<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> >> = {<std::__1::__vector_base_common<true>> = {<No data fields>}, __begin_ = 0x61000127f140, __end_ = 0x61000127f200, __end_cap_ = {<std::__1::__compressed_pair_elem<DB::ColumnWithTypeAndName*, 0, false>> = {__value_ = 0x61000127f200}, <std::__1::__compressed_pair_elem<std::__1::allocator<DB::ColumnWithTypeAndName>, 1, true>> = {<std::__1::allocator<DB::ColumnWithTypeAndName>> = {<No data fields>}, <No data fields>}, <No data fields>}}, <No data fields>}, inputs_pos = {<std::__1::__vector_base<long, std::__1::allocator<long> >> = {<std::__1::__vector_base_common<true>> = {<No data fields>}, __begin_ = 0x60200065c950, __end_ = 0x60200065c958, __end_cap_ = {<std::__1::__compressed_pair_elem<long*, 0, false>> = {__value_ = 0x60200065c958}, <std::__1::__compressed_pair_elem<std::__1::allocator<long>, 1, true>> = {<std::__1::allocator<long>> = {<No data fields>}, <No data fields>}, <No data fields>}}, <No data fields>}, num_rows = 65505}
2022-01-11 13:26:36         res = {data = {<std::__1::__vector_base<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> >> = {<std::__1::__vector_base_common<true>> = {<No data fields>}, __begin_ = 0x7fb10f381460, __end_ = 0x0, __end_cap_ = {<std::__1::__compressed_pair_elem<DB::ColumnWithTypeAndName*, 0, false>> = {__value_ = 0x7fb10f3815b0}, <std::__1::__compressed_pair_elem<std::__1::allocator<DB::ColumnWithTypeAndName>, 1, true>> = {<std::__1::allocator<DB::ColumnWithTypeAndName>> = {<No data fields>}, <No data fields>}, <No data fields>}}, <No data fields>}, index_by_name = {__table_ = {__bucket_list_ = {__ptr_ = {<std::__1::__compressed_pair_elem<std::__1::__hash_node_base<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*>*>**, 0, false>> = {__value_ = 0x2558497b <DB::Block::cloneWithColumns(std::__1::vector<COW<DB::IColumn>::immutable_ptr<DB::IColumn>, std::__1::allocator<COW<DB::IColumn>::immutable_ptr<DB::IColumn> > > const&) const+1147>}, <std::__1::__compressed_pair_elem<std::__1::__bucket_list_deallocator<std::__1::allocator<std::__1::__hash_node_base<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*>*>*> >, 1, false>> = {__value_ = {__data_ = {<std::__1::__compressed_pair_elem<unsigned long, 0, false>> = {__value_ = 1172321806}, <std::__1::__compressed_pair_elem<std::__1::allocator<std::__1::__hash_node_base<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*>*>*>, 1, true>> = {<std::__1::allocator<std::__1::__hash_node_base<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*>*>*>> = {<No data fields>}, <No data fields>}, <No data fields>}}}, <No data fields>}}, __p1_ = {<std::__1::__compressed_pair_elem<std::__1::__hash_node_base<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*>*>, 0, false>> = {__value_ = {__next_ = 0x53bc7cf}}, <std::__1::__compressed_pair_elem<std::__1::allocator<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*> >, 1, true>> = {<std::__1::allocator<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*> >> = {<No data fields>}, <No data fields>}, <No data fields>}, __p2_ = {<std::__1::__compressed_pair_elem<unsigned long, 0, false>> = {__value_ = 626541824}, <std::__1::__compressed_pair_elem<std::__1::__unordered_map_hasher<std::__1::string, std::__1::__hash_value_type<std::__1::string, unsigned long>, std::__1::hash<std::__1::string>, std::__1::equal_to<std::__1::string>, true>, 1, true>> = {<std::__1::__unordered_map_hasher<std::__1::string, std::__1::__hash_value_type<std::__1::string, unsigned long>, std::__1::hash<std::__1::string>, std::__1::equal_to<std::__1::string>, true>> = {<std::__1::hash<std::__1::string>> = {<std::__1::unary_function<std::__1::string, unsigned long>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, __p3_ = {<std::__1::__compressed_pair_elem<float, 0, false>> = {__value_ = 1.08057459e-31}, <std::__1::__compressed_pair_elem<std::__1::__unordered_map_equal<std::__1::string, std::__1::__hash_value_type<std::__1::string, unsigned long>, std::__1::equal_to<std::__1::string>, std::__1::hash<std::__1::string>, true>, 1, true>> = {<std::__1::__unordered_map_equal<std::__1::string, std::__1::__hash_value_type<std::__1::string, unsigned long>, std::__1::equal_to<std::__1::string>, std::__1::hash<std::__1::string>, true>> = {<std::__1::equal_to<std::__1::string>> = {<std::__1::binary_function<std::__1::string, std::__1::string, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}}}, info = {is_overflows = true, bucket_num = 0}}
2022-01-11 13:26:36 #23 0x0000000029ddf069 in DB::FilterTransform::transform (this=<optimized out>, chunk=...) at ../src/Processors/Transforms/FilterTransform.cpp:109
2022-01-11 13:26:36         block = {data = {<std::__1::__vector_base<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> >> = {<std::__1::__vector_base_common<true>> = {<No data fields>}, __begin_ = 0x604001399890, __end_ = 0x6040013998c0, __end_cap_ = {<std::__1::__compressed_pair_elem<DB::ColumnWithTypeAndName*, 0, false>> = {__value_ = 0x6040013998c0}, <std::__1::__compressed_pair_elem<std::__1::allocator<DB::ColumnWithTypeAndName>, 1, true>> = {<std::__1::allocator<DB::ColumnWithTypeAndName>> = {<No data fields>}, <No data fields>}, <No data fields>}}, <No data fields>}, index_by_name = {__table_ = {__bucket_list_ = {__ptr_ = {<std::__1::__compressed_pair_elem<std::__1::__hash_node_base<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*>*>**, 0, false>> = {__value_ = 0x6020010c00b0}, <std::__1::__compressed_pair_elem<std::__1::__bucket_list_deallocator<std::__1::allocator<std::__1::__hash_node_base<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*>*>*> >, 1, false>> = {__value_ = {__data_ = {<std::__1::__compressed_pair_elem<unsigned long, 0, false>> = {__value_ = 2}, <std::__1::__compressed_pair_elem<std::__1::allocator<std::__1::__hash_node_base<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*>*>*>, 1, true>> = {<std::__1::allocator<std::__1::__hash_node_base<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*>*>*>> = {<No data fields>}, <No data fields>}, <No data fields>}}}, <No data fields>}}, __p1_ = {<std::__1::__compressed_pair_elem<std::__1::__hash_node_base<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*>*>, 0, false>> = {__value_ = {__next_ = 0x604002e96650}}, <std::__1::__compressed_pair_elem<std::__1::allocator<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*> >, 1, true>> = {<std::__1::allocator<std::__1::__hash_node<std::__1::__hash_value_type<std::__1::string, unsigned long>, void*> >> = {<No data fields>}, <No data fields>}, <No data fields>}, __p2_ = {<std::__1::__compressed_pair_elem<unsigned long, 0, false>> = {__value_ = 1}, <std::__1::__compressed_pair_elem<std::__1::__unordered_map_hasher<std::__1::string, std::__1::__hash_value_type<std::__1::string, unsigned long>, std::__1::hash<std::__1::string>, std::__1::equal_to<std::__1::string>, true>, 1, true>> = {<std::__1::__unordered_map_hasher<std::__1::string, std::__1::__hash_value_type<std::__1::string, unsigned long>, std::__1::hash<std::__1::string>, std::__1::equal_to<std::__1::string>, true>> = {<std::__1::hash<std::__1::string>> = {<std::__1::unary_function<std::__1::string, unsigned long>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, __p3_ = {<std::__1::__compressed_pair_elem<float, 0, false>> = {__value_ = 1}, <std::__1::__compressed_pair_elem<std::__1::__unordered_map_equal<std::__1::string, std::__1::__hash_value_type<std::__1::string, unsigned long>, std::__1::equal_to<std::__1::string>, std::__1::hash<std::__1::string>, true>, 1, true>> = {<std::__1::__unordered_map_equal<std::__1::string, std::__1::__hash_value_type<std::__1::string, unsigned long>, std::__1::equal_to<std::__1::string>, std::__1::hash<std::__1::string>, true>> = {<std::__1::equal_to<std::__1::string>> = {<std::__1::binary_function<std::__1::string, std::__1::string, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}}}, info = {is_overflows = false, bucket_num = -1}}
2022-01-11 13:26:36         columns = {<std::__1::__vector_base<COW<DB::IColumn>::immutable_ptr<DB::IColumn>, std::__1::allocator<COW<DB::IColumn>::immutable_ptr<DB::IColumn> > >> = {<std::__1::__vector_base_common<true>> = {<No data fields>}, __begin_ = 0x6020006a29f0, __end_ = 0x6020006a29f0, __end_cap_ = {<std::__1::__compressed_pair_elem<COW<DB::IColumn>::immutable_ptr<DB::IColumn>*, 0, false>> = {__value_ = 0x6020006a29f8}, <std::__1::__compressed_pair_elem<std::__1::allocator<COW<DB::IColumn>::immutable_ptr<DB::IColumn> >, 1, true>> = {<std::__1::allocator<COW<DB::IColumn>::immutable_ptr<DB::IColumn> >> = {<No data fields>}, <No data fields>}, <No data fields>}}, <No data fields>}
2022-01-11 13:26:36         filter_and_holder = {data = 0x7fb10f381860, data_holder = {<boost::intrusive_ptr<DB::IColumn const>> = {px = 0xff6a1e682f4}, <No data fields>}}
2022-01-11 13:26:36         num_rows_before_filtration = <optimized out>
2022-01-11 13:26:36         num_columns = <optimized out>
2022-01-11 13:26:36         filter_column = <optimized out>
2022-01-11 13:26:36         first_non_constant_column = <optimized out>
2022-01-11 13:26:36         num_filtered_rows = <optimized out>
2022-01-11 13:26:36 #24 0x000000002015b77e in DB::ISimpleTransform::transform (this=0x7fb10f37f6c0, input_chunk=..., output_chunk=...) at ../src/Processors/ISimpleTransform.h:37
2022-01-11 13:26:36 No locals.
2022-01-11 13:26:36 #25 0x00000000298ab318 in DB::ISimpleTransform::work (this=0x614001eca658) at ../src/Processors/ISimpleTransform.cpp:89
2022-01-11 13:26:36 No locals.
2022-01-11 13:26:36 #26 0x000000002990d806 in DB::executeJob (processor=0x614001eca658) at ../src/Processors/Executors/ExecutionThreadContext.cpp:45
2022-01-11 13:26:36         exception = <optimized out>
2022-01-11 13:26:36 #27 DB::ExecutionThreadContext::executeTask (this=<optimized out>) at ../src/Processors/Executors/ExecutionThreadContext.cpp:63
2022-01-11 13:26:36 No locals.
2022-01-11 13:26:36 #28 0x00000000298edce7 in DB::PipelineExecutor::executeStepImpl (this=<optimized out>, thread_num=<optimized out>, yield_flag=<optimized out>) at ../src/Processors/Executors/PipelineExecutor.cpp:213
2022-01-11 13:26:37         yield = false
2022-01-11 13:26:37         context = @0x60f000cde820: {async_tasks = {c = {<std::__1::__deque_base<DB::ExecutingGraph::Node*, std::__1::allocator<DB::ExecutingGraph::Node*> >> = {<std::__1::__deque_base_common<true>> = {<No data fields>}, static __block_size = <optimized out>, __map_ = {<std::__1::__split_buffer_common<true>> = {<No data fields>}, __first_ = 0x0, __begin_ = 0x0, __end_ = 0x0, __end_cap_ = {<std::__1::__compressed_pair_elem<DB::ExecutingGraph::Node***, 0, false>> = {__value_ = 0x0}, <std::__1::__compressed_pair_elem<std::__1::allocator<DB::ExecutingGraph::Node**>, 1, true>> = {<std::__1::allocator<DB::ExecutingGraph::Node**>> = {<No data fields>}, <No data fields>}, <No data fields>}}, __start_ = 0, __size_ = {<std::__1::__compressed_pair_elem<unsigned long, 0, false>> = {__value_ = 0}, <std::__1::__compressed_pair_elem<std::__1::allocator<DB::ExecutingGraph::Node*>, 1, true>> = {<std::__1::allocator<DB::ExecutingGraph::Node*>> = {<No data fields>}, <No data fields>}, <No data fields>}}, <No data fields>}}, has_async_tasks = {<std::__1::__atomic_base<bool, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<bool>> = {__a_value = false}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, condvar = {__cv_ = {__data = {{__wseq = 0, __wseq32 = {__low = 0, __high = 0}}, {__g1_start = 0, __g1_start32 = {__low = 0, __high = 0}}, __g_refs = {0, 0}, __g_size = {0, 0}, __g1_orig_size = 0, __wrefs = 0, __g_signals = {0, 0}}, __size = '\000' <repeats 47 times>, __align = 0}}, mutex = {__m_ = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}}, wake_flag = false, node = 0x611003239c00, exception = {__ptr_ = 0x0}, thread_number = 17}
2022-01-11 13:26:37 #29 0x00000000298f09e6 in DB::PipelineExecutor::executeSingleThread (this=0x61300462f7d8, thread_num=32) at ../src/Processors/Executors/PipelineExecutor.cpp:178
2022-01-11 13:26:37 No locals.
2022-01-11 13:26:37 #30 DB::PipelineExecutor::executeImpl(unsigned long)::$_1::operator()() const (this=<optimized out>) at ../src/Processors/Executors/PipelineExecutor.cpp:311
2022-01-11 13:26:37         scope_exit307 = {static is_nullable = false, function = {<No data fields>}}
2022-01-11 13:26:37 #31 std::__1::__invoke_constexpr<DB::PipelineExecutor::executeImpl(unsigned long)::$_1&> (__f=...) at ../contrib/libcxx/include/type_traits:3682
2022-01-11 13:26:37 No locals.
2022-01-11 13:26:37 #32 std::__1::__apply_tuple_impl<DB::PipelineExecutor::executeImpl(unsigned long)::$_1&, std::__1::tuple<>&>(DB::PipelineExecutor::executeImpl(unsigned long)::$_1&, std::__1::tuple<>&, std::__1::__tuple_indices<>) (__f=..., __t=...) at ../contrib/libcxx/include/tuple:1415
2022-01-11 13:26:37 No locals.
2022-01-11 13:26:37 #33 std::__1::apply<DB::PipelineExecutor::executeImpl(unsigned long)::$_1&, std::__1::tuple<>&>(DB::PipelineExecutor::executeImpl(unsigned long)::$_1&, std::__1::tuple<>&) (__f=..., __t=...) at ../contrib/libcxx/include/tuple:1424
2022-01-11 13:26:37 No locals.
2022-01-11 13:26:37 #34 ThreadFromGlobalPool::ThreadFromGlobalPool<DB::PipelineExecutor::executeImpl(unsigned long)::$_1>(DB::PipelineExecutor::executeImpl(unsigned long)::$_1&&)::{lambda()#1}::operator()() (this=<optimized out>) at ../src/Common/ThreadPool.h:188
2022-01-11 13:26:38         function = {this = 0x61300462f7d8, thread_group = {__ptr_ = 0x614005376658, __cntrl_ = 0x614005376640}, thread_num = 17}
2022-01-11 13:26:40         thread_status = {<boost::noncopyable_::noncopyable> = {<boost::noncopyable_::base_token> = {<No data fields>}, <No data fields>}, thread_id = 6322, os_thread_priority = 0, performance_counters = {counters = 0x61c006972080, counters_holder = {__ptr_ = {<std::__1::__compressed_pair_elem<std::__1::atomic<unsigned long>*, 0, false>> = {__value_ = 0x61c006972080}, <std::__1::__compressed_pair_elem<std::__1::default_delete<std::__1::atomic<unsigned long> []>, 1, true>> = {<std::__1::default_delete<std::__1::atomic<unsigned long> []>> = {<No data fields>}, <No data fields>}, <No data fields>}}, parent = 0x614005376680, level = VariableContext::Thread, static num_counters = 238}, memory_tracker = {amount = {<std::__1::__atomic_base<long, true>> = {<std::__1::__atomic_base<long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<long>> = {__a_value = 5766952}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, peak = {<std::__1::__atomic_base<long, true>> = {<std::__1::__atomic_base<long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<long>> = {__a_value = 5766952}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, hard_limit = {<std::__1::__atomic_base<long, true>> = {<std::__1::__atomic_base<long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, profiler_limit = {<std::__1::__atomic_base<long, true>> = {<std::__1::__atomic_base<long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, profiler_step = 0, fault_probability = 0, sample_probability = 0, parent = {<std::__1::__atomic_base<MemoryTracker*, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<MemoryTracker*>> = {__a_value = 0x6140053766a0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, metric = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 77}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, description_ptr = {<std::__1::__atomic_base<char const*, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<char const*>> = {__a_value = 0x6744420 <str> "(for thread)"}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, static USAGE_EVENT_NAME = <optimized out>, level = VariableContext::Thread}, untracked_memory = 0, untracked_memory_limit = 1048576, progress_in = {read_rows = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 4847370}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, read_bytes = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 38778960}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, read_raw_bytes = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, total_rows_to_read = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, total_raw_bytes_to_read = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, written_rows = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, written_bytes = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}}, progress_out = {read_rows = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, read_bytes = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, read_raw_bytes = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, total_rows_to_read = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, total_raw_bytes_to_read = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, written_rows = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, written_bytes = {<std::__1::__atomic_base<unsigned long, true>> = {<std::__1::__atomic_base<unsigned long, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<unsigned long>> = {__a_value = 0}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}}, deleter = {<std::__1::__function::__maybe_derive_from_unary_function<void ()>> = {<No data fields>}, <std::__1::__function::__maybe_derive_from_binary_function<void ()>> = {<No data fields>}, __f_ = {__buf_ = {__small = "\200\337\256'", '\000' <repeats 11 times>, __large = 0x27aedf80 <DB::CurrentThread::defaultThreadDeleter()>}, __invoker_ = {__call_ = 0x27aff3c0 <std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<void (*)(), void ()> >(std::__1::__function::__policy_storage const*)>}, __policy_ = 0x6744c28 <std::__1::__function::__policy::__choose_policy<std::__1::__function::__default_alloc_func<void (*)(), void ()> >(std::__1::integral_constant<bool, true>)::__policy_>}}, thread_trace_context = {trace_id = {t = {items = {0, 0}}}, span_id = 0, tracestate = {<std::__1::__basic_string_common<true>> = {<No data fields>}, static __short_mask = 128, static __long_mask = 9223372036854775808, __r_ = {<std::__1::__compressed_pair_elem<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >::__rep, 0, false>> = {__value_ = {{__l = {__data_ = 0x0, __size_ = 0, __cap_ = 0}, __s = {__data_ = '\000' <repeats 22 times>, {<std::__1::__padding<char, 1>> = {<No data fields>}, __size_ = 0 '\000'}}, __r = {__words = {0, 0, 0}}}}}, <std::__1::__compressed_pair_elem<std::__1::allocator<char>, 1, true>> = {<std::__1::allocator<char>> = {<No data fields>}, <No data fields>}, <No data fields>}, static npos = 18446744073709551615}, trace_flags = 0 '\000'}, thread_group = {__ptr_ = 0x614005376658, __cntrl_ = 0x614005376640}, thread_state = {<std::__1::__atomic_base<int, true>> = {<std::__1::__atomic_base<int, false>> = {__a_ = {<std::__1::__cxx_atomic_base_impl<int>> = {__a_value = 1}, <No data fields>}, static is_always_lock_free = <optimized out>}, <No data fields>}, <No data fields>}, global_context = {__ptr_ = 0x624000052100, __cntrl_ = 0x603000095b60}, query_context = {__ptr_ = 0x624002000100, __cntrl_ = 0x603001452880}, query_id = {<std::__1::__basic_string_common<true>> = {<No data fields>}, static __short_mask = 128, static __long_mask = 9223372036854775808, __r_ = {<std::__1::__compressed_pair_elem<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >::__rep, 0, false>> = {__value_ = {{__l = {__data_ = 0x60400333e650 "9d69cb63-fe52-4264-9cd7-b145beb65201", __size_ = 36, __cap_ = 9223372036854775856}, __s = {__data_ = "P\346\063\003@`\000\000$\000\000\000\000\000\000\000\060\000\000\000\000\000", {<std::__1::__padding<char, 1>> = {<No data fields>}, __size_ = 128 '\200'}}, __r = {__words = {105828047906384, 36, 9223372036854775856}}}}}, <std::__1::__compressed_pair_elem<std::__1::allocator<char>, 1, true>> = {<std::__1::allocator<char>> = {<No data fields>}, <No data fields>}, <No data fields>}, static npos = 18446744073709551615}, logs_queue_ptr = {__ptr_ = 0x61100406f918, __cntrl_ = 0x61100406f900}, profile_queue_ptr = {__ptr_ = 0x611003244258, __cntrl_ = 0x611003244240}, performance_counters_finalized = false, query_start_time_nanoseconds = 1641896773803597000, query_start_time_microseconds = 1641896773803597, query_start_time = 1641896773, queries_started = 1, query_profiler_real = {__ptr_ = {<std::__1::__compressed_pair_elem<DB::QueryProfilerReal*, 0, false>> = {__value_ = 0x0}, <std::__1::__compressed_pair_elem<std::__1::default_delete<DB::QueryProfilerReal>, 1, true>> = {<std::__1::default_delete<DB::QueryProfilerReal>> = {<No data fields>}, <No data fields>}, <No data fields>}}, query_profiler_cpu = {__ptr_ = {<std::__1::__compressed_pair_elem<DB::QueryProfilerCPU*, 0, false>> = {__value_ = 0x0}, <std::__1::__compressed_pair_elem<std::__1::default_delete<DB::QueryProfilerCPU>, 1, true>> = {<std::__1::default_delete<DB::QueryProfilerCPU>> = {<No data fields>}, <No data fields>}, <No data fields>}}, log = 0x606000060e60, last_rusage = {__ptr_ = {<std::__1::__compressed_pair_elem<DB::RUsageCounters*, 0, false>> = {__value_ = 0x604003bc29d0}, <std::__1::__compressed_pair_elem<std::__1::default_delete<DB::RUsageCounters>, 1, true>> = {<std::__1::default_delete<DB::RUsageCounters>> = {<No data fields>}, <No data fields>}, <No data fields>}}, taskstats = {__ptr_ = {<std::__1::__compressed_pair_elem<DB::TasksStatsCounters*, 0, false>> = {__value_ = 0x6130005fd900}, <std::__1::__compressed_pair_elem<std::__1::default_delete<DB::TasksStatsCounters>, 1, true>> = {<std::__1::default_delete<DB::TasksStatsCounters>> = {<No data fields>}, <No data fields>}, <No data fields>}}, fatal_error_callback = {<std::__1::__function::__maybe_derive_from_unary_function<void ()>> = {<No data fields>}, <std::__1::__function::__maybe_derive_from_binary_function<void ()>> = {<No data fields>}, __f_ = {__buf_ = {__small = "\200\n\203\003\240a\000\000\000\000\000\000\000\000\000", __large = 0x61a003830a80}, __invoker_ = {__call_ = 0x29812ea0 <std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<DB::TCPHandler::runImpl()::$_2, void ()> >(std::__1::__function::__policy_storage const*)>}, __policy_ = 0x68cfde0 <std::__1::__function::__policy::__choose_policy<std::__1::__function::__default_alloc_func<DB::TCPHandler::runImpl()::$_2, void ()> >(std::__1::integral_constant<bool, true>)::__policy_>}}, query_profiler_enabled = true}
2022-01-11 13:26:40         event = {__ptr_ = 0x60c000da9c98, __cntrl_ = 0x60c000da9c80}
2022-01-11 13:26:40         scope_exit176 = <optimized out>
2022-01-11 13:26:40         arguments = <optimized out>
2022-01-11 13:26:40 #35 std::__1::__invoke<ThreadFromGlobalPool::ThreadFromGlobalPool<DB::PipelineExecutor::executeImpl(unsigned long)::$_1>(DB::PipelineExecutor::executeImpl(unsigned long)::$_1&&)::{lambda()#1}&> (__f=...) at ../contrib/libcxx/include/type_traits:3676
2022-01-11 13:26:40 No locals.
2022-01-11 13:26:40 #36 std::__1::__invoke_void_return_wrapper<void>::__call<ThreadFromGlobalPool::ThreadFromGlobalPool<DB::PipelineExecutor::executeImpl(unsigned long)::$_1>(DB::PipelineExecutor::executeImpl(unsigned long)::$_1&&)::{lambda()#1}&>(ThreadFromGlobalPool::ThreadFromGlobalPool<DB::PipelineExecutor::executeImpl(unsigned long)::$_1>(DB::PipelineExecutor::executeImpl(unsigned long)::$_1&&)::{lambda()#1}&) (__args=...) at ../contrib/libcxx/include/__functional_base:348
2022-01-11 13:26:40 No locals.
2022-01-11 13:26:40 #37 std::__1::__function::__default_alloc_func<ThreadFromGlobalPool::ThreadFromGlobalPool<DB::PipelineExecutor::executeImpl(unsigned long)::$_1>(DB::PipelineExecutor::executeImpl(unsigned long)::$_1&&)::{lambda()#1}, void ()>::operator()() (this=<optimized out>) at ../contrib/libcxx/include/functional:1608
2022-01-11 13:26:40 No locals.
2022-01-11 13:26:40 #38 std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<ThreadFromGlobalPool::ThreadFromGlobalPool<DB::PipelineExecutor::executeImpl(unsigned long)::$_1>(DB::PipelineExecutor::executeImpl(unsigned long)::$_1&&)::{lambda()#1}, void ()> >(std::__1::__function::__policy_storage const*) (__buf=<optimized out>) at ../contrib/libcxx/include/functional:2089
2022-01-11 13:26:40         __f = <optimized out>
2022-01-11 13:26:40 #39 0x000000000c28b4c7 in std::__1::__function::__policy_func<void ()>::operator()() const (this=0x7fb10f3825a0) at ../contrib/libcxx/include/functional:2221
2022-01-11 13:26:40 No locals.
2022-01-11 13:26:40 #40 std::__1::function<void ()>::operator()() const (this=0x7fb10f3825a0) at ../contrib/libcxx/include/functional:2560
2022-01-11 13:26:40 No locals.
2022-01-11 13:26:40 #41 ThreadPoolImpl<std::__1::thread>::worker (this=<optimized out>, thread_it=...) at ../src/Common/ThreadPool.cpp:274
2022-01-11 13:26:40         metric_active_threads = {what = <optimized out>, amount = 1}
2022-01-11 13:26:40         job = {<std::__1::__function::__maybe_derive_from_unary_function<void ()>> = {<No data fields>}, <std::__1::__function::__maybe_derive_from_binary_function<void ()>> = {<No data fields>}, __f_ = {__buf_ = {__small = "pL\033\003p`\000\000\000\000\000\000\000\000\000", __large = 0x6070031b4c70}, __invoker_ = {__call_ = 0x298f05c0 <std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<ThreadFromGlobalPool::ThreadFromGlobalPool<DB::PipelineExecutor::executeImpl(unsigned long)::$_1>(DB::PipelineExecutor::executeImpl(unsigned long)::$_1&&)::{lambda()#1}, void ()> >(std::__1::__function::__policy_storage const*)>}, __policy_ = 0x68dee60 <std::__1::__function::__policy::__choose_policy<std::__1::__function::__default_alloc_func<ThreadFromGlobalPool::ThreadFromGlobalPool<DB::PipelineExecutor::executeImpl(unsigned long)::$_1>(DB::PipelineExecutor::executeImpl(unsigned long)::$_1&&)::{lambda()#1}, void ()> >(std::__1::integral_constant<bool, false>)::__policy_>}}
2022-01-11 13:26:40         need_shutdown = <optimized out>
2022-01-11 13:26:40         metric_all_threads = {what = <optimized out>, amount = 1}
2022-01-11 13:26:40 #42 0x000000000c293837 in ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, int, std::__1::optional<unsigned long>)::{lambda()#2}::operator()() const (this=<optimized out>) at ../src/Common/ThreadPool.cpp:139
2022-01-11 13:26:40 No locals.
2022-01-11 13:26:40 #43 std::__1::__invoke<ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, int, std::__1::optional<unsigned long>)::{lambda()#2}> (__f=...) at ../contrib/libcxx/include/type_traits:3676
2022-01-11 13:26:40 No locals.
2022-01-11 13:26:40 #44 std::__1::__thread_execute<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, int, std::__1::optional<unsigned long>)::{lambda()#2}>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, int, std::__1::optional<unsigned long>)::{lambda()#2}>&, std::__1::__tuple_indices<>) (__t=...) at ../contrib/libcxx/include/thread:280
2022-01-11 13:26:40 No locals.
2022-01-11 13:26:40 #45 std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, int, std::__1::optional<unsigned long>)::{lambda()#2}> >(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, int, std::__1::optional<unsigned long>)::{lambda()#2}>) (__vp=<optimized out>) at ../contrib/libcxx/include/thread:291
2022-01-11 13:26:40         __p = {__ptr_ = {<std::__1::__compressed_pair_elem<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, (lambda at ../src/Common/ThreadPool.cpp:139:42)> *, 0, false>> = {__value_ = 0x603000761080}, <std::__1::__compressed_pair_elem<std::__1::default_delete<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, (lambda at ../src/Common/ThreadPool.cpp:139:42)> >, 1, true>> = {<std::__1::default_delete<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, (lambda at ../src/Common/ThreadPool.cpp:139:42)> >> = {<No data fields>}, <No data fields>}, <No data fields>}}
2022-01-11 13:26:40 #46 0x00007fb3c1c26609 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
2022-01-11 13:26:40 No symbol table info available.
2022-01-11 13:26:40 #47 0x00007fb3c1b4d293 in clone () from /lib/x86_64-linux-gnu/libc.so.6
2022-01-11 13:26:40 No symbol table info available.

More info:

info locals
2022-01-11 13:26:40 No locals.

info registers
2022-01-11 13:26:40 rax            0x100000011         4294967313
2022-01-11 13:26:40 rbx            0x7fb10f37d458      140398441256024
2022-01-11 13:26:40 rcx            0xffffffffffffffd0  -48
2022-01-11 13:26:40 rdx            0x7fb10f37d7d0      140398441256912
2022-01-11 13:26:40 rsi            0x39a42d60          967060832
2022-01-11 13:26:40 rdi            0x0                 0
2022-01-11 13:26:40 rbp            0x7fb10f37d3f0      0x7fb10f37d3f0
2022-01-11 13:26:40 rsp            0x7fb10f37d3e0      0x7fb10f37d3e0
2022-01-11 13:26:40 r8             0x8                 8
2022-01-11 13:26:40 r9             0xc13ad7a           202616186
2022-01-11 13:26:40 r10            0xa6c35c3           174863811
2022-01-11 13:26:40 r11            0x246               582
2022-01-11 13:26:40 r12            0x0                 0
2022-01-11 13:26:40 r13            0x7fb10f37d7d0      140398441256912
2022-01-11 13:26:40 r14            0x39a42d60          967060832
2022-01-11 13:26:40 r15            0x3                 3
2022-01-11 13:26:40 rip            0x37ea9be8          0x37ea9be8 <libunwind::DwarfInstructions<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::getSavedRegister(libunwind::LocalAddressSpace&, libunwind::Registers_x86_64 const&, unsigned long, libunwind::RegisterLocation const&)+40>
2022-01-11 13:26:40 eflags         0x10297             [ CF PF AF SF IF RF ]
2022-01-11 13:26:40 cs             0x33                51
2022-01-11 13:26:40 ss             0x2b                43
2022-01-11 13:26:40 ds             0x0                 0
2022-01-11 13:26:40 es             0x0                 0
2022-01-11 13:26:40 fs             0x0                 0
2022-01-11 13:26:40 gs             0x0                 0
2022-01-11 13:26:40 k0             0x0                 0
2022-01-11 13:26:40 k1             0x0                 0
2022-01-11 13:26:40 k2             0x0                 0
2022-01-11 13:26:40 k3             0x0                 0
2022-01-11 13:26:40 k4             0x0                 0
2022-01-11 13:26:40 k5             0x0                 0
2022-01-11 13:26:40 k6             0x0                 0
2022-01-11 13:26:40 k7             0x0                 0

disassemble /s
2022-01-11 13:26:40 Dump of assembler code for function libunwind::DwarfInstructions<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::getSavedRegister(libunwind::LocalAddressSpace&, libunwind::Registers_x86_64 const&, unsigned long, libunwind::RegisterLocation const&):
2022-01-11 13:26:40 ../contrib/libunwind/src/DwarfInstructions.hpp:
2022-01-11 13:26:40 97	in ../contrib/libunwind/src/DwarfInstructions.hpp
2022-01-11 13:26:40    0x0000000037ea9bc0 <+0>:	push   %rbp
2022-01-11 13:26:40    0x0000000037ea9bc1 <+1>:	mov    %rsp,%rbp
2022-01-11 13:26:40    0x0000000037ea9bc4 <+4>:	push   %rbx
2022-01-11 13:26:40    0x0000000037ea9bc5 <+5>:	push   %rax
2022-01-11 13:26:40    0x0000000037ea9bc6 <+6>:	mov    %rdx,%rax
2022-01-11 13:26:40    0x0000000037ea9bc9 <+9>:	mov    %rsi,%rdx
2022-01-11 13:26:40    0x0000000037ea9bcc <+12>:	mov    %rdi,%rsi
2022-01-11 13:26:40 
2022-01-11 13:26:40 98	in ../contrib/libunwind/src/DwarfInstructions.hpp
2022-01-11 13:26:40    0x0000000037ea9bcf <+15>:	mov    (%rcx),%edi
2022-01-11 13:26:40    0x0000000037ea9bd1 <+17>:	add    $0xffffffff,%edi
2022-01-11 13:26:40    0x0000000037ea9bd4 <+20>:	cmp    $0x4,%edi
2022-01-11 13:26:40    0x0000000037ea9bd7 <+23>:	ja     0x37ea9d01 <libunwind::DwarfInstructions<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::getSavedRegister(libunwind::LocalAddressSpace&, libunwind::Registers_x86_64 const&, unsigned long, libunwind::RegisterLocation const&)+321>
2022-01-11 13:26:40    0x0000000037ea9bdd <+29>:	jmpq   *0x97fabf0(,%rdi,8)
2022-01-11 13:26:40 
2022-01-11 13:26:40 99	in ../contrib/libunwind/src/DwarfInstructions.hpp
2022-01-11 13:26:40 100	in ../contrib/libunwind/src/DwarfInstructions.hpp
2022-01-11 13:26:40    0x0000000037ea9be4 <+36>:	mov    0x8(%rcx),%rcx
2022-01-11 13:26:40 
2022-01-11 13:26:40 ../contrib/libunwind/src/AddressSpace.hpp:
2022-01-11 13:26:40 => 0x0000000037ea9be8 <+40>:	mov    (%rcx,%rax,1),%rax
2022-01-11 13:26:40 
2022-01-11 13:26:40 ../contrib/libunwind/src/DwarfInstructions.hpp:
2022-01-11 13:26:40    0x0000000037ea9bec <+44>:	add    $0x8,%rsp
2022-01-11 13:26:40    0x0000000037ea9bf0 <+48>:	pop    %rbx
2022-01-11 13:26:40    0x0000000037ea9bf1 <+49>:	pop    %rbp
2022-01-11 13:26:40    0x0000000037ea9bf2 <+50>:	retq   
2022-01-11 13:26:40 
...

About this issue

  • Original URL
  • State: closed
  • Created 2 years ago
  • Reactions: 1
  • Comments: 22 (17 by maintainers)

Most upvoted comments

@bakwc likely your stacktrace is not full, it shows that unwind triggered from the SIGSEGV (sig=11), so something went wrong before.

And back to the topic, there is no proper fix yet, but the workaround for production could be to disable query profilers (query_profiler_real_time_period_ns, query_profiler_cpu_time_period_ns)

@alexey-milovidov I’ve seen some failures in production for the last couple of months (with 23.10 + few custom patches, nothing related), so I don’t think that it is fixed.

I’ve extracted here: #60460

Thanks for providing much of details, @seandhaynes. It would be great if we could get this re-opened, I completely agree with @azat and @seandhaynes the underlying issue seems to be still present.

Let us know @alexey-milovidov if you need more information on it to get it assigned.

@alexey-milovidov Are you OK with us re-opening this please?

We’re seeing the same thing on all of our production clusters running v23.3, but it’s re-producible in clean builds from master from my testing.

What we see: ClickHouse servers across our entire fleet are occasionally segfaulting, with only the below information in logs:

2024.02.24 13:21:53.198661 [ 7 ] {} <Fatal> Application: Child process was terminated by signal 11

No consistent pattern in the query log or server logs stand out to me before or after we see the segfaults.

Reviewing coredumps, here is an example stack trace (almost identical to a few on this ticket):

gdb) bt
#0  0x0000000015da2fd1 in libunwind::DwarfInstructions<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::getSavedRegister (addressSpace=..., registers=..., cfa=cfa@entry=140650633216840, savedReg=...) at ./contrib/libunwind/src/DwarfInstructions.hpp:111
#1  0x0000000015da1f81 in libunwind::DwarfInstructions<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::stepWithDwarf (addressSpace=..., pc=<optimized out>, fdeStart=<optimized out>, registers=..., isSignalFrame=@0x7fedb2ae2609: true) at ./contrib/libunwind/src/DwarfInstructions.hpp:258
#2  0x0000000015da1aa2 in libunwind::UnwindCursor<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::stepWithDwarfFDE (this=0x7fedb2ae2508) at ./contrib/libunwind/src/UnwindCursor.hpp:1002
#3  libunwind::UnwindCursor<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::step (this=0x7fedb2ae2508) at ./contrib/libunwind/src/UnwindCursor.hpp:2818
#4  0x0000000015da11c8 in unw_backtrace (buffer=0x7fedb2ae2718, size=45) at ./contrib/libunwind/src/libunwind.cpp:350
#5  0x000000000ec18088 in StackTrace::tryCapture (this=0x7fedb2ae2708) at ./src/Common/StackTrace.cpp:287
#6  StackTrace::StackTrace (this=0x7fedb2ae2708, signal_context=...) at ./src/Common/StackTrace.cpp:258
#7  0x000000000edffead in signalHandler (sig=<optimized out>, info=0x7fedb2ae2bf0, context=0x7fedb2ae2ac0) at ./src/Daemon/BaseDaemon.cpp:148
#8  <signal handler called>
#9  0x0000000015da2fd1 in libunwind::DwarfInstructions<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::getSavedRegister (addressSpace=..., registers=..., cfa=cfa@entry=140650633216840, savedReg=...) at ./contrib/libunwind/src/DwarfInstructions.hpp:111
#10 0x0000000015da1f81 in libunwind::DwarfInstructions<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::stepWithDwarf (addressSpace=..., pc=<optimized out>, fdeStart=<optimized out>, registers=..., isSignalFrame=@0x7febc7077799: true) at ./contrib/libunwind/src/DwarfInstructions.hpp:258
#11 0x0000000015da1aa2 in libunwind::UnwindCursor<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::stepWithDwarfFDE (this=0x7febc7077698) at ./contrib/libunwind/src/UnwindCursor.hpp:1002
#12 libunwind::UnwindCursor<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::step (this=0x7febc7077698) at ./contrib/libunwind/src/UnwindCursor.hpp:2818
#13 0x0000000015da11c8 in unw_backtrace (buffer=0x7febc7077838, size=45) at ./contrib/libunwind/src/libunwind.cpp:350
#14 0x000000000ec18088 in StackTrace::tryCapture (this=0x7febc7077828) at ./src/Common/StackTrace.cpp:287
#15 StackTrace::StackTrace (this=0x7febc7077828, signal_context=...) at ./src/Common/StackTrace.cpp:258
#16 0x000000000ec2c5cb in DB::(anonymous namespace)::writeTraceInfo (trace_type=DB::TraceType::Real, info=<optimized out>, context=0x7febc7077d80) at ./src/Common/QueryProfiler.cpp:68
#17 <signal handler called>
#18 0x0000000015ce6629 in LZ4_compress_default (src=<optimized out>, dst=<optimized out>, srcSize=<optimized out>, maxOutputSize=<optimized out>) at ./contrib/lz4/lib/lz4.c:1387
#19 0x0000000012ea3a53 in DB::ICompressionCodec::compress (this=0x7fe0d527b620, source=0x7fd72fab4040 "\003", source_size=65536, dest=0x7fd72db49abb "\202e151024b\"\003") at ./src/Compression/ICompressionCodec.cpp:88
#20 0x0000000012e36d00 in DB::CompressedWriteBuffer::nextImpl (this=0x7fed6b6d4cd8) at ./src/Compression/CompressedWriteBuffer.cpp:42
#21 0x0000000014353684 in std::__1::__uninitialized_allocator_copy[abi:v15000]<std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::regex_token_iterator<std::__1::__wrap_iter<char const*>, char, std::__1::regex_traits<char> >, std::__1::regex_token_iterator<std::__1::__wrap_iter<char const*>, char, std::__1::regex_traits<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >*>(std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >&, std::__1::regex_token_iterator<std::__1::__wrap_iter<char const*>, char, std::__1::regex_traits<char> >, std::__1::regex_token_iterator<std::__1::__wrap_iter<char const*>, char, std::__1::regex_traits<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >*) (__alloc=..., __first1=..., __last1=..., __first2=0x606f100 <vtable for DB::HashingWriteBuffer+16>) at ./contrib/llvm-project/libcxx/include/__memory/uninitialized_algorithms.h:545
#22 0x0000000000000000 in ?? ()

Seeing the below frame:

#16 0x000000000ec2c5cb in DB::(anonymous namespace)::writeTraceInfo (trace_type=DB::TraceType::Real, info=<optimized out>, context=0x7febc7077d80) at ./src/Common/QueryProfiler.cpp:68

It seems like our case aligns with what @azat mentioned about this occuring as a result of query profiling, so we can try and turn them off as a first step just to confirm.

These frames are also pretty consistent from the coredumps I have reviewed so far:

#18 0x0000000015ce6629 in LZ4_compress_default (src=<optimized out>, dst=<optimized out>, srcSize=<optimized out>, maxOutputSize=<optimized out>) at ./contrib/lz4/lib/lz4.c:1387
#19 0x0000000012ea3a53 in DB::ICompressionCodec::compress (this=0x7fe0d527b620, source=0x7fd72fab4040 "\003", source_size=65536, dest=0x7fd72db49abb "\202e151024b\"\003") at ./src/Compression/ICompressionCodec.cpp:88
#20 0x0000000012e36d00 in DB::CompressedWriteBuffer::nextImpl (this=0x7fed6b6d4cd8) at ./src/Compression/CompressedWriteBuffer.cpp:42

Could this be an issue of trying to unwind a stack trace of an INSERT query that is behaving unexpectedly?

Another pattern that I noticed is that we see far more segfaults on ClickHouse replicas that receive fairly heavy INSERT workloads, but I will review some more coredumps and investigate a bit more to make sure this is definitely the case.

You’re welcome to assign it to me if you want to and I will try to dig a bit deeper and find the cause if we don’t already know what it is.

You can look into the logs - grep Fatal /var/log/clickhouse-server/clickhouse-server.err.log

Or/and get backtrace for all threads:

  • gdb - thread apply all bt
  • lldb - bt all

Crashed here: https://github.com/ClickHouse-Extras/libunwind/blob/c4ea9848a697747dfa35325af9b3452f30841685/src/DwarfInstructions.hpp#L99-L100 or

2022-01-11 13:26:40 => 0x0000000037ea9be8 <+40>:	mov    (%rcx,%rax,1),%rax

Seems like rax is cfa and rcx is savedReg.value, but values are strange:

2022-01-11 13:26:40 rax            0x100000011         4294967313
2022-01-11 13:26:40 rcx            0xffffffffffffffd0  -48

Looks like some garbage was passed to the function. It was called from: https://github.com/ClickHouse-Extras/libunwind/blob/c4ea9848a697747dfa35325af9b3452f30841685/src/DwarfInstructions.hpp#L215-L216

But according to the output of gdb

2022-01-11 13:26:34 #1  0x0000000037ea986f in libunwind::DwarfInstructions<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::stepWithDwarf (addressSpace=..., pc=<optimized out>, fdeStart=<optimized out>, registers=...) at ../contrib/libunwind/src/DwarfInstructions.hpp:215
2022-01-11 13:26:34         i = 3
2022-01-11 13:26:34         newRegisters = {_registers = {__rax = <optimized out>, __rbx = <optimized out>, __rcx = <optimized out>, __rdx = <optimized out>, __rdi = <optimized out>, __rsi = <optimized out>, __rbp = <optimized out>, __rsp = <optimized out>, __r8 = <optimized out>, __r9 = <optimized out>, __r10 = <optimized out>, __r11 = <optimized out>, __r12 = <optimized out>, __r13 = <optimized out>, __r14 = <optimized out>, __r15 = <optimized out>, __rip = <optimized out>, __rflags = 140398441257872, __cs = 17549805157236, __fs = 10, __gs = 105828009323536}}
2022-01-11 13:26:34         cfa = 140398441256912
2022-01-11 13:26:34         lastReg = 32
2022-01-11 13:26:34         returnAddress = 0
2022-01-11 13:26:34         prolog = {cfaRegister = 6, cfaRegisterOffset = 16, cfaExpression = 0, spExtraArgSize = 0, codeOffsetAtStackDecrement = 1, registersInOtherRegisters = false, sameValueUsed = false, savedRegisters = {{location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterInCFA, value = -48}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterInCFA, value = -16}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterInCFA, value = -40}, {location = libunwind::kRegisterUnused, value = 0}, {location = libunwind::kRegisterInCFA, value = -32}, {location = libunwind::kRegisterInCFA, value = -24}, {location = libunwind::kRegisterInCFA, value = -8}, {location = libunwind::kRegisterUnused, value = 0} <repeats 16 times>}}
2022-01-11 13:26:34         fdeInfo = {fdeStart = 174863768, fdeLength = 64, fdeInstructions = 174863785, pcStart = 202615776, pcEnd = 202616495, lsda = 0}
2022-01-11 13:26:34         cieInfo = {cieStart = 174806504, cieLength = 24, cieInstructions = 174806521, pointerEncoding = 27 '\033', lsdaEncoding = 255 '\377', personalityEncoding = 0 '\000', personalityOffsetInCIE = 0 '\000', personality = 0, codeAlignFactor = 1, dataAlignFactor = -8, isSignalFrame = false, fdesHaveAugmentationData = true, returnAddressRegister = 16 '\020'}

cfa was 140398441256912 (or 0x7fb10f37d7d0 which looks sane, because rsp is 0x7fb10f37d3e0) and prolog.savedRegisters[3] was {location = libunwind::kRegisterInCFA, value = -48} (looks ok).

What happened to cfa during the function call? I could assume that stack was corrupted by another call, but cfa was passed through rdx, not through stack:

2022-01-11 13:26:40 215	in ../contrib/libunwind/src/DwarfInstructions.hpp
2022-01-11 13:26:40    0x0000000037ea985d <+701>:	mov    %r14,%rdi
2022-01-11 13:26:40    0x0000000037ea9860 <+704>:	mov    %r13,%rsi
2022-01-11 13:26:40    0x0000000037ea9863 <+707>:	mov    -0x30(%rbp),%rdx
2022-01-11 13:26:40    0x0000000037ea9867 <+711>:	mov    %rbx,%rcx
2022-01-11 13:26:40    0x0000000037ea986a <+714>:	callq  0x37ea9bc0 <libunwind::DwarfInstructions<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::getSavedRegister(libunwind::LocalAddressSpace&, libunwind::Registers_x86_64 const&, unsigned long, libunwind::RegisterLocation const&)>
2022-01-11 13:26:40 
2022-01-11 13:26:40 ../contrib/libunwind/src/Registers.hpp:
2022-01-11 13:26:40 => 0x0000000037ea986f <+719>:	cmp    $0x10,%r15d

Maybe -0x30(%rbp) was corrupted, but then how gdb have printed the value that seems to be correct?