milvus: [Bug]: Use query `category_path like "4%" ` and then crash
Is there an existing issue for this?
- I have searched the existing issues
Environment
- Milvus version: 2.3.2
- Deployment mode(standalone or cluster): standalone
- MQ type(rocksmq, pulsar or kafka):
- SDK version(e.g. pymilvus v2.0.0rc2):
- OS(Ubuntu or CentOS):
- CPU/Memory:
- GPU:
- Others:
Current Behavior
schema
Approx 523,632 entity count
I use category_path like "4%" to search and then it crash on version milvusdb/milvus:v2.2.11
error
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x7fe4c3f01403]
runtime stack:
runtime.throw({0x41f7b4f?, 0x7fe2d2f3ec08?})
/usr/local/go/src/runtime/panic.go:992 +0x71
runtime.sigpanic()
/usr/local/go/src/runtime/signal_unix.go:802 +0x3a9
goroutine 97246 [syscall]:
runtime.cgocall(0x32921b0, 0xc064d71980)
/usr/local/go/src/runtime/cgocall.go:157 +0x5c fp=0xc064d71958 sp=0xc064d71920 pc=0x14fe57c
github.com/milvus-io/milvus/internal/querynode._Cfunc_Retrieve(0x7fe2d2f3ec00, 0x7fe36aa5b060, 0x62e27382a380001, 0xc002a79750)
_cgo_gotypes.go:837 +0x55 fp=0xc064d71980 sp=0xc064d71958 pc=0x2f19055
github.com/milvus-io/milvus/internal/querynode.(*Segment).retrieve.func1(0x418addb?, 0xb?, 0x1d56bdb?, 0xc002a79750)
/go/src/github.com/milvus-io/milvus/internal/querynode/segment.go:414 +0xa6 fp=0xc064d719d8 sp=0xc064d71980 pc=0x2f34ba6
github.com/milvus-io/milvus/internal/querynode.(*Segment).retrieve(0xc0015189a0, 0xc00d7fe4c8)
/go/src/github.com/milvus-io/milvus/internal/querynode/segment.go:414 +0x1f0 fp=0xc064d71b98 sp=0xc064d719d8 pc=0x2f34550
github.com/milvus-io/milvus/internal/querynode.retrieveOnSegments({0x46612b0, 0xc00deb5c80}, {0x468b490, 0xc0010a9900}, 0x3, 0x0?, 0x62e2738296c0001?, {0xc0023c8380, 0xe, 0xe}, ...)
/go/src/github.com/milvus-io/milvus/internal/querynode/retrieve.go:51 +0x1e5 fp=0xc064d71c88 sp=0xc064d71b98 pc=0x2edd105
github.com/milvus-io/milvus/internal/querynode.retrieveHistorical({0x46612b0, 0xc00deb5c80}, {0x468b490, 0xc0010a9900}, 0x1?, 0xc003b00620?, {0x0?, 0x14?, 0x1?}, {0xc0023c8380, ...}, ...)
/go/src/github.com/milvus-io/milvus/internal/querynode/retrieve.go:76 +0x146 fp=0xc064d71d38 sp=0xc064d71c88 pc=0x2edd5e6
github.com/milvus-io/milvus/internal/querynode.(*queryTask).queryOnHistorical(0xc0045a4f00)
/go/src/github.com/milvus-io/milvus/internal/querynode/task_query.go:128 +0x1d3 fp=0xc064d71ec0 sp=0xc064d71d38 pc=0x2f02ff3
github.com/milvus-io/milvus/internal/querynode.(*queryTask).Execute(0xc0045a4f00?, {0x13?, 0x418dc34?})
/go/src/github.com/milvus-io/milvus/internal/querynode/task_query.go:151 +0x2a fp=0xc064d71ef8 sp=0xc064d71ec0 pc=0x2f035ea
github.com/milvus-io/milvus/internal/querynode.(*taskScheduler).processReadTask(0xc000371ef0, {0x467fb08, 0xc0045a4f00})
/go/src/github.com/milvus-io/milvus/internal/querynode/task_scheduler.go:336 +0xf2 fp=0xc064d71f70 sp=0xc064d71ef8 pc=0x2f09752
github.com/milvus-io/milvus/internal/querynode.(*taskScheduler).executeReadTasks.func1({0x467fb08, 0xc0045a4f00})
/go/src/github.com/milvus-io/milvus/internal/querynode/task_scheduler.go:286 +0xbb fp=0xc064d71fc0 sp=0xc064d71f70 pc=0x2f094bb
github.com/milvus-io/milvus/internal/querynode.(*taskScheduler).executeReadTasks.func5()
/go/src/github.com/milvus-io/milvus/internal/querynode/task_scheduler.go:314 +0x2e fp=0xc064d71fe0 sp=0xc064d71fc0 pc=0x2f0936e
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc064d71fe8 sp=0xc064d71fe0 pc=0x1567da1
created by github.com/milvus-io/milvus/internal/querynode.(*taskScheduler).executeReadTasks
/go/src/github.com/milvus-io/milvus/internal/querynode/task_scheduler.go:314 +0x48d
goroutine 1 [chan receive, 5 minutes]:
github.com/milvus-io/milvus/cmd/roles.(*MilvusRoles).Run(0xc001965e48, 0x1, {0x0, 0x0})
/go/src/github.com/milvus-io/milvus/cmd/roles/roles.go:346 +0xaf1
github.com/milvus-io/milvus/cmd/milvus.(*run).execute(0xc00027d770, {0xc000194060?, 0x3, 0x3}, 0xc0001104e0)
/go/src/github.com/milvus-io/milvus/cmd/milvus/run.go:117 +0x6ae
github.com/milvus-io/milvus/cmd/milvus.RunMilvus({0xc000194060?, 0x3, 0x3})
/go/src/github.com/milvus-io/milvus/cmd/milvus/milvus.go:60 +0x21e
main.main()
/go/src/github.com/milvus-io/milvus/cmd/main.go:26 +0x2e
goroutine 176 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00097a3c0, 0x1)
/go/pkg/mod/google.golang.org/grpc@v1.46.0/internal/transport/controlbuf.go:407 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00053b560)
/go/pkg/mod/google.golang.org/grpc@v1.46.0/internal/transport/controlbuf.go:534 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
/go/pkg/mod/google.golang.org/grpc@v1.46.0/internal/transport/http2_client.go:415 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.46.0/internal/transport/http2_client.go:413 +0x1f91
Then I want to upgrade to milvusdb/milvus:v2.3.2 and hope to solve the problem ,but failed
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x7f7a8057a313]
runtime stack:
runtime.throw({0x4742f65?, 0x7f7a82dd8bb0?})
/usr/local/go/src/runtime/panic.go:1047 +0x5d fp=0x7f6cae1fede8 sp=0x7f6cae1fedb8 pc=0x178405d
runtime.sigpanic()
/usr/local/go/src/runtime/signal_unix.go:825 +0x3e9 fp=0x7f6cae1fee48 sp=0x7f6cae1fede8 pc=0x179bec9
goroutine 96539 [syscall, locked to thread]:
non-Go function
pc=0x7f7a8057a313
_ZN6milvus11PrefixMatchESt17basic_string_viewIcSt11char_traitsIcEES3_
/go/src/github.com/milvus-io/milvus/internal/core/src/common/Utils.h:80 pc=0x7f7a84d8c598
_ZN6milvus5query5MatchISt17basic_string_viewIcSt11char_traitsIcEENSt7__cxx1112basic_stringIcS4_SaIcEEEEEbRKT_RKT0_NS_5proto4plan6OpTypeE
/go/src/github.com/milvus-io/milvus/internal/core/src/query/Utils.h:48 pc=0x7f7a84d8c598
_ZZN6milvus5query15ExecExprVisitor35ExecUnaryRangeVisitorDispatcherImplISt17basic_string_viewIcSt11char_traitsIcEEEEN5boost14dynamic_bitsetImSaImEEERNS0_14UnaryRangeExprEENKUlS6_E12_clES6_
/go/src/github.com/milvus-io/milvus/internal/core/src/query/visitors/ExecExprVisitor.cpp:440 pc=0x7f7a84d8c598
_ZN6milvus5query15ExecExprVisitor20ExecRangeVisitorImplISt17basic_string_viewIcSt11char_traitsIcEEZNS1_35ExecUnaryRangeVisitorDispatcherImplIS6_EEN5boost14dynamic_bitsetImSaImEEERNS0_14UnaryRangeExprEEUlPNS_5index11ScalarIndexINSt7__cxx1112basic_stringIcS5_SaIcEEEEEE11_ZNS7_IS6_EESB_SD_EUlS6_E12_EESB_N6fluent9NamedTypeIlNS_4impl10FieldIdTagEJNSO_10ComparableENSO_8HashableEEEET0_T1_
/go/src/github.com/milvus-io/milvus/internal/core/src/query/visitors/ExecExprVisitor.cpp:305 pc=0x7f7a84d8c598
_ZN6milvus5query15ExecExprVisitor35ExecUnaryRangeVisitorDispatcherImplISt17basic_string_viewIcSt11char_traitsIcEEEEN5boost14dynamic_bitsetImSaImEEERNS0_14UnaryRangeExprE
/go/src/github.com/milvus-io/milvus/internal/core/src/query/visitors/ExecExprVisitor.cpp:442 pc=0x7f7a84e3de6a
_ZN6milvus5query15ExecExprVisitor31ExecUnaryRangeVisitorDispatcherISt17basic_string_viewIcSt11char_traitsIcEEEEN5boost14dynamic_bitsetImSaImEEERNS0_14UnaryRangeExprE
/go/src/github.com/milvus-io/milvus/internal/core/src/query/visitors/ExecExprVisitor.cpp:507 pc=0x7f7a84d6fc72
_ZN6milvus5query15ExecExprVisitor5visitERNS0_14UnaryRangeExprE
/go/src/github.com/milvus-io/milvus/internal/core/src/query/visitors/ExecExprVisitor.cpp:1646 pc=0x7f7a84d6fc72
runtime.cgocall(0x3e3c7c0, 0xc002f24590)
/usr/local/go/src/runtime/cgocall.go:157 +0x5c fp=0xc002f24568 sp=0xc002f24530 pc=0x174c77c
github.com/milvus-io/milvus/internal/querynodev2/segments._Cfunc_Retrieve(0x7f78d9418700, 0x7f791ea10360, {0xc0063989e0, 0xc0063989f0, 0x0, {0x0, 0x0, 0x0, 0x0, 0x0, ...}}, ...)
_cgo_gotypes.go:813 +0x55 fp=0xc002f24590 sp=0xc002f24568 pc=0x3c25435
github.com/milvus-io/milvus/internal/querynodev2/segments.(*LocalSegment).Retrieve.func1.1(0xc006080248?, 0xc005053e60?, {0xc0063989e0, 0xc0063989f0, 0x0, {0x0, 0x0, 0x0, 0x0, 0x0, ...}}, ...)
/go/src/github.com/milvus-io/milvus/internal/querynodev2/segments/segment.go:398 +0xfa fp=0xc002f24628 sp=0xc002f24590 pc=0x3c340fa
github.com/milvus-io/milvus/internal/querynodev2/segments.(*LocalSegment).Retrieve.func1()
/go/src/github.com/milvus-io/milvus/internal/querynodev2/segments/segment.go:398 +0xc5 fp=0xc002f24728 sp=0xc002f24628 pc=0x3c33dc5
github.com/milvus-io/milvus/pkg/util/conc.(*Pool[...]).Submit.func1()
/go/src/github.com/milvus-io/milvus/pkg/util/conc/pool.go:79 +0xbc fp=0xc002f24788 sp=0xc002f24728 pc=0x3c1493c
github.com/panjf2000/ants/v2.(*goWorker).run.func1()
/go/pkg/mod/github.com/panjf2000/ants/v2@v2.7.2/worker.go:67 +0x97 fp=0xc002f247e0 sp=0xc002f24788 pc=0x2c03c97
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc002f247e8 sp=0xc002f247e0 pc=0x17bd9a1
created by github.com/panjf2000/ants/v2.(*goWorker).run
/go/pkg/mod/github.com/panjf2000/ants/v2@v2.7.2/worker.go:48 +0x65
goroutine 1 [chan receive, 3 minutes]:
runtime.gopark(0xc00104f940?, 0xc006c53810?, 0x7e?, 0x7d?, 0x4173a40?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0019eb7b8 sp=0xc0019eb798 pc=0x17870d6
runtime.chanrecv(0xc0000f6de0, 0x0, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc0019eb848 sp=0xc0019eb7b8 pc=0x174f5bd
runtime.chanrecv1(0xc0010acdb0?, 0x4bfad98?)
/usr/local/go/src/runtime/chan.go:442 +0x18 fp=0xc0019eb870 sp=0xc0019eb848 pc=0x174f0b8
github.com/milvus-io/milvus/cmd/roles.(*MilvusRoles).Run(0xc00110c3c0)
/go/src/github.com/milvus-io/milvus/cmd/roles/roles.go:389 +0x985 fp=0xc0019ebc70 sp=0xc0019eb870 pc=0x3d406c5
github.com/milvus-io/milvus/cmd/milvus.(*run).execute(0x10?, {0xc000052180?, 0x3, 0x3}, 0xc00037b8c0)
/go/src/github.com/milvus-io/milvus/cmd/milvus/run.go:44 +0x2ab fp=0xc0019ebd40 sp=0xc0019ebc70 pc=0x3d4b8cb
github.com/milvus-io/milvus/cmd/milvus.RunMilvus({0xc000052180?, 0x3, 0x3})
/go/src/github.com/milvus-io/milvus/cmd/milvus/milvus.go:60 +0x20e fp=0xc0019ebdb8 sp=0xc0019ebd40 pc=0x3d4b58e
main.main()
/go/src/github.com/milvus-io/milvus/cmd/main.go:95 +0x3c5 fp=0xc0019ebf80 sp=0xc0019ebdb8 pc=0x3d500c5
runtime.main()
/usr/local/go/src/runtime/proc.go:250 +0x207 fp=0xc0019ebfe0 sp=0xc0019ebf80 pc=0x1786ca7
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0019ebfe8 sp=0xc0019ebfe0 pc=0x17bd9a1
goroutine 2 [force gc (idle), 3 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000bcfb0 sp=0xc0000bcf90 pc=0x17870d6
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:387
runtime.forcegchelper()
/usr/local/go/src/runtime/proc.go:305 +0xb0 fp=0xc0000bcfe0 sp=0xc0000bcfb0 pc=0x1786f10
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000bcfe8 sp=0xc0000bcfe0 pc=0x17bd9a1
created by runtime.init.6
/usr/local/go/src/runtime/proc.go:293 +0x25
goroutine 3 [GC sweep wait]:
runtime.gopark(0x47fe301?, 0x6706360?, 0xc?, 0x14?, 0x1?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000bd780 sp=0xc0000bd760 pc=0x17870d6
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:387
runtime.bgsweep(0x0?)
/usr/local/go/src/runtime/mgcsweep.go:319 +0xde fp=0xc0000bd7c8 sp=0xc0000bd780 pc=0x176f85e
runtime.gcenable.func1()
/usr/local/go/src/runtime/mgc.go:178 +0x26 fp=0xc0000bd7e0 sp=0xc0000bd7c8 pc=0x1764926
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000bd7e8 sp=0xc0000bd7e0 pc=0x17bd9a1
created by runtime.gcenable
/usr/local/go/src/runtime/mgc.go:178 +0x6b
goroutine 4 [GC scavenge wait]:
runtime.gopark(0x670ea80?, 0x1170e2?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000bdf70 sp=0xc0000bdf50 pc=0x17870d6
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:387
runtime.(*scavengerState).park(0x670ea80)
/usr/local/go/src/runtime/mgcscavenge.go:400 +0x53 fp=0xc0000bdfa0 sp=0xc0000bdf70 pc=0x176d693
runtime.bgscavenge(0x0?)
/usr/local/go/src/runtime/mgcscavenge.go:633 +0x65 fp=0xc0000bdfc8 sp=0xc0000bdfa0 pc=0x176dca5
runtime.gcenable.func2()
/usr/local/go/src/runtime/mgc.go:179 +0x26 fp=0xc0000bdfe0 sp=0xc0000bdfc8 pc=0x17648c6
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000bdfe8 sp=0xc0000bdfe0 pc=0x17bd9a1
created by runtime.gcenable
/usr/local/go/src/runtime/mgc.go:179 +0xaa
goroutine 5 [finalizer wait]:
runtime.gopark(0x0?, 0x47fa008?, 0xe0?, 0x41?, 0x1000000010?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001d19e28 sp=0xc001d19e08 pc=0x17870d6
runtime.runfinq()
/usr/local/go/src/runtime/mfinal.go:193 +0x107 fp=0xc001d19fe0 sp=0xc001d19e28 pc=0x1763967
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001d19fe8 sp=0xc001d19fe0 pc=0x17bd9a1
created by runtime.createfing
/usr/local/go/src/runtime/mfinal.go:163 +0x45
goroutine 6 [GC worker (idle), 3 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000be750 sp=0xc0000be730 pc=0x17870d6
runtime.gcBgMarkWorker()
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0000be7e0 sp=0xc0000be750 pc=0x1766691
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000be7e8 sp=0xc0000be7e0 pc=0x17bd9a1
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
Expected Behavior
No response
Steps To Reproduce
No response
Milvus Log
No response
Anything else?
No response
About this issue
- Original URL
- State: closed
- Created 8 months ago
- Comments: 17 (13 by maintainers)
two corner case: