pebble: panic on arm64

[5058]: unexpected fault address 0x6c1ed94dad07336
[5058]: fatal error: fault
[5058]: [signal SIGSEGV: segmentation violation code=0x1 addr=0x6c1ed94dad07336 pc=0x601e70]
[5058]: goroutine 30 [running]:
[5058]: runtime.throw(0x98160e, 0x5)
[5058]:         /usr/local/go/src/runtime/panic.go:1116 +0x54 fp=0x400060ed80 sp=0x400060ed50 pc=0x413d4
[5058]: runtime.sigpanic()
[5058]:         /usr/local/go/src/runtime/signal_unix.go:727 +0x3b8 fp=0x400060edb0 sp=0x400060ed80 pc=0x56a88
[5058]: github.com/cockroachdb/pebble/internal/cache.(*entry).linkFile(...)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/internal/cache/entry.go:120
[5058]: github.com/cockroachdb/pebble/internal/cache.(*shard).metaAdd(0x4000158d70, 0x2, 0xa, 0x93c04, 0x4000523ec0, 0x4000523ec0)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/internal/cache/clockpro.go:310 +0x140 fp=0x400060ee00 sp=0x400060edc0 pc=0x601e70
[5058]: github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0x4000158d70, 0x2, 0xa, 0x93c04, 0x4000136600, 0x0)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/internal/cache/clockpro.go:141 +0x528 fp=0x400060eea0 sp=0x400060ee00 pc=0x6015d8
[5058]: github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0x40004aa6e0, 0x2, 0xa, 0x93c04, 0x4000136600, 0x931)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/internal/cache/clockpro.go:658 +0x6c fp=0x400060eee0 sp=0x400060eea0 pc=0x602ffc
[5058]: github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0x400040ea00, 0x93c04, 0x92c, 0x0, 0x400017d030, 0x5, 0x24, 0x40004fa000)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/sstable/reader.go:1519 +0x220 fp=0x400060f0f0 sp=0x400060eee0 pc=0x617750
[5058]: github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0x400017ce00, 0x400017cef8)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/sstable/reader.go:212 +0x124 fp=0x400060f150 sp=0x400060f0f0 pc=0x612c04
[5058]: github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).skipForward(0x400017ce00, 0x0, 0x0, 0x0, 0x0)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/sstable/reader.go:422 +0x44 fp=0x400060f1c0 sp=0x400060f150 pc=0x613bf4
[5058]: github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).Next(0x400017ce00, 0x66062c, 0x4000083380, 0x879, 0x400060f2e8)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/sstable/reader.go:398 +0xf8 fp=0x400060f230 sp=0x400060f1c0 pc=0x613a08
[5058]: github.com/cockroachdb/pebble/sstable.(*twoLevelCompactionIterator).Next(0x400000dea0, 0x61eb98, 0x0, 0xc, 0x400060f2e8)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/sstable/reader.go:913 +0x50 fp=0x400060f290 sp=0x400060f230 pc=0x616730
[5058]: github.com/cockroachdb/pebble.(*mergingIter).nextEntry(0x40004c4180, 0x4000083380)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/merging_iter.go:495 +0x6c fp=0x400060f2f0 sp=0x400060f290 pc=0x66053c
[5058]: github.com/cockroachdb/pebble.(*mergingIter).Next(0x40004c4180, 0x40000a29a8, 0x400060f418, 0x61d650, 0x40000a29a8)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/merging_iter.go:981 +0x64 fp=0x400060f330 sp=0x400060f2f0 pc=0x662474
[5058]: github.com/cockroachdb/pebble.(*compactionIter).iterNext(...)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/compaction_iter.go:392
[5058]: github.com/cockroachdb/pebble.(*compactionIter).nextInStripe(0x40000de000, 0x0)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/compaction_iter.go:409 +0x34 fp=0x400060f390 sp=0x400060f330 pc=0x63e2d4
[5058]: github.com/cockroachdb/pebble.(*compactionIter).skipInStripe(0x40000de000)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/compaction_iter.go:377 +0x38 fp=0x400060f3b0 sp=0x400060f390 pc=0x63e268
[5058]: github.com/cockroachdb/pebble.(*compactionIter).Next(0x40000de000, 0x40004fa0c0, 0x24, 0x30, 0x1)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/compaction_iter.go:247 +0x50 fp=0x400060f480 sp=0x400060f3b0 pc=0x63d610
[5058]: github.com/cockroachdb/pebble.(*DB).runCompaction(0x400004f000, 0x3, 0x40000ca900, 0xad33a0, 0x1094be0, 0x40003a40f0, 0x400041a108, 0x1, 0x1, 0x0, ...)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/compaction.go:2234 +0xde0 fp=0x400060fc20 sp=0x400060f480 pc=0x63b0a0
[5058]: github.com/cockroachdb/pebble.(*DB).compact1(0x400004f000, 0x40000ca900, 0x0, 0x0, 0x0)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/compaction.go:1744 +0x144 fp=0x400060fe80 sp=0x400060fc20 pc=0x639cd4
[5058]: github.com/cockroachdb/pebble.(*DB).compact.func1(0xaeaea0, 0x400009b050)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/compaction.go:1705 +0x90 fp=0x400060fef0 sp=0x400060fe80 pc=0x675320
[5058]: runtime/pprof.Do(0xaeaea0, 0x400009b050, 0x40003b3600, 0x1, 0x1, 0x4000489798)
[5058]:         /usr/local/go/src/runtime/pprof/runtime.go:40 +0xac fp=0x400060ff60 sp=0x400060fef0 pc=0x605a5c
[5058]: github.com/cockroachdb/pebble.(*DB).compact(0x400004f000, 0x40000ca900, 0x0)
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/compaction.go:1702 +0xa0 fp=0x400060ffc0 sp=0x400060ff60 pc=0x639b70
[5058]: runtime.goexit()
[5058]:         /usr/local/go/src/runtime/asm_arm64.s:1136 +0x4 fp=0x400060ffc0 sp=0x400060ffc0 pc=0x72c64
[5058]: created by github.com/cockroachdb/pebble.(*DB).maybeScheduleCompactionPicker
[5058]:         /go/pkg/mod/github.com/cockroachdb/pebble@v0.0.0-20200930153740-4e0eb739ab1a/compaction.go:1499 +0x2e8
[5058]: goroutine 1 [select]:
[5058]: net/http.(*Transport).getConn(0x40003ec780, 0x40004a7000, 0x0, 0x4000097080, 0x5, 0x4000039020, 0x1d, 0x0, 0x0, 0x0, ...)

About this issue

Original URL
State: closed
Created 4 years ago
Comments: 40 (1 by maintainers)

Most upvoted comments

I’m pretty sure that this is the same problem as described in #1010 and should be fixed by #1011.

@ivanjaros I see you reached a similar conclusion as to the problem, but I never saw your comment because Github doesn’t alert me when a comment is edited, only when it is added.

petermattis on Nov 25, 2020

btw if pebble is used in cdb these days, don’t you have some tests that run on windows?

Yes, and those tests aren’t seeing this problem, just as the Pebble CI tests on Windows aren’t seeing this problem.

btw2: could this have something to do with the prefix scans that we’ve been talking about in other issue(s)? like there is something wrong with the lower bound and upper bound values and their user/parsing? because i use this a lot in my code base.

It is super hard to just look at code like this and see a problem. I need a fully working reproduction case.

That said, are you closing the prefixIterator? I think that should only cause a memory leak, but it could possibly be related.

petermattis on Nov 5, 2020