go: runtime: r2 corrupted by mid-stack inlining on ppc64le

What version of Go are you using (go version)?

$ go version
go version go1.12rc1 linux/ppc64le

Does this issue reproduce with the latest release?

Yes, reproduced with 1.12rc1

What operating system and processor architecture are you using (go env)?

go env Output
$ go env
GOARCH="ppc64le"
GOBIN=""
GOCACHE="/root/.cache/go-build"
GOEXE=""
GOFLAGS=""
GOHOSTARCH="ppc64le"
GOHOSTOS="linux"
GOOS="linux"
GOPATH="/go"
GOPROXY=""
GORACE=""
GOROOT="/usr/local/go"
GOTMPDIR=""
GOTOOLDIR="/usr/local/go/pkg/tool/linux_ppc64le"
GCCGO="gccgo"
CC="gcc"
CXX="g++"
CGO_ENABLED="1"
GOMOD=""
CGO_CFLAGS="-g -O2"
CGO_CPPFLAGS=""
CGO_CXXFLAGS="-g -O2"
CGO_FFLAGS="-g -O2"
CGO_LDFLAGS="-g -O2"
PKG_CONFIG="pkg-config"
GOGCCFLAGS="-fPIC -pthread -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build360914728=/tmp/go-build -gno-record-gcc-switches"
#

What did you do?

Steps to reproduce this issues:

root@ip9-114-192-113:~# docker run -ti golang:rc-stretch
root@361515a017ed:/go#
root@361515a017ed:/go# go version
go version go1.12rc1 linux/ppc64le
root@361515a017ed:~# mkdir -p containerd_ws/src/github.com/containerd
root@361515a017ed:~# cd containerd_ws/src/github.com/containerd/
root@361515a017ed:~/containerd_ws/src/github.com/containerd# git clone https://github.com/containerd/containerd.git
Cloning into 'containerd'...
remote: Enumerating objects: 9, done.
remote: Counting objects: 100% (9/9), done.
remote: Compressing objects: 100% (9/9), done.
remote: Total 38383 (delta 2), reused 0 (delta 0), pack-reused 38374
Receiving objects: 100% (38383/38383), 49.63 MiB | 24.02 MiB/s, done.
Resolving deltas: 100% (22575/22575), done.
root@361515a017ed:~/containerd_ws/src/github.com/containerd#
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd# cd metadata/
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata#
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata# export GOPATH=/root/containerd_ws
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata# go test -buildmode pie -v
=== RUN   TestContainersList
unexpected fault address 0x135889140
fatal error: fault
[signal SIGSEGV: segmentation violation code=0x1 addr=0x135889140 pc=0x135dbd820]

goroutine 19 [running]:
runtime.throw(0x13634f828, 0x5)
	/usr/local/go/src/runtime/panic.go:617 +0x68 fp=0xc0001fb0f0 sp=0xc0001fb0b0 pc=0x135ddff38
runtime.sigpanic()
	/usr/local/go/src/runtime/signal_unix.go:397 +0x464 fp=0xc0001fb130 sp=0xc0001fb0f0 pc=0x135df8d64
runtime.mapiterinit(0x135889100, 0xc00020e360, 0xc0001fb328)
	/usr/local/go/src/runtime/map.go:823 +0xa0 fp=0xc0001fb180 sp=0xc0001fb150 pc=0x135dbd820
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).spill(0xc0000bb400, 0xc00020e300, 0xc0001fb530)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/bucket.go:526 +0x80 fp=0xc0001fb388 sp=0xc0001fb180 pc=0x13600eb40
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).spill(0xc0000bb380, 0xc00020e200, 0xc0001fb738)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/bucket.go:535 +0x308 fp=0xc0001fb590 sp=0xc0001fb388 pc=0x13600edc8
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).spill(0xc0000bb300, 0xc00020e200, 0xc0001fb940)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/bucket.go:535 +0x308 fp=0xc0001fb798 sp=0xc0001fb590 pc=0x13600edc8
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).spill(0xc0000bb280, 0xc00020e100, 0xc0001fbb48)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/bucket.go:535 +0x308 fp=0xc0001fb9a0 sp=0xc0001fb798 pc=0x13600edc8
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).spill(0xc0001fe1d8, 0x62c845, 0x136b6a680)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/bucket.go:535 +0x308 fp=0xc0001fbba8 sp=0xc0001fb9a0 pc=0x13600edc8
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Tx).Commit(0xc0001fe1c0, 0x0, 0x0)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/tx.go:160 +0xd0 fp=0xc0001fbce8 sp=0xc0001fbba8 pc=0x13601c1e0
github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*DB).Update(0xc00019a000, 0xc0001fbed8, 0x0, 0x0)
	/root/containerd_ws/src/github.com/containerd/containerd/vendor/go.etcd.io/bbolt/db.go:677 +0x108 fp=0xc0001fbd48 sp=0xc0001fbce8 pc=0x136013ef8
github.com/containerd/containerd/metadata.TestContainersList(0xc0000f4400)
	/root/containerd_ws/src/github.com/containerd/containerd/metadata/containers_test.go:74 +0x4e0 fp=0xc0001fbf70 sp=0xc0001fbd48 pc=0x13631b2d0
testing.tRunner(0xc0000f4400, 0x1366ca600)
	/usr/local/go/src/testing/testing.go:865 +0xdc fp=0xc0001fbfb0 sp=0xc0001fbf70 pc=0x135eb1dfc
runtime.goexit()
	/usr/local/go/src/runtime/asm_ppc64x.s:856 +0x4 fp=0xc0001fbfb0 sp=0xc0001fbfb0 pc=0x135e135f4
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:916 +0x304

goroutine 1 [chan receive]:
testing.(*T).Run(0xc0000f4400, 0x136359315, 0x12, 0x1366ca600, 0x136b27f00)
	/usr/local/go/src/testing/testing.go:917 +0x320
testing.runTests.func1(0xc0000f4300)
	/usr/local/go/src/testing/testing.go:1157 +0x8c
testing.tRunner(0xc0000f4300, 0xc0000d3dd8)
	/usr/local/go/src/testing/testing.go:865 +0xdc
testing.runTests(0xc000193660, 0x136b29600, 0x10, 0x10, 0x0)
	/usr/local/go/src/testing/testing.go:1155 +0x2a0
testing.(*M).Run(0xc0000f2280, 0x0)
	/usr/local/go/src/testing/testing.go:1072 +0x174
main.main()
	_testmain.go:74 +0x150
exit status 2
FAIL	github.com/containerd/containerd/metadata	0.013s
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata#

root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata# uname -a
Linux 361515a017ed 4.4.0-141-generic #167-Ubuntu SMP Wed Dec 5 10:33:00 UTC 2018 ppc64le GNU/Linux
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata# cat /etc/os-release
PRETTY_NAME="Debian GNU/Linux 9 (stretch)"
NAME="Debian GNU/Linux"
VERSION_ID="9"
VERSION="9 (stretch)"
ID=debian
HOME_URL="https://www.debian.org/"
SUPPORT_URL="https://www.debian.org/support"
BUG_REPORT_URL="https://bugs.debian.org/"
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata# lscpu
Architecture:          ppc64le
Byte Order:            Little Endian
CPU(s):                16
On-line CPU(s) list:   0-15
Thread(s) per core:    8
Core(s) per socket:    1
Socket(s):             2
NUMA node(s):          2
Model:                 2.1 (pvr 004b 0201)
Model name:            POWER8 (architected), altivec supported
L1d cache:             64K
L1i cache:             32K
NUMA node0 CPU(s):     0-15
NUMA node2 CPU(s):
root@361515a017ed:~/containerd_ws/src/github.com/containerd/containerd/metadata#

What did you expect to see?

No panic

What did you see instead?

Seen panic with fatal error: fault error

About this issue

  • Original URL
  • State: closed
  • Created 5 years ago
  • Comments: 18 (11 by maintainers)

Commits related to this issue

Most upvoted comments

@mkumatag Can you test this out to verify it fixes your issue? I verified it on the cases I was aware of.

@laboger I ran the containerd tests(tests, integration) with the submitted patch. I don’t see any issues.

test logs: https://gist.github.com/mkumatag/dd0cafcf968700119da401e6a652683f

+2
mkumatag on Feb 22, 2019

I have found that it starts to fail with commit 69c2c564. @randall77

The SEGV occurs because r2 has been corrupted during a call to github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).inlineable from github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).spill.

Before this change, the code at the end of github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).inlineable looks like this:

  599bf4:       08 00 05 e9     ld      r8,8(r5)
  599bf8:       08 00 08 e9     ld      r8,8(r8)
  599bfc:       10 00 e7 38     addi    r7,r7,16
  599c00:       80 00 08 e9     ld      r8,128(r8)
  599c04:       76 fe 09 7d     sradi   r9,r8,63
  599c08:       a0 17 29 79     rldicl  r9,r9,2,62
  599c0c:       14 4a 08 7d     add     r8,r8,r9
  599c10:       74 16 08 7d     sradi   r8,r8,2
  599c14:       00 40 27 7c     cmpd    r7,r8
  599c18:       24 00 81 41     bgt     599c3c <github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).inlineable+0x16c>
  599c1c:       28 00 01 e9     ld      r8,40(r1)
  599c20:       01 00 08 39     addi    r8,r8,1
  599c24:       00 30 28 7c     cmpd    r8,r6
  599c28:       14 ff 80 41     blt     599b3c <github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).inlineable+0x6c>
  599c2c:       01 00 60 38     li      r3,1
  599c30:       e0 00 61 98     stb     r3,224(r1)
  599c34:       b8 00 21 38     addi    r1,r1,184
  599c38:       20 00 80 4e     blr
  599c3c:       e0 00 01 98     stb     r0,224(r1)
  599c40:       b8 00 21 38     addi    r1,r1,184
  599c44:       20 00 80 4e     blr
  599c48:       e0 00 01 98     stb     r0,224(r1)
  599c4c:       b8 00 21 38     addi    r1,r1,184
  599c50:       20 00 80 4e     blr
  599c54:       e0 00 01 98     stb     r0,224(r1)
  599c58:       b8 00 21 38     addi    r1,r1,184
  599c5c:       20 00 80 4e     blr

After this change, the same code looks like this. It is incorrectly loading the value of r2 from 24(r1). That location has never been initialized with the value of r2 so some random value is being loaded and then used by the code when this function returns.

After this change, the same code looks like this:
  5aa1c4:       08 00 05 e9     ld      r8,8(r5)
  5aa1c8:       08 00 08 e9     ld      r8,8(r8)
  5aa1cc:       10 00 e7 38     addi    r7,r7,16
  5aa1d0:       78 03 00 7c     mr      r0,r0
  5aa1d4:       18 00 41 e8     ld      r2,24(r1)  <----- This load is wrong, r2 has never been saved here.
  5aa1d8:       80 00 08 e9     ld      r8,128(r8)
  5aa1dc:       76 fe 09 7d     sradi   r9,r8,63
  5aa1e0:       a0 17 29 79     rldicl  r9,r9,2,62
  5aa1e4:       14 4a 08 7d     add     r8,r8,r9
  5aa1e8:       74 16 08 7d     sradi   r8,r8,2
  5aa1ec:       00 40 27 7c     cmpd    r7,r8
  5aa1f0:       24 00 81 41     bgt     5aa214 <github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).inlineable+0x174>
  5aa1f4:       28 00 01 e9     ld      r8,40(r1)
  5aa1f8:       01 00 08 39     addi    r8,r8,1
  5aa1fc:       00 30 28 7c     cmpd    r8,r6
  5aa200:       0c ff 80 41     blt     5aa10c <github.com/containerd/containerd/vendor/go.etcd.io/bbolt.(*Bucket).inlineable+0x6c>
  5aa204:       01 00 60 38     li      r3,1
  5aa208:       e0 00 61 98     stb     r3,224(r1)
  5aa20c:       b8 00 21 38     addi    r1,r1,184
  5aa210:       20 00 80 4e     blr
  5aa214:       e0 00 01 98     stb     r0,224(r1)
  5aa218:       b8 00 21 38     addi    r1,r1,184
  5aa21c:       20 00 80 4e     blr
  5aa220:       e0 00 01 98     stb     r0,224(r1)
  5aa224:       b8 00 21 38     addi    r1,r1,184
  5aa228:       20 00 80 4e     blr
  5aa22c:       e0 00 01 98     stb     r0,224(r1)
  5aa230:       b8 00 21 38     addi    r1,r1,184
  5aa234:       20 00 80 4e     blr
+2
laboger on Feb 18, 2019

Yes, anything built for PIC (i.e., using -buildmode=pie or -shared) could hit this bug. The commit that I noted above is adding the load of r2 from a location that might not have been initialized, and if that is used to generate an address then a SEGV can definitely occur.

It doesn’t look like anything has been happening here, I have a fix for it but not sure what the process is at this point. Submit into the go1.12 branch only or upstream too?

+1
laboger on Feb 21, 2019

This same error happens in pkg math test Nextafter32 when build with -buildmode=pie

go test -buildmode=pie -test.run=Nextafter32 unexpected fault address 0xbffff3a4f8 fatal error: fault [signal SIGSEGV: segmentation violation code=0x1 addr=0xbffff3a4f8 pc=0x108bc53c8]

goroutine 5 [running]: runtime.throw(0x108c67b5f, 0x5) /home/boger/golang/baseline/go/src/runtime/panic.go:627 +0x68 fp=0xc00003fdf0 sp=0xc00003fdb0 pc=0x108b73608 runtime.sigpanic() /home/boger/golang/baseline/go/src/runtime/signal_unix.go:397 +0x464 fp=0xc00003fe30 sp=0xc00003fdf0 pc=0x108b8a344 math.Nextafter32(0x41200000409f5411, 0x0) /home/boger/golang/baseline/go/src/math/nextafter.go:19 +0x68 fp=0xc00003fe90 sp=0xc00003fe50 pc=0x108bc53c8 math_test.TestNextafter32(0xc0000ca100) /home/boger/golang/baseline/go/src/math/all_test.go:2738 +0x8c fp=0xc00003ff70 sp=0xc00003fe90 pc=0x108c55a9c testing.tRunner(0xc0000ca100, 0x108d21df0) /home/boger/golang/baseline/go/src/testing/testing.go:865 +0xdc fp=0xc00003ffb0 sp=0xc00003ff70 pc=0x108c0b10c runtime.goexit() /home/boger/golang/baseline/go/src/runtime/asm_ppc64x.s:856 +0x4 fp=0xc00003ffb0 sp=0xc00003ffb0 pc=0x108ba4874 created by testing.(*T).Run /home/boger/golang/baseline/go/src/testing/testing.go:916 +0x304

goroutine 1 [chan receive]: testing.(*T).Run(0xc0000ca100, 0x108c69cfe, 0xf, 0x108d21df0, 0x108df7f01) /home/boger/golang/baseline/go/src/testing/testing.go:917 +0x320 testing.runTests.func1(0xc0000ca000) /home/boger/golang/baseline/go/src/testing/testing.go:1157 +0x8c testing.tRunner(0xc0000ca000, 0xc0000a7dd8) /home/boger/golang/baseline/go/src/testing/testing.go:865 +0xdc testing.runTests(0xc00000e0a0, 0x108df7900, 0x46, 0x46, 0x0) /home/boger/golang/baseline/go/src/testing/testing.go:1155 +0x2a0 testing.(*M).Run(0xc00000c080, 0x0) /home/boger/golang/baseline/go/src/testing/testing.go:1072 +0x174 main.main() _testmain.go:366 +0x150 exit status 2 FAIL math 0.005s

+1
laboger on Feb 19, 2019
Read more comments on GitHub
← go: runtime: qemu-arm fails to emulate go user programs
go: runtime: race between stack shrinking and channel send/recv leads to bad sudog values →