go: net,runtime: apparent deadlock in (*net.conn).Close and runtime.netpollblock on arm64 platforms

One theory:

Inside netpollblock and netpollunblock, gpp is read using a non-atomic read. In the normal case, writes use atomics, creating a barrier.

However, when consuming a ready event, netpollblock resets gpp using a non-atomic store, thus skipping a barrier.

In netpollunblock, we can return early if gpp == pdReady, not actually performing an unblock.

Thus I posit that an earlier netpollblock sees pdReady and resets to 0. Later, on another core we read gpp and see the stale pdReady and incorrectly return early without triggering a new unblock.

I haven’t validated this theory, but my log does show that this pollDesc has earlier calls to netpollblock, so it is possible. If this is the case, I believe the fix is simply to use a atomic store to reset gpp to 0.

I reproduced this on linux-arm64-aws quite quickly, so I’ll dig in some more:

$ GOROOT=$(pwd) gomote run $(gomote-instance) ./go/bin/go test -count=1000 crypto/tls
panic: test timed out after 10m0s

goroutine 117595 [running]:
testing.(*M).startAlarm.func1()
        /workdir/go/src/testing/testing.go:1961 +0x8c
created by time.goFunc
        /workdir/go/src/time/sleep.go:176 +0x3c

goroutine 1 [chan receive, 6 minutes]:
testing.(*T).Run(0x40003e4340, {0x2cce47, 0x75bdd8b719}, 0x2eef18)
        /workdir/go/src/testing/testing.go:1425 +0x33c
testing.runTests.func1(0x0)
        /workdir/go/src/testing/testing.go:1771 +0x74
testing.tRunner(0x40003e4340, 0x4000133b68)
        /workdir/go/src/testing/testing.go:1377 +0x110
testing.runTests(0x4000106b40, {0x4e2da0, 0x95, 0x95}, {0x46c, 0x4000014280, 0x4e7040})
        /workdir/go/src/testing/testing.go:1769 +0x3e0
testing.(*M).Run(0x4000106b40)
        /workdir/go/src/testing/testing.go:1657 +0x510
crypto/tls.runMain(0x400005c180)
        /workdir/go/src/crypto/tls/handshake_test.go:381 +0x5b4
crypto/tls.TestMain(0x4000045ec8)
        /workdir/go/src/crypto/tls/handshake_test.go:333 +0x80
main.main()
        _testmain.go:353 +0x1ec

goroutine 6 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0xffff65e8e818, 0x72)
        /workdir/go/src/runtime/netpoll.go:229 +0x9c
internal/poll.(*pollDesc).wait(0x400013a080, 0x87ac4, 0x0)
        /workdir/go/src/internal/poll/fd_poll_runtime.go:84 +0x2c
internal/poll.(*pollDesc).waitRead(...)
        /workdir/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0x400013a080)
        /workdir/go/src/internal/poll/fd_unix.go:508 +0x1d8
net.(*netFD).accept(0x400013a080)
        /workdir/go/src/net/fd_unix.go:173 +0x28
net.(*TCPListener).accept(0x400000e198)
        /workdir/go/src/net/tcpsock_posix.go:140 +0x2c
net.(*TCPListener).Accept(0x400000e198)
        /workdir/go/src/net/tcpsock.go:262 +0x30
crypto/tls.localServer({0x340ac8, 0x400000e198})
        /workdir/go/src/crypto/tls/handshake_test.go:247 +0x5c
created by crypto/tls.runMain
        /workdir/go/src/crypto/tls/handshake_test.go:353 +0x1e0

goroutine 117621 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0xffff65e8db68, 0x72)
        /workdir/go/src/runtime/netpoll.go:229 +0x9c
internal/poll.(*pollDesc).wait(0x400013a500, 0x40004e3440, 0x0)
        /workdir/go/src/internal/poll/fd_poll_runtime.go:84 +0x2c
internal/poll.(*pollDesc).waitRead(...)
        /workdir/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0x400013a500, {0x40004e3440, 0x205, 0x205})
        /workdir/go/src/internal/poll/fd_unix.go:167 +0x1ec
net.(*netFD).Read(0x400013a500, {0x40004e3440, 0x0, 0x1c58933d99f3})
        /workdir/go/src/net/fd_posix.go:56 +0x2c
net.(*conn).Read(0x4000558058, {0x40004e3440, 0x40003f2698, 0x160ca8})
        /workdir/go/src/net/net.go:183 +0x38
crypto/tls.(*atLeastReader).Read(0x40004e05a0, {0x40004e3440, 0x0, 0x1ac18})
        /workdir/go/src/crypto/tls/conn.go:777 +0x44
bytes.(*Buffer).ReadFrom(0x4000077af8, {0x33f328, 0x40004e05a0})
        /workdir/go/src/bytes/buffer.go:204 +0x94
crypto/tls.(*Conn).readFromUntil(0x4000077880, {0x33f588, 0x4000558058}, 0x160ca8)
        /workdir/go/src/crypto/tls/conn.go:799 +0xd8
crypto/tls.(*Conn).readRecordOrCCS(0x4000077880, 0x0)
        /workdir/go/src/crypto/tls/conn.go:606 +0xdc
crypto/tls.(*Conn).readRecord(...)
        /workdir/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).readHandshake(0x4000077880)
        /workdir/go/src/crypto/tls/conn.go:1009 +0x4c
crypto/tls.(*Conn).clientHandshake(0x4000077880, {0x340dd0, 0x40005bc480})
        /workdir/go/src/crypto/tls/handshake_client.go:179 +0x1c0
crypto/tls.(*Conn).handshakeContext(0x4000077880, {0x340e08, 0x400001a040})
        /workdir/go/src/crypto/tls/conn.go:1445 +0x3c8
crypto/tls.(*Conn).HandshakeContext(...)
        /workdir/go/src/crypto/tls/conn.go:1395
crypto/tls.(*Conn).Handshake(...)
        /workdir/go/src/crypto/tls/conn.go:1379
crypto/tls.TestHostnameInSNI.func1({0x2cacbe, 0xb})
        /workdir/go/src/crypto/tls/handshake_client_test.go:1439 +0x154
created by crypto/tls.TestHostnameInSNI
        /workdir/go/src/crypto/tls/handshake_client_test.go:1438 +0x184

goroutine 117585 [semacquire, 6 minutes]:
internal/poll.runtime_Semacquire(0xffff65e8db68)
        /workdir/go/src/runtime/sema.go:61 +0x2c
internal/poll.(*FD).Close(0x400013a500)
        /workdir/go/src/internal/poll/fd_unix.go:116 +0x68
net.(*netFD).Close(0x400013a500)
        /workdir/go/src/net/fd_posix.go:38 +0x44
net.(*conn).Close(0x4000558058)
        /workdir/go/src/net/net.go:207 +0x34
crypto/tls.TestHostnameInSNI(0x40003e4820)
        /workdir/go/src/crypto/tls/handshake_client_test.go:1453 +0x278
testing.tRunner(0x40003e4820, 0x2eef18)
        /workdir/go/src/testing/testing.go:1377 +0x110
created by testing.(*T).Run
        /workdir/go/src/testing/testing.go:1424 +0x328
FAIL    crypto/tls      600.010s
FAIL
Error running run: exit status 1