cloudflared: Crash after updating to 2021.3.1

After updating to 2021.3.1 from 2020.5.0 I am unable to run many cloudflared operations, but surprisingly I’m able to run some, such as cloudflared -v which gives the correct output. Other commands, such as cloudflared tunnel create output a very long error:
SIGABRT: abort
PC=0x7f71ed29318b m=12 sigcode=18446744073709551610

goroutine 0 [idle]:
runtime: unknown pc 0x7f71ed29318b
stack: frame={sp:0x7f71b4ff8790, fp:0x0} stack=[0x7f71b47f9288,0x7f71b4ff8e88)
00007f71b4ff8690:  0000000000000000  0002ffff00001fa0 
00007f71b4ff86a0:  0000000000000000  0000000000000000 
00007f71b4ff86b0:  0000000000000000  0000000000000000 
00007f71b4ff86c0:  0000000000000000  0000000000000000 
00007f71b4ff86d0:  0000000000000000  0000000000000000 
00007f71b4ff86e0:  0000000000000000  0000000000000000 
00007f71b4ff86f0:  0000000000000000  0000000000000000 
00007f71b4ff8700:  0000000000000000  0000000000000000 
00007f71b4ff8710:  0000000000000000  0000000000000000 
00007f71b4ff8720:  00007f71ed439643  00007f71ed439644 
00007f71b4ff8730:  00007f71ed439643  0000000000000000 
00007f71b4ff8740:  00007f71ed439644  0000000000000000 
00007f71b4ff8750:  0000000000000000  0000000000000000 
00007f71b4ff8760:  0000000000000000  0000000000000000 
00007f71b4ff8770:  000000ff000000ff  000000ff000000ff 
00007f71b4ff8780:  3661313662376639  3033363238663637 
00007f71b4ff8790: <0000000000000000  736c656e6e75742f 
00007f71b4ff87a0:  3361666531333562  6236383936663466 
00007f71b4ff87b0:  6230323031316238  6465643132316636 
00007f71b4ff87c0:  6232356635356632  3539396663633064 
00007f71b4ff87d0:  6563623735366162  3139313361356331 
00007f71b4ff87e0:  3832346362346161  6634333133316634 
00007f71b4ff87f0:  3132383938643634  3166636532616233 
00007f71b4ff8800:  6530363961373936  3832326261636133 
00007f71b4ff8810:  fffffffe7fffffff  ffffffffffffffff 
00007f71b4ff8820:  ffffffffffffffff  ffffffffffffffff 
00007f71b4ff8830:  ffffffffffffffff  ffffffffffffffff 
00007f71b4ff8840:  ffffffffffffffff  ffffffffffffffff 
00007f71b4ff8850:  ffffffffffffffff  ffffffffffffffff 
00007f71b4ff8860:  ffffffffffffffff  ffffffffffffffff 
00007f71b4ff8870:  ffffffffffffffff  ffffffffffffffff 
00007f71b4ff8880:  ffffffffffffffff  ffffffffffffffff 
runtime: unknown pc 0x7f71ed29318b
stack: frame={sp:0x7f71b4ff8790, fp:0x0} stack=[0x7f71b47f9288,0x7f71b4ff8e88)
00007f71b4ff8690:  0000000000000000  0002ffff00001fa0 
00007f71b4ff86a0:  0000000000000000  0000000000000000 
00007f71b4ff86b0:  0000000000000000  0000000000000000 
00007f71b4ff86c0:  0000000000000000  0000000000000000 
00007f71b4ff86d0:  0000000000000000  0000000000000000 
00007f71b4ff86e0:  0000000000000000  0000000000000000 
00007f71b4ff86f0:  0000000000000000  0000000000000000 
00007f71b4ff8700:  0000000000000000  0000000000000000 
00007f71b4ff8710:  0000000000000000  0000000000000000 
00007f71b4ff8720:  00007f71ed439643  00007f71ed439644 
00007f71b4ff8730:  00007f71ed439643  0000000000000000 
00007f71b4ff8740:  00007f71ed439644  0000000000000000 
00007f71b4ff8750:  0000000000000000  0000000000000000 
00007f71b4ff8760:  0000000000000000  0000000000000000 
00007f71b4ff8770:  000000ff000000ff  000000ff000000ff 
00007f71b4ff8780:  3661313662376639  3033363238663637 
00007f71b4ff8790: <0000000000000000  736c656e6e75742f 
00007f71b4ff87a0:  3361666531333562  6236383936663466 
00007f71b4ff87b0:  6230323031316238  6465643132316636 
00007f71b4ff87c0:  6232356635356632  3539396663633064 
00007f71b4ff87d0:  6563623735366162  3139313361356331 
00007f71b4ff87e0:  3832346362346161  6634333133316634 
00007f71b4ff87f0:  3132383938643634  3166636532616233 
00007f71b4ff8800:  6530363961373936  3832326261636133 
00007f71b4ff8810:  fffffffe7fffffff  ffffffffffffffff 
00007f71b4ff8820:  ffffffffffffffff  ffffffffffffffff 
00007f71b4ff8830:  ffffffffffffffff  ffffffffffffffff 
00007f71b4ff8840:  ffffffffffffffff  ffffffffffffffff 
00007f71b4ff8850:  ffffffffffffffff  ffffffffffffffff 
00007f71b4ff8860:  ffffffffffffffff  ffffffffffffffff 
00007f71b4ff8870:  ffffffffffffffff  ffffffffffffffff 
00007f71b4ff8880:  ffffffffffffffff  ffffffffffffffff 

goroutine 1 [syscall]:
runtime.cgocall(0xedae60, 0xc0006235b8, 0xeada2f)
	/usr/local/go/src/runtime/cgocall.go:133 +0x5b fp=0xc000623588 sp=0xc000623550 pc=0x409c3b
crypto/internal/boring._Cfunc__goboringcrypto_RAND_bytes(0xc00064ffc0, 0x20, 0xc000000000)
	_cgo_gotypes.go:1139 +0x4d fp=0xc0006235b8 sp=0xc000623588 pc=0x55798d
crypto/internal/boring.randReader.Read(0x0, 0xc00064ffc0, 0x20, 0x20, 0xc0000cdcc0, 0x413c50, 0xc000478f40)
	/usr/local/go/src/crypto/internal/boring/rand.go:21 +0x51 fp=0xc0006235e0 sp=0xc0006235b8 pc=0x55cb51
crypto/internal/boring.(*randReader).Read(0x1410b80, 0xc00064ffc0, 0x20, 0x20, 0x0, 0x20, 0x7f71ed20a7d0)
	<autogenerated>:1 +0x5a fp=0xc000623628 sp=0xc0006235e0 pc=0x564f3a
io.ReadAtLeast(0x142ca20, 0x1410b80, 0xc00064ffc0, 0x20, 0x20, 0x20, 0x1106400, 0x1106101, 0xc00064ffc0)
	/usr/local/go/src/io/io.go:314 +0x87 fp=0xc000623688 sp=0xc000623628 pc=0x490827
io.ReadFull(...)
	/usr/local/go/src/io/io.go:333
crypto/rand.Read(0xc00064ffc0, 0x20, 0x20, 0xc00064ffc0, 0x0, 0x2000)
	/usr/local/go/src/crypto/rand/rand.go:24 +0x5c fp=0xc0006236e0 sp=0xc000623688 pc=0x592a1c
github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel.generateTunnelSecret(...)
	/cfsetup_build/cmd/cloudflared/tunnel/subcommands.go:139
github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel.(*subcommandContext).create(0xc0000e6f40, 0x7fff9c5eb78b, 0xa, 0xc00003d580, 0x3a, 0x120, 0xc000623900, 0x411b0a)
	/cfsetup_build/cmd/cloudflared/tunnel/subcommand_context.go:156 +0xc5 fp=0xc0006238a0 sp=0xc0006236e0 pc=0xec69e5
github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel.createCommand(0xc0000e6ec0, 0x0, 0x0)
	/cfsetup_build/cmd/cloudflared/tunnel/subcommands.go:157 +0x1c7 fp=0xc000623928 sp=0xc0006238a0 pc=0xeca0c7
github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil.ErrorHandler.func1(0xc0000e6ec0, 0x3, 0x4)
	/cfsetup_build/cmd/cloudflared/cliutil/errors.go:26 +0x44 fp=0xc0006239c8 sp=0xc000623928 pc=0xdc0804
github.com/urfave/cli/v2.(*Command).Run(0xc0002326c0, 0xc000310780, 0x0, 0x0)
	/cfsetup_build/vendor/github.com/urfave/cli/v2/command.go:163 +0x43e fp=0xc000623a90 sp=0xc0006239c8 pc=0x96eade
github.com/urfave/cli/v2.(*App).RunAsSubcommand(0xc00020ab60, 0xc000532440, 0x0, 0x0)
	/cfsetup_build/vendor/github.com/urfave/cli/v2/app.go:434 +0xa0d fp=0xc000623bc8 sp=0xc000623a90 pc=0x96d36d
github.com/urfave/cli/v2.(*Command).startApp(0xc000168a20, 0xc000532440, 0x1, 0x0)
	/cfsetup_build/vendor/github.com/urfave/cli/v2/command.go:279 +0x6db fp=0xc000623c70 sp=0xc000623bc8 pc=0x96fd5b
github.com/urfave/cli/v2.(*Command).Run(0xc000168a20, 0xc000532440, 0x0, 0x0)
	/cfsetup_build/vendor/github.com/urfave/cli/v2/command.go:94 +0x80d fp=0xc000623d38 sp=0xc000623c70 pc=0x96eead
github.com/urfave/cli/v2.(*App).RunContext(0xc00020a820, 0x1442ea0, 0xc000036100, 0xc000032080, 0x4, 0x4, 0x0, 0x0)
	/cfsetup_build/vendor/github.com/urfave/cli/v2/app.go:313 +0x790 fp=0xc000623e58 sp=0xc000623d38 pc=0x96c510
github.com/urfave/cli/v2.(*App).Run(...)
	/cfsetup_build/vendor/github.com/urfave/cli/v2/app.go:224
main.runApp(0xc00020a820, 0xc000100120)
	/cfsetup_build/cmd/cloudflared/linux_service.go:42 +0x2c5 fp=0xc000623ed8 sp=0xc000623e58 pc=0xed46e5
main.main()
	/cfsetup_build/cmd/cloudflared/main.go:86 +0x56e fp=0xc000623f88 sp=0xc000623ed8 pc=0xed6e2e
runtime.main()
	/usr/local/go/src/runtime/proc.go:204 +0x209 fp=0xc000623fe0 sp=0xc000623f88 pc=0x440369
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc000623fe8 sp=0xc000623fe0 pc=0x4765c1

goroutine 53 [select]:
net.(*sysDialer).dialParallel(0xc00021e080, 0x1442ee0, 0xc0002063c0, 0xc0000ae1e0, 0x2, 0x2, 0xc0000ae200, 0x2, 0x2, 0x0, ...)
	/usr/local/go/src/net/dial.go:490 +0x385
net.(*Dialer).DialContext(0xc0000422a0, 0x1442ee0, 0xc000206060, 0x12abe13, 0x3, 0xc00001e040, 0x19, 0x0, 0x0, 0x0, ...)
	/usr/local/go/src/net/dial.go:423 +0x4d4
net/http.(*Transport).dial(0x1d41620, 0x1442ee0, 0xc000206060, 0x12abe13, 0x3, 0xc00001e040, 0x19, 0x0, 0x0, 0x0, ...)
	/usr/local/go/src/net/http/transport.go:1141 +0x1fd
net/http.(*Transport).dialConn(0x1d41620, 0x1442ee0, 0xc000206060, 0x0, 0x12c7698, 0x5, 0xc00001e040, 0x19, 0x0, 0xc0003126c0, ...)
	/usr/local/go/src/net/http/transport.go:1575 +0x1abb
net/http.(*Transport).dialConnFor(0x1d41620, 0xc0000de0b0)
	/usr/local/go/src/net/http/transport.go:1421 +0xc6
created by net/http.(*Transport).queueForDial
	/usr/local/go/src/net/http/transport.go:1390 +0x40f

goroutine 21 [select]:
net/http.(*Transport).getConn(0x1d41620, 0xc000428040, 0x0, 0x12c7698, 0x5, 0xc00001e040, 0x19, 0x0, 0x0, 0x0, ...)
	/usr/local/go/src/net/http/transport.go:1347 +0x5ac
net/http.(*Transport).roundTrip(0x1d41620, 0xc00024e100, 0x127b2c0, 0x1, 0xc00024e100)
	/usr/local/go/src/net/http/transport.go:569 +0x77c
net/http.(*Transport).RoundTrip(0x1d41620, 0xc00024e100, 0x1d41620, 0xc00d8114c2b55ccb, 0xdf9de5511)
	/usr/local/go/src/net/http/roundtrip.go:17 +0x35
net/http.send(0xc00024e000, 0x142c200, 0x1d41620, 0xc00d8114c2b55ccb, 0xdf9de5511, 0x1db5480, 0xc0000b0020, 0xc00d8114c2b55ccb, 0x1, 0x0)
	/usr/local/go/src/net/http/client.go:252 +0x453
net/http.(*Client).send(0xc000708000, 0xc00024e000, 0xc00d8114c2b55ccb, 0xdf9de5511, 0x1db5480, 0xc0000b0020, 0x0, 0x1, 0xc00048c1a5)
	/usr/local/go/src/net/http/client.go:176 +0xff
net/http.(*Client).do(0xc000708000, 0xc00024e000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:718 +0x45f
net/http.(*Client).Do(...)
	/usr/local/go/src/net/http/client.go:586
github.com/cloudflare/cloudflared/cmd/cloudflared/updater.(*WorkersService).Check(0xc000214000, 0x0, 0x0, 0x0, 0x0)
	/cfsetup_build/cmd/cloudflared/updater/workers_service.go:73 +0x312
github.com/cloudflare/cloudflared/cmd/cloudflared/updater.CheckForUpdate(0x1, 0x12ab410, 0x0, 0x0, 0x0, 0x0, 0x0)
	/cfsetup_build/cmd/cloudflared/updater/update.go:100 +0x16c
github.com/cloudflare/cloudflared/cmd/cloudflared/updater.StartWarningCheck.func1(0xc0000e6ec0, 0xc0000fc000)
	/cfsetup_build/cmd/cloudflared/updater/check.go:25 +0xb3
created by github.com/cloudflare/cloudflared/cmd/cloudflared/updater.StartWarningCheck
	/cfsetup_build/cmd/cloudflared/updater/check.go:17 +0x67

goroutine 56 [IO wait]:
internal/poll.runtime_pollWait(0x7f71c4549eb8, 0x77, 0xc0001b6900)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc00021e118, 0x77, 0x1442e00, 0xc000206c00, 0xc00021e100)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitWrite(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:96
internal/poll.(*FD).WaitWrite(...)
	/usr/local/go/src/internal/poll/fd_unix.go:508
net.(*netFD).connect(0xc00021e100, 0x1442ee0, 0xc000206c00, 0x0, 0x0, 0x142c6c0, 0xc00001e1a0, 0x0, 0x0, 0x0, ...)
	/usr/local/go/src/net/fd_unix.go:141 +0x25c
net.(*netFD).dial(0xc00021e100, 0x1442ee0, 0xc000206c00, 0x14496a0, 0x0, 0x14496a0, 0xc000709020, 0x0, 0xc000077b01, 0xc000077b40)
	/usr/local/go/src/net/sock_posix.go:149 +0x108
net.socket(0x1442ee0, 0xc000206c00, 0x12abe13, 0x3, 0x2, 0x1, 0x0, 0x0, 0x14496a0, 0x0, ...)
	/usr/local/go/src/net/sock_posix.go:70 +0x1c5
net.internetSocket(0x1442ee0, 0xc000206c00, 0x12abe13, 0x3, 0x14496a0, 0x0, 0x14496a0, 0xc000709020, 0x1, 0x0, ...)
	/usr/local/go/src/net/ipsock_posix.go:141 +0x145
net.(*sysDialer).doDialTCP(0xc00021e080, 0x1442ee0, 0xc000206c00, 0x0, 0xc000709020, 0x1159ec0, 0x1de7c60, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:65 +0xc5
net.(*sysDialer).dialTCP(0xc00021e080, 0x1442ee0, 0xc000206c00, 0x0, 0xc000709020, 0x37fb2dc90, 0xc000214640, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:61 +0xd7
net.(*sysDialer).dialSingle(0xc00021e080, 0x1442ee0, 0xc000206c00, 0x14372a0, 0xc000709020, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/dial.go:580 +0x5e5
net.(*sysDialer).dialSerial(0xc00021e080, 0x1442e60, 0xc000428200, 0xc0000ae1e0, 0x2, 0x2, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/dial.go:548 +0x152
net.(*sysDialer).dialParallel.func1(0x1442e60, 0xc000428200, 0xc000492101)
	/usr/local/go/src/net/dial.go:468 +0x9b
created by net.(*sysDialer).dialParallel
	/usr/local/go/src/net/dial.go:483 +0x265

goroutine 57 [select]:
net.(*netFD).connect.func2(0x1442ee0, 0xc000206c00, 0xc00021e100, 0xc000266c60, 0xc000266c00)
	/usr/local/go/src/net/fd_unix.go:118 +0xc5
created by net.(*netFD).connect
	/usr/local/go/src/net/fd_unix.go:117 +0x234

rax    0x0
rbx    0x7f71b4ff9700
rcx    0x7f71ed29318b
rdx    0x0
rdi    0x2
rsi    0x7f71b4ff8790
rbp    0x1de76a8
rsp    0x7f71b4ff8790
r8     0x0
r9     0x7f71b4ff8790
r10    0x8
r11    0x246
r12    0x7f71b4ff8c20
r13    0x7f7190000c98
r14    0x7f7190000dd0
r15    0x7f7190000dd0
rip    0x7f71ed29318b
rflags 0x246
cs     0x33
fs     0x0
gs     0x0
Any help would be appreciated.
About this issue

Original URL
State: open
Created 3 years ago
Comments: 16 (7 by maintainers)
Most upvoted comments

I’m having the same issue on a single VPS over five. All of them are with Ubuntu 20.04. All of them configured with Ansible, so they are virtually identical. Just a single one fails the same. This is a first installation of cloudflared, no config. It crashes with cloudflared login already. I also confirm that the issue is not present with cloudflared version 2021.1.5.
dgraziotin on Apr 25, 2021