go: runtime: "fatal: morestack on g0" on amd64 after upgrade to Go 1.21

Does this issue reproduce with the latest release?

Yes.

What did you do?

https://github.com/VirrageS/fluent-bit-go-bug here is full code to repo. I will try to make it smaller but for now this is all I have.

When running this project with:

docker build --build-arg="GO_VERSION=1.20" -t fluentbit .
docker run -t fluentbit

it runs correctly.

But when I change to GO_VERSION=1.21 it fails with fatal: morestack on g0:

docker build --build-arg="GO_VERSION=1.21" -t fluentbit .
docker run -t fluentbit

It looks like regression in Go 1.21. I found similar issues but those seem to be quite old and are not tied to Go 1.21 version.

I have tried to strace the binary and this is what I got:

Details

18    <... set_robust_list resumed>)    = 0
18    prctl(PR_SET_NAME, "flb-pipeline" <unfinished ...>
18    <... prctl resumed>)              = 0
18    futex(0x55c46520b460, FUTEX_WAKE_PRIVATE, 2147483647 <unfinished ...>
18    <... futex resumed>)              = 0
18    mmap(NULL, 2097152, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0x7f93877fd000
18    munmap(0x7f93877fd000, 2097152 <unfinished ...>
18    <... munmap resumed>)             = 0
18    mmap(NULL, 4190208, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0x7f93875fe000
18    munmap(0x7f93875fe000, 8192 <unfinished ...>
18    <... munmap resumed>)             = 0
18    munmap(0x7f9387800000, 2084864 <unfinished ...>
18    <... munmap resumed>)             = 0
18    mmap(NULL, 2097152, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0 <unfinished ...>
18    <... mmap resumed>)               = 0x7f9387400000
18    epoll_create1(EPOLL_CLOEXEC)      = 8
18    pipe([9, 10])                     = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 9, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=3559108056, u64=140272896028120}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    futex(0x55c46520b480, FUTEX_WAKE_PRIVATE, 2147483647) = 0
18    epoll_create1(EPOLL_CLOEXEC <unfinished ...>
18    <... epoll_create1 resumed>)      = 11
18    pipe([12, 13])                    = 0
18    epoll_ctl(11, EPOLL_CTL_ADD, 12, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269204480, u64=140271606124544}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    pipe([14, 15])                    = 0
18    epoll_ctl(11, EPOLL_CTL_ADD, 14, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269286400, u64=140271606206464}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    pipe([16, 17])                    = 0
18    epoll_ctl(11, EPOLL_CTL_ADD, 16, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269286592, u64=140271606206656}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    mmap(NULL, 8392704, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f9386bff000
18    mprotect(0x7f9386c00000, 8388608, PROT_READ|PROT_WRITE|PROT_EXEC <unfinished ...>
18    <... mprotect resumed>)           = 0
18    clone(child_stack=0x7f93873f54f0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID <unfinished ...>
18    <... clone resumed>, parent_tid=[19], tls=0x7f93873ff700, child_tidptr=0x7f93873ff9d0) = 19
18    futex(0x7f93874150a8, FUTEX_WAIT_PRIVATE, 0, NULL <unfinished ...>
18    <... futex resumed>)              = 0
18    futex(0x7f93874150b0, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
18    <... futex resumed>)              = 0
18    getpid()                          = 10
18    openat(AT_FDCWD, "/etc/localtime", O_RDONLY|O_CLOEXEC <unfinished ...>
18    <... openat resumed>)             = 18
18    fstat(18,  <unfinished ...>
18    <... fstat resumed>{st_mode=S_IFREG|0644, st_size=118, ...}) = 0
18    fstat(18,  <unfinished ...>
18    <... fstat resumed>{st_mode=S_IFREG|0644, st_size=118, ...}) = 0
18    read(18, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1\0\0\0\1\0\0\0\0"..., 4096) = 118
18    lseek(18, -62, SEEK_CUR <unfinished ...>
18    <... lseek resumed>)              = 56
18    read(18, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1\0\0\0\1\0\0\0\0"..., 4096) = 62
18    close(18 <unfinished ...>
18    <... close resumed>)              = 0
18    write(15, "T\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096 <unfinished ...>
18    <... write resumed>)              = 4096
18    pipe( <unfinished ...>
18    <... pipe resumed>[18, 19])       = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 18, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=3559107648, u64=140272896027712}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    write(15, "m\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096 <unfinished ...>
18    <... write resumed>)              = 4096
18    write(15, "7\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096 <unfinished ...>
18    <... write resumed>)              = 4096
18    write(15, "7\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096 <unfinished ...>
18    <... write resumed>)              = 4096
18    timerfd_create(CLOCK_MONOTONIC, 0 <unfinished ...>
18    <... timerfd_create resumed>)     = 20
18    timerfd_settime(20, TFD_TIMER_ABSTIME, {it_interval={tv_sec=10, tv_nsec=0}, it_value={tv_sec=16839370, tv_nsec=0}},  <unfinished ...>
18    <... timerfd_settime resumed>NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 20, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269405184, u64=140271606325248}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    write(15, "=\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096) = 4096
18    write(15, "X\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096 <unfinished ...>
18    <... write resumed>)              = 4096
18    pipe( <unfinished ...>
18    <... pipe resumed>[21, 22])       = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 21, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=3559552000, u64=140272896472064}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    openat(AT_FDCWD, "/sys/devices/system/cpu/online", O_RDONLY|O_CLOEXEC <unfinished ...>
18    <... openat resumed>)             = 23
18    read(23,  <unfinished ...>
18    <... read resumed>"0-11\n", 8192) = 5
18    close(23)                         = 0
18    openat(AT_FDCWD, "/proc/stat", O_RDONLY <unfinished ...>
18    <... openat resumed>)             = 23
18    fstat(23, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
18    read(23,  <unfinished ...>
18    <... read resumed>"cpu  531693645 1691808 251625914"..., 1024) = 1024
18    close(23)                         = 0
18    pipe( <unfinished ...>
18    <... pipe resumed>[23, 24])       = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 23, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=3559483392, u64=140272896403456}}) = 0
18    futex(0x55c46520b45c, FUTEX_WAKE_PRIVATE, 2147483647 <unfinished ...>
18    <... futex resumed>)              = 0
18    epoll_create1(EPOLL_CLOEXEC)      = 25
18    pipe( <unfinished ...>
18    <... pipe resumed>[26, 27])       = 0
18    epoll_ctl(25, EPOLL_CTL_ADD, 26, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269552640, u64=140271606472704}}) = 0
18    pipe( <unfinished ...>
18    <... pipe resumed>[28, 29])       = 0
18    epoll_ctl(11, EPOLL_CTL_ADD, 28, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269286784, u64=140271606206848}}) = 0
18    mmap(NULL, 8392704, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f93863fe000
18    mprotect(0x7f93863ff000, 8388608, PROT_READ|PROT_WRITE|PROT_EXEC <unfinished ...>
18    <... mprotect resumed>)           = 0
18    clone(child_stack=0x7f9386bf44f0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID <unfinished ...>
18    <... clone resumed>, parent_tid=[20], tls=0x7f9386bfe700, child_tidptr=0x7f9386bfe9d0) = 20
18    pipe( <unfinished ...>
18    <... pipe resumed>[30, 31])       = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 30, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=3559484160, u64=140272896404224}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    rt_sigprocmask(SIG_SETMASK, NULL,  <unfinished ...>
18    <... rt_sigprocmask resumed>[], 8) = 0
18    rt_sigprocmask(SIG_SETMASK, ~[],  <unfinished ...>
18    <... rt_sigprocmask resumed>NULL, 8) = 0
18    sched_getaffinity(18, 32,  <unfinished ...>
18    <... sched_getaffinity resumed>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) = 8
18    sigaltstack(NULL, {ss_sp=NULL, ss_flags=SS_DISABLE, ss_size=0}) = 0
18    sigaltstack({ss_sp=0xc000180000, ss_flags=0, ss_size=32768},  <unfinished ...>
18    <... sigaltstack resumed>NULL)    = 0
18    rt_sigprocmask(SIG_SETMASK, [],  <unfinished ...>
18    <... rt_sigprocmask resumed>NULL, 8) = 0
18    gettid( <unfinished ...>
18    <... gettid resumed>)             = 18
18    futex(0x7f93d3dcc900, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
18    <... futex resumed>)              = 1
18    futex(0x7f93d3dcc148, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
18    <... futex resumed>)              = 1
18    futex(0xc000100d48, FUTEX_WAIT_PRIVATE, 0, NULL <unfinished ...>
18    <... futex resumed>)              = -1 EAGAIN (Resource temporarily unavailable)
18    --- SIGURG {si_signo=SIGURG, si_code=SI_TKILL, si_pid=10, si_uid=0} ---
18    rt_sigreturn({mask=[]} <unfinished ...>
18    <... rt_sigreturn resumed>)       = -1 EAGAIN (Resource temporarily unavailable)
18    write(1, "[flb-go] plugin parameter = ''\n", 31 <unfinished ...>
18    <... write resumed>)              = 31
18    timerfd_create(CLOCK_MONOTONIC, 0 <unfinished ...>
18    <... timerfd_create resumed>)     = 36
18    timerfd_settime(36, TFD_TIMER_ABSTIME, {it_interval={tv_sec=1, tv_nsec=0}, it_value={tv_sec=16839361, tv_nsec=0}},  <unfinished ...>
18    <... timerfd_settime resumed>NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 36, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=3559107912, u64=140272896027976}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    timerfd_create(CLOCK_MONOTONIC, 0 <unfinished ...>
18    <... timerfd_create resumed>)     = 37
18    timerfd_settime(37, TFD_TIMER_ABSTIME, {it_interval={tv_sec=5, tv_nsec=0}, it_value={tv_sec=16839365, tv_nsec=0}},  <unfinished ...>
18    <... timerfd_settime resumed>NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 37, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269406144, u64=140271606326208}}) = 0
18    timerfd_create(CLOCK_MONOTONIC, 0 <unfinished ...>
18    <... timerfd_create resumed>)     = 38
18    timerfd_settime(38, TFD_TIMER_ABSTIME, {it_interval={tv_sec=1, tv_nsec=0}, it_value={tv_sec=16839361, tv_nsec=0}},  <unfinished ...>
18    <... timerfd_settime resumed>NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 38, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269396208, u64=140271606316272}}) = 0
18    write(15, "<\0\0\0\0\0\0\0[2023/09/04 10:18:13] [ "..., 4096 <unfinished ...>
18    <... write resumed>)              = 4096
18    timerfd_create(CLOCK_MONOTONIC, 0 <unfinished ...>
18    <... timerfd_create resumed>)     = 39
18    timerfd_settime(39, TFD_TIMER_ABSTIME, {it_interval={tv_sec=1, tv_nsec=0}, it_value={tv_sec=16839361, tv_nsec=0}},  <unfinished ...>
18    <... timerfd_settime resumed>NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 39, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269405504, u64=140271606325568}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    timerfd_create(CLOCK_MONOTONIC, 0) = 40
18    timerfd_settime(40, TFD_TIMER_ABSTIME, {it_interval={tv_sec=1, tv_nsec=500000000}, it_value={tv_sec=16839361, tv_nsec=0}},  <unfinished ...>
18    <... timerfd_settime resumed>NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 40, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269406304, u64=140271606326368}}) = 0
18    timerfd_create(CLOCK_MONOTONIC, 0 <unfinished ...>
18    <... timerfd_create resumed>)     = 41
18    timerfd_settime(41, TFD_TIMER_ABSTIME, {it_interval={tv_sec=0, tv_nsec=250000000}, it_value={tv_sec=16839360, tv_nsec=0}}, NULL) = 0
18    epoll_ctl(8, EPOLL_CTL_ADD, 41, {EPOLLIN|EPOLLERR|EPOLLHUP|EPOLLRDHUP, {u32=2269406464, u64=140271606326528}} <unfinished ...>
18    <... epoll_ctl resumed>)          = 0
18    write(7, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[], 256, 0) = 0
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8, [{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8, [{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[{EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}, {EPOLLIN, {u32=2269406304, u64=140271606326368}}, {EPOLLIN, {u32=3559107912, u64=140272896027976}}, {EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 5
18    read(40,  <unfinished ...>
18    <... read resumed>"\1\0\0\0\0\0\0\0", 8) = 8
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[{EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}, {EPOLLIN, {u32=3559107912, u64=140272896027976}}, {EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, 0) = 4
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [{EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}, {EPOLLIN, {u32=3559107912, u64=140272896027976}}], 256, 0) = 3
18    read(36, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [{EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}], 256, 0) = 2
18    read(38, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [{EPOLLIN, {u32=2269405504, u64=140271606325568}}], 256, 0) = 1
18    read(39, "\1\0\0\0\0\0\0\0", 8)   = 8
18    openat(AT_FDCWD, "/proc/stat", O_RDONLY) = 42
18    fstat(42, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
18    read(42, "cpu  531693681 1691808 251625926"..., 1024) = 1024
18    close(42)                         = 0
18    getpid()                          = 10
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8, [{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8, [{EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 1
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [], 256, 0)         = 0
18    epoll_wait(8, [{EPOLLIN, {u32=3559107912, u64=140272896027976}}, {EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}, {EPOLLIN, {u32=2269406464, u64=140271606326528}}], 256, -1) = 4
18    read(41, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8, [{EPOLLIN, {u32=3559107912, u64=140272896027976}}, {EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}], 256, 0) = 3
18    read(36, "\1\0\0\0\0\0\0\0", 8)   = 8
18    write(27, "@fD\207\223\177\0\0", 8) = 8
18    write(10, "P5E\207\223\177\0\0", 8 <unfinished ...>
18    <... write resumed>)              = 8
18    epoll_wait(8, [{EPOLLIN, {u32=2269396208, u64=140271606316272}}, {EPOLLIN, {u32=2269405504, u64=140271606325568}}, {EPOLLIN, {u32=3559108056, u64=140272896028120}}], 256, 0) = 3
18    read(38, "\1\0\0\0\0\0\0\0", 8)   = 8
18    epoll_wait(8,  <unfinished ...>
18    <... epoll_wait resumed>[{EPOLLIN, {u32=2269405504, u64=140271606325568}}, {EPOLLIN, {u32=3559108056, u64=140272896028120}}], 256, 0) = 2
18    read(39, "\1\0\0\0\0\0\0\0", 8)   = 8
18    openat(AT_FDCWD, "/proc/stat", O_RDONLY <unfinished ...>
18    <... openat resumed>)             = 42
18    fstat(42,  <unfinished ...>
18    <... fstat resumed>{st_mode=S_IFREG|0444, st_size=0, ...}) = 0
18    read(42,  <unfinished ...>
18    <... read resumed>"cpu  531693722 1691808 251625947"..., 1024) = 1024
18    close(42 <unfinished ...>
18    <... close resumed>)              = 0
18    getpid( <unfinished ...>
18    <... getpid resumed>)             = 10
18    epoll_wait(8, [{EPOLLIN, {u32=3559108056, u64=140272896028120}}, {EPOLLIN, {u32=3559483392, u64=140272896403456}}], 256, 0) = 2
18    read(9, "P5E\207\223\177\0\0", 8) = 8
18    write(2, "fatal: morestack on g0\n", 23) = 23
18    --- SIGTRAP {si_signo=SIGTRAP, si_code=SI_KERNEL} ---
18    +++ killed by SIGTRAP (core dumped) +++

What did you expect to see?

Same behavior as with GO_VERSION=1.20.

What did you see instead?

fatal: morestack on g0

About this issue

  • Original URL
  • State: closed
  • Created 10 months ago
  • Comments: 28 (11 by maintainers)

Commits related to this issue

Most upvoted comments

@gopherbot please backport to 1.21. This is a fatal crash with no workaround for applications using C coroutine libraries and calling into Go. The fix here is fairly complex, so this is a medium+ risk backport, but we don’t have a simpler workaround to backport. That said, the fix has baked in post-submit for 2+ weeks without further issue.

@VirrageS could you verify that your application works properly with https://go.dev/cl/525455?

(If you aren’t familiar with building the toolchain from a CL, you can use gotip download 525455: https://pkg.go.dev/golang.org/dl/gotip)