moby: containerd segmentation fault during startup on armhf
Description
It seems the issues in moby/moby#38175 may not be entirely resolved. I originally opened containerd/containerd#4112 upstream but have been pointed back here as it may be a build or packing issue.
Fresh install on Raspbian 10 Buster and it’s not possible to start containerd as it always exits with a segfault.
Steps to reproduce the issue:
- Follow docker-ce installation instructions
- During installation docker fails to start, but in fact that’s because dependency containerd failed to start
Describe the results you received:
Attempting to start containerd in debug mode fails with a segfault before anything happens, so running it through strace may help identify the issue:
# strace /usr/bin/containerd -debug
execve("/usr/bin/containerd", ["/usr/bin/containerd", "-debug"], 0xbedb58b4 /* 56 vars */) = 0
brk(NULL) = 0x31c2000
uname({sysname="Linux", nodename="raspbian", ...}) = 0
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=41986, ...}) = 0
mmap2(NULL, 41986, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb6f0b000
close(3) = 0
openat(AT_FDCWD, "/lib/arm-linux-gnueabihf/libpthread.so.0", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\3\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\224O\0\0004\0\0\0"..., 512) = 512
_llseek(3, 129016, [129016], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1400) = 1400
_llseek(3, 94664, [94664], SEEK_SET) = 0
read(3, "A.\0\0\0aeabi\0\1$\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\23\1\24"..., 47) = 47
_llseek(3, 129016, [129016], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1400) = 1400
_llseek(3, 94664, [94664], SEEK_SET) = 0
read(3, "A.\0\0\0aeabi\0\1$\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\23\1\24"..., 47) = 47
_llseek(3, 129016, [129016], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1400) = 1400
_llseek(3, 94664, [94664], SEEK_SET) = 0
read(3, "A.\0\0\0aeabi\0\1$\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\23\1\24"..., 47) = 47
_llseek(3, 129016, [129016], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1400) = 1400
_llseek(3, 94664, [94664], SEEK_SET) = 0
read(3, "A.\0\0\0aeabi\0\1$\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\23\1\24"..., 47) = 47
_llseek(3, 129016, [129016], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1400) = 1400
_llseek(3, 94664, [94664], SEEK_SET) = 0
read(3, "A.\0\0\0aeabi\0\1$\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\23\1\24"..., 47) = 47
_llseek(3, 129016, [129016], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1400) = 1400
_llseek(3, 94664, [94664], SEEK_SET) = 0
read(3, "A.\0\0\0aeabi\0\1$\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\23\1\24"..., 47) = 47
fstat64(3, {st_mode=S_IFREG|0755, st_size=130416, ...}) = 0
mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6f09000
mmap2(NULL, 168560, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6ebc000
mprotect(0xb6ed3000, 61440, PROT_NONE) = 0
mmap2(0xb6ee2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0xb6ee2000
mmap2(0xb6ee4000, 4720, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb6ee4000
close(3) = 0
openat(AT_FDCWD, "/usr/lib/arm-linux-gnueabihf/libseccomp.so.2", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0X\275\0\0004\0\0\0"..., 512) = 512
_llseek(3, 139832, [139832], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1000) = 1000
_llseek(3, 139500, [139500], SEEK_SET) = 0
read(3, "A,\0\0\0aeabi\0\1\"\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\24\1\25"..., 45) = 45
_llseek(3, 139832, [139832], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1000) = 1000
_llseek(3, 139500, [139500], SEEK_SET) = 0
read(3, "A,\0\0\0aeabi\0\1\"\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\24\1\25"..., 45) = 45
_llseek(3, 139832, [139832], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1000) = 1000
_llseek(3, 139500, [139500], SEEK_SET) = 0
read(3, "A,\0\0\0aeabi\0\1\"\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\24\1\25"..., 45) = 45
_llseek(3, 139832, [139832], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1000) = 1000
_llseek(3, 139500, [139500], SEEK_SET) = 0
read(3, "A,\0\0\0aeabi\0\1\"\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\24\1\25"..., 45) = 45
_llseek(3, 139832, [139832], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1000) = 1000
_llseek(3, 139500, [139500], SEEK_SET) = 0
read(3, "A,\0\0\0aeabi\0\1\"\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\24\1\25"..., 45) = 45
fstat64(3, {st_mode=S_IFREG|0644, st_size=140832, ...}) = 0
mmap2(NULL, 205040, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6e89000
mprotect(0xb6ea0000, 61440, PROT_NONE) = 0
mmap2(0xb6eaf000, 53248, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0xb6eaf000
close(3) = 0
openat(AT_FDCWD, "/lib/arm-linux-gnueabihf/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\300x\1\0004\0\0\0"..., 512) = 512
_llseek(3, 1293204, [1293204], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 2800) = 2800
_llseek(3, 1289788, [1289788], SEEK_SET) = 0
read(3, "A.\0\0\0aeabi\0\1$\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\23\1\24"..., 47) = 47
_llseek(3, 1293204, [1293204], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 2800) = 2800
_llseek(3, 1289788, [1289788], SEEK_SET) = 0
read(3, "A.\0\0\0aeabi\0\1$\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\23\1\24"..., 47) = 47
_llseek(3, 1293204, [1293204], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 2800) = 2800
_llseek(3, 1289788, [1289788], SEEK_SET) = 0
read(3, "A.\0\0\0aeabi\0\1$\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\23\1\24"..., 47) = 47
_llseek(3, 1293204, [1293204], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 2800) = 2800
_llseek(3, 1289788, [1289788], SEEK_SET) = 0
read(3, "A.\0\0\0aeabi\0\1$\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\23\1\24"..., 47) = 47
_llseek(3, 1293204, [1293204], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 2800) = 2800
_llseek(3, 1289788, [1289788], SEEK_SET) = 0
read(3, "A.\0\0\0aeabi\0\1$\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\23\1\24"..., 47) = 47
_llseek(3, 1293204, [1293204], SEEK_SET) = 0
read(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 2800) = 2800
_llseek(3, 1289788, [1289788], SEEK_SET) = 0
read(3, "A.\0\0\0aeabi\0\1$\0\0\0\0056\0\6\6\10\1\t\1\n\2\22\4\23\1\24"..., 47) = 47
fstat64(3, {st_mode=S_IFREG|0755, st_size=1296004, ...}) = 0
mmap2(NULL, 1364764, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6d3b000
mprotect(0xb6e73000, 65536, PROT_NONE) = 0
mmap2(0xb6e83000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x138000) = 0xb6e83000
mmap2(0xb6e86000, 8988, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb6e86000
close(3) = 0
set_tls(0xb6f0a2f0) = 0
mprotect(0xb6e83000, 8192, PROT_READ) = 0
mprotect(0xb6eaf000, 49152, PROT_READ) = 0
mprotect(0xb6ee2000, 4096, PROT_READ) = 0
mprotect(0x15f9000, 14655488, PROT_READ) = 0
mprotect(0xb6f16000, 4096, PROT_READ) = 0
munmap(0xb6f0b000, 41986) = 0
set_tid_address(0xb6f09e98) = 2293
set_robust_list(0xb6f09ea0, 12) = 0
rt_sigaction(SIGRTMIN, {sa_handler=0xb6ec08e8, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0xb6d68130}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0xb6ec09a4, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0xb6d68130}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
ugetrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
--- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR, si_addr=NULL} ---
+++ killed by SIGSEGV +++
[1] 2290 segmentation fault strace /usr/bin/containerd -debug
Configuration is default as distributed:
$ cat /etc/containerd/config.toml
# Copyright 2018-2020 Docker Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
disabled_plugins = ["cri"]
#root = "/var/lib/containerd"
#state = "/run/containerd"
#subreaper = true
#oom_score = 0
#[grpc]
# address = "/run/containerd/containerd.sock"
# uid = 0
# gid = 0
#[debug]
# address = "/run/containerd/debug.sock"
# uid = 0
# gid = 0
# level = "info"
Describe the results you expected:
containerd should startup normally.
Additional information you deem important (e.g. issue happens only occasionally):
Output of containerd --version:
# containerd --version
[1] 2311 segmentation fault containerd --version
# dpkg -l containerd.io
...
ii containerd.io 1.2.13-1 armhf An open and reliable container runtime
Also tried with containerd.io 1.2.10-2 as recommended in moby/moby#38175 - with the same result - a segfault. For the avoidance of doubt, the strace above is from containerd.io 1.2.13-1.
# dpkg -l containerd.io
Desired=Unknown/Install/Remove/Purge/Hold
| Status=Not/Inst/Conf-files/Unpacked/halF-conf/Half-inst/trig-aWait/Trig-pend
|/ Err?=(none)/Reinst-required (Status,Err: uppercase=bad)
||/ Name Version Architecture Description
+++-==============-============-============-======================================
ii containerd.io 1.2.10-2 armhf An open and reliable container runtime
Output of docker version:
# docker version
[1] 15954 illegal hardware instruction docker version
# dpkg -l docker-ce
...
iF docker-ce 5:19.03.8~3-0~debian-buster armhf Docker: the open-source application container engine
Output of docker info:
# docker info
[1] 15962 illegal hardware instruction docker info
FWIW this is a fresh install with no config changes applied yet.
Additional environment details (AWS, VirtualBox, physical, etc.):
$ free
total used free shared buff/cache available
Mem: 491536 38084 131868 6492 321584 435932
Swap: 0 0 0
$ cat /proc/cpuinfo
processor : 0
model name : ARMv6-compatible processor rev 7 (v6l)
BogoMIPS : 847.05
Features : half thumb fastmult vfp edsp java tls
CPU implementer : 0x41
CPU architecture: 7
CPU variant : 0x0
CPU part : 0xb76
CPU revision : 7
Hardware : BCM2835
Revision : 100000e
Serial : 000000002146e2eb
Model : Raspberry Pi Model B Rev 2
About this issue
- Original URL
- State: closed
- Created 4 years ago
- Comments: 18 (11 by maintainers)
Aargh!!!
@thaJeztah - you’re absolutely right. I changed the source to be
raspbianrather thandebianand updated the three packages and it works!Here’s the
apt-cache policyoutput, which is now clearly different to what I posted a couple of days ago. The oldest available version is 1.2.6, whereas I had also tried out 1.2.5 and 1.2.4.I must admit I didn’t notice the
Raspbian users cannot use this method!notice at https://docs.docker.com/engine/install/debian/. Having said that - the package method does appear to work, so perhaps that warning can be removed? I’m happy to raise the change request if there’s no other reason for that advice being there.Apologies for all this. Really appreciate the time and effort you’ve all put in.
Oh, just trying to read up now; @jinnko you’re running raspbian, but I just now now see the package you have installed looks to be the debian package;
Do you encounter the same problem if you install the raspbian packages? https://download.docker.com/linux/raspbian/dists/buster/pool/stable/armhf/