moby: runtime/cgo: pthread_create failed: Resource temporarily unavailable

see https://groups.google.com/forum/#!topic/docker-user/BG7lBIizmQw

I use a script to start containers, sleep 2s when started every 10 containers. But each time, when started about 420~430 containers, will encounter following errors.
any comments on this error?
Thanks.


runtime/cgo: pthread_create failed: Resource temporarily unavailable
SIGABRT: abort
PC=0x99d249
goroutine 0 [idle]:

goroutine 20 [syscall]:
runtime.notetsleepg(0x123f9a0, 0xffffffffffffffff)
        /usr/local/go/src/pkg/runtime/lock_futex.c:198 +0x46 fp=0x7ff9d1cebf40 sp=0x7ff9d1cebf10
runtime.signal_recv(0x0)
        /usr/local/go/src/pkg/runtime/sigqueue.goc:110 +0xea fp=0x7ff9d1cebf68 sp=0x7ff9d1cebf40
os/signal.loop()
        /usr/local/go/src/pkg/os/signal/signal_unix.go:21 +0x1e fp=0x7ff9d1cebfa8 sp=0x7ff9d1cebf68
runtime.goexit()
        /usr/local/go/src/pkg/runtime/proc.c:1445 fp=0x7ff9d1cebfb0 sp=0x7ff9d1cebfa8
created by os/signal.init·1
        /usr/local/go/src/pkg/os/signal/signal_unix.go:27 +0x32

goroutine 16 [runnable]:
fmt.(*buffer).WriteRune(0xc208004550, 0xc20000006e, 0x0, 0x0)
        /usr/local/go/src/pkg/fmt/print.go:93
fmt.(*ss).token(0xc208004540, 0xc200000001, 0xd8be60, 0x0, 0x0, 0x0)
        /usr/local/go/src/pkg/fmt/scan.go:479 +0xdd
fmt.(*ss).convertString(0xc208004540, 0x73, 0x0, 0x0)
        /usr/local/go/src/pkg/fmt/scan.go:831 +0x122
fmt.(*ss).scanOne(0xc208004540, 0x73, 0xa55f00, 0xc2080a1fb0)
        /usr/local/go/src/pkg/fmt/scan.go:988 +0x122a
fmt.(*ss).doScanf(0xc208004540, 0xc78610, 0x15, 0xc2080f7ad8, 0x7, 0x7, 0x5, 0x0, 0x0)
        /usr/local/go/src/pkg/fmt/scan.go:1160 +0x38e
fmt.Fscanf(0x7ff9d1e72a80, 0xc208104390, 0xc78610, 0x15, 0xc2080f7ad8, 0x7, 0x7, 0x434595, 0x0, 0x0)
        /usr/local/go/src/pkg/fmt/scan.go:145 +0xc4
fmt.Sscanf(0xc2081037a0, 0x10b, 0xc78610, 0x15, 0xc2080f7ad8, 0x7, 0x7, 0x7, 0x0, 0x0)
        /usr/local/go/src/pkg/fmt/scan.go:117 +0xce
github.com/docker/docker/pkg/mount.parseInfoFile(0x7ff9d1e723c8, 0xc20803a040, 0x0, 0x0, 0x0, 0x0, 0x0)
        /go/src/github.com/docker/docker/pkg/mount/mountinfo_linux.go:58 +0x4a6
github.com/docker/docker/pkg/mount.parseMountTable(0x0, 0x0, 0x0, 0x0, 0x0)
        /go/src/github.com/docker/docker/pkg/mount/mountinfo_linux.go:37 +0x135
github.com/docker/docker/pkg/mount.GetMounts(0x0, 0x0, 0x0, 0x0, 0x0)
        /go/src/github.com/docker/docker/pkg/mount/mount.go:8 +0x4b
github.com/docker/libcontainer/cgroups.FindCgroupMountpoint(0xc37960, 0x3, 0x0, 0x0, 0x0, 0x0)
        /go/src/github.com/docker/docker/vendor/src/github.com/docker/libcontainer/cgroups/utils.go:18 +0x59
github.com/docker/libcontainer/cgroups/fs.init·1()
        /go/src/github.com/docker/docker/vendor/src/github.com/docker/libcontainer/cgroups/fs/apply_raw.go:33 +0x34
github.com/docker/libcontainer/cgroups/fs.init()
        /go/src/github.com/docker/docker/vendor/src/github.com/docker/libcontainer/cgroups/fs/utils.go:62 +0x533
github.com/docker/libcontainer/cgroups/systemd.init()
        /go/src/github.com/docker/docker/vendor/src/github.com/docker/libcontainer/cgroups/systemd/apply_systemd.go:358 +0x59
github.com/docker/libcontainer/namespaces.init()
        /go/src/github.com/docker/docker/vendor/src/github.com/docker/libcontainer/namespaces/types_linux.go:16 +0x9e
github.com/docker/docker/daemon/execdriver/native.init()
        /go/src/github.com/docker/docker/daemon/execdriver/native/utils.go:35 +0x69
main.init()
        /go/src/github.com/docker/docker/docker/flags.go:101 +0x9e

goroutine 19 [finalizer wait]:
runtime.park(0x41df50, 0x123cdd8, 0x1222a49)
        /usr/local/go/src/pkg/runtime/proc.c:1369 +0x89
runtime.parkunlock(0x123cdd8, 0x1222a49)
        /usr/local/go/src/pkg/runtime/proc.c:1385 +0x3b
runfinq()
        /usr/local/go/src/pkg/runtime/mgc0.c:2644 +0xcf
runtime.goexit()
        /usr/local/go/src/pkg/runtime/proc.c:1445

goroutine 17 [syscall]:
runtime.goexit()
        /usr/local/go/src/pkg/runtime/proc.c:1445

rax     0x0
rbx     0xb
rcx     0xffffffffffffffff
rdx     0x6
rdi     0x1758
rsi     0x3b0c
rbp     0x7ff9d0bcec40
rsp     0x7ff9d0bcea78
r8      0x7ff9d0bcf700
r9      0x616e7520796c6972
r10     0x8
r11     0x202
r12     0x7ff9c80008c0
r13     0x0
r14     0x7ff9d0bcf9c0
r15     0x7ff9d0bcf700
rip     0x99d249
rflags  0x202
cs      0x33
fs      0x0
gs      0x0
runtime/cgo: pthread_create failed: Resource temporarily unavailable
SIGABRT: abort
PC=0x99d249

goroutine 0 [idle]:

goroutine 16 [runnable]:
fmt.(*ss).scanOne(0xc208004000, 0x64, 0xa4dd40, 0xc208091a98)
        /usr/local/go/src/pkg/fmt/scan.go:933 +0xa4
fmt.(*ss).doScanf(0xc208004000, 0xc78610, 0x15, 0xc2080efad8, 0x7, 0x7, 0x3, 0x0, 0x0)
        /usr/local/go/src/pkg/fmt/scan.go:1160 +0x38e
fmt.Fscanf(0x7f646c699ba0, 0xc2080754f0, 0xc78610, 0x15, 0xc2080efad8, 0x7, 0x7, 0x434595, 0x0, 0x0)
        /usr/local/go/src/pkg/fmt/scan.go:145 +0xc4
fmt.Sscanf(0xc2080b78c0, 0x10b, 0xc78610, 0x15, 0xc2080efad8, 0x7, 0x7, 0x7, 0x0, 0x0)
        /usr/local/go/src/pkg/fmt/scan.go:117 +0xce
github.com/docker/docker/pkg/mount.parseInfoFile(0x7f646c6993c8, 0xc20803a040, 0x0, 0x0, 0x0, 0x0, 0x0)
        /go/src/github.com/docker/docker/pkg/mount/mountinfo_linux.go:58 +0x4a6
github.com/docker/docker/pkg/mount.parseMountTable(0x0, 0x0, 0x0, 0x0, 0x0)
        /go/src/github.com/docker/docker/pkg/mount/mountinfo_linux.go:37 +0x135
github.com/docker/docker/pkg/mount.GetMounts(0x0, 0x0, 0x0, 0x0, 0x0)
        /go/src/github.com/docker/docker/pkg/mount/mount.go:8 +0x4b
github.com/docker/libcontainer/cgroups.FindCgroupMountpoint(0xc37960, 0x3, 0x0, 0x0, 0x0, 0x0)
        /go/src/github.com/docker/docker/vendor/src/github.com/docker/libcontainer/cgroups/utils.go:18 +0x59
github.com/docker/libcontainer/cgroups/fs.init·1()
        /go/src/github.com/docker/docker/vendor/src/github.com/docker/libcontainer/cgroups/fs/apply_raw.go:33 +0x34
github.com/docker/libcontainer/cgroups/fs.init()
        /go/src/github.com/docker/docker/vendor/src/github.com/docker/libcontainer/cgroups/fs/utils.go:62 +0x533
github.com/docker/libcontainer/cgroups/systemd.init()
        /go/src/github.com/docker/docker/vendor/src/github.com/docker/libcontainer/cgroups/systemd/apply_systemd.go:358 +0x59
github.com/docker/libcontainer/namespaces.init()
        /go/src/github.com/docker/docker/vendor/src/github.com/docker/libcontainer/namespaces/types_linux.go:16 +0x9e
github.com/docker/docker/daemon/execdriver/native.init()
        /go/src/github.com/docker/docker/daemon/execdriver/native/utils.go:35 +0x69
main.init()
        /go/src/github.com/docker/docker/docker/flags.go:101 +0x9e

goroutine 17 [runnable]:
runtime.MHeap_Scavenger()
        /usr/local/go/src/pkg/runtime/mheap.c:507
runtime.goexit()
        /usr/local/go/src/pkg/runtime/proc.c:1445

goroutine 18 [runnable]:
bgsweep()
        /usr/local/go/src/pkg/runtime/mgc0.c:1976
runtime.goexit()
        /usr/local/go/src/pkg/runtime/proc.c:1445

goroutine 19 [runnable]:
runfinq()
        /usr/local/go/src/pkg/runtime/mgc0.c:2606
runtime.goexit()
        /usr/local/go/src/pkg/runtime/proc.c:1445

goroutine 20 [runnable]:
os/signal.loop()
        /usr/local/go/src/pkg/os/signal/signal_unix.go:19
created by os/signal.init·1
        /usr/local/go/src/pkg/os/signal/signal_unix.go:27 +0x32

goroutine 17 [syscall]:
runtime.goexit()
        /usr/local/go/src/pkg/runtime/proc.c:1445

rax     0x0
rbx     0xb
rcx     0xffffffffffffffff
rdx     0x6
rdi     0x1671
rsi     0x1671
rbp     0x7fff38800f90
rsp     0x7fff38800dc8
r8      0x14f8880
r9      0x616e7520796c6972
r10     0x8
r11     0x206
r12     0x14fae10
r13     0x7
r14     0x10b
r15     0xc2080b78c0
rip     0x99d249
rflags  0x206
cs      0x33
fs      0x0
gs      0x0
2014/12/15 17:58:46 Error response from daemon: Error activating devmapper device for '08e80b210baa930a2416e1370808574fd3d01c76fc45d1b7ae4e5204c99f848b-init': Error running DeviceCreate (activateDevice) dm_task_run failed
2014/12/15 17:58:48 Post http:///var/run/docker.sock/v1.15/containers/create: EOF
2014/12/15 17:58:48 Post http:///var/run/docker.sock/v1.15/containers/create: EOF
2014/12/15 17:58:48 Post http:///var/run/docker.sock/v1.15/containers/create: EOF
2014/12/15 17:58:48 Post http:///var/run/docker.sock/v1.15/containers/create: EOF
2014/12/15 17:58:48 Post http:///var/run/docker.sock/v1.15/containers/create: EOF
2014/12/15 17:58:48 Post http:///var/run/docker.sock/v1.15/containers/create: EOF
2014/12/15 17:58:48 Post http:///var/run/docker.sock/v1.15/containers/create: EOF

Click here to Reply



Ling Lan    
12/16/14


system info:

^C-bash-4.2# docker info
Containers: 390
Images: 13
Storage Driver: devicemapper
 Pool Name: docker-253:0-18414011-pool
 Pool Blocksize: 65.54 kB
 Data file: /var/lib/docker/devicemapper/devicemapper/data
 Metadata file: /var/lib/docker/devicemapper/devicemapper/metadata
 Data Space Used: 3.904 GB
 Data Space Total: 107.4 GB
 Metadata Space Used: 35.62 MB
 Metadata Space Total: 2.147 GB
 Library Version: 1.02.82-git (2013-10-04)
Execution Driver: native-0.2
Kernel Version: 3.10.0-123.9.3.el7.x86_64
Operating System: Red Hat Enterprise Linux Server 7.0 (Maipo)

About this issue

  • Original URL
  • State: closed
  • Created 9 years ago
  • Comments: 16 (7 by maintainers)

Most upvoted comments

I resolved a similar issue.

PID full on my machine. It can check lsof | wc -l Fixed by applying kernel.pid_max = 65535 to /etc/sysctl.conf

+9
uzyexe on Sep 13, 2016

Is this the same issue? https://github.com/docker/docker/issues/19124

I am going to test out the proposed solution: add TasksMax=infinity to your docker.service file.

@cdauth Thanks for that suggestion, unfortunately it was a fresh install. I tried linux-lts but couldn’t get my nvidia-lts driver to work properly.

+1
micahthomas on Jan 15, 2016
Read more comments on GitHub
← moby: Running systemctl in container fails with "Failed to get D-Bus connection"
moby: Save of Windows image fails on base OS image layer. →