moby: docker build exits with i/o error

Using docker 8.1 on archlinux, I’m seeing the following:

$ docker build .
Uploading context 28.67 kB
Uploading context 
Step 0 : FROM ubuntu:13.10
 ---> 9f676bd305a4
Step 1 : RUN apt-get update && apt-get -y dist-upgrade
 ---> Using cache
 ---> e283f122891a
Step 2 : RUN apt-get update
2014/03/07 12:03:50 build: mkdir /var/lib/docker/devicemapper/mnt/528767f5722a7591aaa1ac68da0674c8f2b82d6f4ab1697a0792d9ce19f13989-init/rootfs: input/output error

After the abort, the filesystem is left mounted:

$ mount
/dev/mapper/docker-8:2-2525695-528767f5722a7591aaa1ac68da0674c8f2b82d6f4ab1697a0792d9ce19f13989-init on /var/lib/docker/devicemapper/mnt/528767f5722a7591aaa1ac68da0674c8f2b82d6f4ab1697a0792d9ce19f13989-init type ext4 (rw,relatime,discard,stripe=16,data=ordered)

$ df -h
Filesystem                                                                                            Size  Used Avail Use% Mounted on
/dev/sda2                                                                                              50G   30G   18G  64% /
dev                                                                                                   1.5G     0  1.5G   0% /dev
run                                                                                                   1.5G  344K  1.5G   1% /run
tmpfs                                                                                                 1.5G     0  1.5G   0% /dev/shm
tmpfs                                                                                                 1.5G     0  1.5G   0% /sys/fs/cgroup
tmpfs                                                                                                 1.5G  8.0K  1.5G   1% /tmp
none                                                                                                  298G  237G   62G  80% /media/sf_Public
none                                                                                                  298G  237G   62G  80% /media/sf_Public
/dev/mapper/docker-8:2-2525695-528767f5722a7591aaa1ac68da0674c8f2b82d6f4ab1697a0792d9ce19f13989-init  9.8G  220M  9.0G   3% /var/lib/docker/devicemapper/mnt/528767f5722a7591aaa1ac68da0674c8f2b82d6f4ab1697a0792d9ce19f13989-init

Trying to touch a file within the mounted filesystem results in an ENOSPC:

$ cd /var/lib/docker/devicemapper/mnt/528767f5722a7591aaa1ac68da0674c8f2b82d6f4ab1697a0792d9ce19f13989-init
$ strace touch foo
execve("/usr/bin/touch", ["touch", "foo"], [/* 14 vars */]) = 0
brk(0)                                  = 0xbb0000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=137288, ...}) = 0
mmap(NULL, 137288, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3d44fc3000
close(3)                                = 0
open("/usr/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\34\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2022349, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3d44fc2000
mmap(NULL, 3832336, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3d44a1d000
mprotect(0x7f3d44bbb000, 2097152, PROT_NONE) = 0
mmap(0x7f3d44dbb000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19e000) = 0x7f3d44dbb000
mmap(0x7f3d44dc1000, 14864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3d44dc1000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3d44fc1000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3d44fc0000
arch_prctl(ARCH_SET_FS, 0x7f3d44fc1700) = 0
mprotect(0x7f3d44dbb000, 16384, PROT_READ) = 0
mprotect(0x60d000, 4096, PROT_READ)     = 0
mprotect(0x7f3d44fe5000, 4096, PROT_READ) = 0
munmap(0x7f3d44fc3000, 137288)          = 0
brk(0)                                  = 0xbb0000
brk(0xbd1000)                           = 0xbd1000
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=1607712, ...}) = 0
mmap(NULL, 1607712, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3d44e37000
close(3)                                = 0
open("foo", O_WRONLY|O_CREAT|O_NOCTTY|O_NONBLOCK, 0666) = -1 ENOSPC (No space left on device)
utimensat(AT_FDCWD, "foo", NULL, 0)     = -1 ENOENT (No such file or directory)
open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2492, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3d44fe4000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2492
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x7f3d44fe4000, 4096)            = 0
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib/charset.alias", O_RDONLY|O_NOFOLLOW) = -1 ENOENT (No such file or directory)
write(2, "touch: ", 7touch: )                  = 7
write(2, "cannot touch \342\200\230foo\342\200\231", 22cannot touch ‘foo’) = 22
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
write(2, ": No space left on device", 25: No space left on device) = 25
write(2, "\n", 1
)                       = 1
close(1)                                = 0
close(2)                                = 0
exit_group(1)                           = ?
+++ exited with 1 +++

Since df reports plenty of space being available and df -i reports inodes not being exhausted, I’m at a bit of a loss as to what space is exhausted:

$ df -i
Filesystem                                                                                            Inodes  IUsed   IFree IUse% Mounted on
/dev/sda2                                                                                            3301376 569942 2731434   18% /
dev                                                                                                   387938    368  387570    1% /dev
run                                                                                                   388614    391  388223    1% /run
tmpfs                                                                                                 388614      1  388613    1% /dev/shm
tmpfs                                                                                                 388614     11  388603    1% /sys/fs/cgroup
tmpfs                                                                                                 388614     13  388601    1% /tmp
none                                                                                                    1000      0    1000    0% /media/sf_Public
none                                                                                                    1000      0    1000    0% /media/sf_Public
/dev/mapper/docker-8:2-2525695-528767f5722a7591aaa1ac68da0674c8f2b82d6f4ab1697a0792d9ce19f13989-init  655360  16951  638409    3% /var/lib/docker/devicemapper/mnt/528767f5722a7591aaa1ac68da0674c8f2b82d6f4ab1697a0792d9ce19f13989-init