moby: Can't access previously created data-only volumes after the 1.10 upgrade

After the 1.10 upgrade, none of my previously created data-only volume containers are accessible. They still exist when running docker ps -a, and no errors are shown in the docker logs.

For example:

docker run -it --rm --volumes-from samba_config ubuntu /bin/bash
root@eaf6e72b4c6e:/# ls /config
ls: cannot access /config: No such file or directory

I would have expected to see the /config directory present (see the samba_config configuration below).

The systemd journal seems to show that the docker0 interface gets disabled, then re-enabled:

Feb 06 10:37:42 homeserver kernel: device vethfecaad7 entered promiscuous mode
Feb 06 10:37:42 homeserver kernel: IPv6: ADDRCONF(NETDEV_UP): vethfecaad7: link is not ready
Feb 06 10:37:42 homeserver kernel: docker0: port 2(vethfecaad7) entered forwarding state
Feb 06 10:37:42 homeserver kernel: docker0: port 2(vethfecaad7) entered forwarding state
Feb 06 10:37:42 homeserver systemd-timesyncd[435]: Network configuration changed, trying to establish connection.
Feb 06 10:37:42 homeserver kernel: docker0: port 2(vethfecaad7) entered disabled state
Feb 06 10:37:42 homeserver systemd-timesyncd[435]: Synchronized to time server 4.53.160.75:123 (2.arch.pool.ntp.org).
Feb 06 10:37:42 homeserver systemd-udevd[17749]: Could not generate persistent MAC address for vethfecaad7: No such file or directory
Feb 06 10:37:42 homeserver systemd-timesyncd[435]: Network configuration changed, trying to establish connection.
Feb 06 10:37:42 homeserver systemd-timesyncd[435]: Synchronized to time server 4.53.160.75:123 (2.arch.pool.ntp.org).
Feb 06 10:37:43 homeserver kernel: eth0: renamed from veth97cdbb1
Feb 06 10:37:43 homeserver systemd-timesyncd[435]: Network configuration changed, trying to establish connection.
Feb 06 10:37:43 homeserver systemd-networkd[442]: vethfecaad7: Gained carrier
Feb 06 10:37:43 homeserver kernel: IPv6: ADDRCONF(NETDEV_CHANGE): vethfecaad7: link becomes ready
Feb 06 10:37:43 homeserver kernel: docker0: port 2(vethfecaad7) entered forwarding state
Feb 06 10:37:43 homeserver kernel: docker0: port 2(vethfecaad7) entered forwarding state
Feb 06 10:37:43 homeserver systemd-timesyncd[435]: Synchronized to time server 4.53.160.75:123 (2.arch.pool.ntp.org).
Feb 06 10:37:44 homeserver systemd-networkd[442]: vethfecaad7: Gained IPv6LL
Feb 06 10:37:44 homeserver systemd-timesyncd[435]: Network configuration changed, trying to establish connection.
Feb 06 10:37:44 homeserver systemd-timesyncd[435]: Synchronized to time server 4.53.160.75:123 (2.arch.pool.ntp.org).

The data-only container samba_config, previously accessible from the --volumes-from directive when running any container:

docker inspect samba_config
[
    {
        "Id": "55781cb6b0c3ab6d3c69377d9368cb8c70bcc3472b512e150fb3fd931b76169c",
        "Created": "2015-06-17T21:12:43.904990426Z",
        "Path": "true",
        "Args": [],
        "State": {
            "Status": "created",
            "Running": false,
            "Paused": false,
            "Restarting": false,
            "OOMKilled": false,
            "Dead": false,
            "Pid": 0,
            "ExitCode": -1,
            "Error": "[8] System error: exec: \"true\": executable file not found in $PATH",
            "StartedAt": "0001-01-01T00:00:00Z",
            "FinishedAt": "0001-01-01T00:00:00Z"
        },
        "Image": "sha256:2935444c7f23efd91dd0c2b6f1bd642552b03b47a5ebdb154212f7141c083f40",
        "ResolvConfPath": "/var/lib/docker/containers/55781cb6b0c3ab6d3c69377d9368cb8c70bcc3472b512e150fb3fd931b76169c/resolv.conf",
        "HostnamePath": "/var/lib/docker/containers/55781cb6b0c3ab6d3c69377d9368cb8c70bcc3472b512e150fb3fd931b76169c/hostname",
        "HostsPath": "/var/lib/docker/containers/55781cb6b0c3ab6d3c69377d9368cb8c70bcc3472b512e150fb3fd931b76169c/hosts",
        "LogPath": "/var/lib/docker/containers/55781cb6b0c3ab6d3c69377d9368cb8c70bcc3472b512e150fb3fd931b76169c/55781cb6b0c3ab6d3c69377d
9368cb8c70bcc3472b512e150fb3fd931b76169c-json.log",
        "Name": "/samba_config",
        "RestartCount": 0,
        "Driver": "btrfs",
        "MountLabel": "",
        "ProcessLabel": "",
        "AppArmorProfile": "",
        "ExecIDs": null,
        "HostConfig": {
            "Binds": null,
            "ContainerIDFile": "",
            "LogConfig": {
                "Type": "json-file",                                            
                "Config": {}
            },
            "NetworkMode": "bridge",
            "PortBindings": {},
            "RestartPolicy": {
                "Name": "no",
                "MaximumRetryCount": 0
            },
            "VolumeDriver": "",
            "VolumesFrom": null,
            "CapAdd": null,
            "CapDrop": null,
            "Dns": [],
            "DnsOptions": [],
            "DnsSearch": [],
            "ExtraHosts": null,
            "GroupAdd": null,
            "IpcMode": "",
            "Links": null,
            "OomScoreAdj": 0,
            "PidMode": "",
            "Privileged": false,
            "PublishAllPorts": false,
            "ReadonlyRootfs": false,
            "SecurityOpt": null,
            "UTSMode": "",
            "ShmSize": 0,
            "ConsoleSize": [
                0,
                0
            ],
            "Isolation": "",
            "CpuShares": 0,
            "CgroupParent": "",
            "BlkioWeight": 0,
            "BlkioWeightDevice": null,
            "BlkioDeviceReadBps": null,
            "BlkioDeviceWriteBps": null,
            "BlkioDeviceReadIOps": null,
            "BlkioDeviceWriteIOps": null,                                                                
            "CpuPeriod": 0,
            "CpuQuota": 0,
            "CpusetCpus": "",
            "CpusetMems": "",
            "Devices": [],
            "KernelMemory": 0,
            "Memory": 0,
            "MemoryReservation": 0,
            "MemorySwap": 0,
            "MemorySwappiness": null,
            "OomKillDisable": null,
            "PidsLimit": 0,
            "Ulimits": null
        },
        "GraphDriver": {
            "Name": "btrfs",
            "Data": null
        },
        "Mounts": [],
        "Config": {
            "Hostname": "55781cb6b0c3",
            "Domainname": "",
            "User": "",
            "AttachStdin": false,
            "AttachStdout": true,
            "AttachStderr": true,
            "Tty": false,
            "OpenStdin": false,
            "StdinOnce": false,
            "Env": [],
            "Cmd": [
                "true"
            ],
            "Image": "myscratch",
            "Volumes": {
                "/config": {}
            },
            "WorkingDir": "",
            "Entrypoint": null,
            "OnBuild": null,                                                             
            "Labels": {}
        },
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": null,
            "SandboxKey": "",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": null
        }
    }
]

Running docker in debug mode shows nothing abnormal:

DEBU[0987] Calling GET /v1.22/containers/b347cffdd2787a2eedc30383ac8e0e4ebfe84303f4c033272887d4c3fad2dd32/json 
DEBU[0987] GET /v1.22/containers/b347cffdd2787a2eedc30383ac8e0e4ebfe84303f4c033272887d4c3fad2dd32/json 
DEBU[0987] Calling DELETE /v1.22/containers/b347cffdd2787a2eedc30383ac8e0e4ebfe84303f4c033272887d4c3fad2dd32 
DEBU[0987] DELETE /v1.22/containers/b347cffdd2787a2eedc30383ac8e0e4ebfe84303f4c033272887d4c3fad2dd32?v=1 
DEBU[0995] Calling POST /v1.22/containers/create        
DEBU[0995] POST /v1.22/containers/create                
DEBU[0995] form data: {"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":null,"Domainname":"","Entrypoint":null,"Env":
[],"HostConfig":{"Binds":null,"BlkioDeviceReadBps":null,"BlkioDeviceReadIOps":null,"BlkioDeviceWriteBps":null,"BlkioDeviceWriteIOps":nul
l,"BlkioWeight":0,"BlkioWeightDevice":null,"CapAdd":null,"CapDrop":null,"CgroupParent":"","ConsoleSize":[0,0],"ContainerIDFile":"","CpuP
eriod":0,"CpuQuota":0,"CpuShares":0,"CpusetCpus":"","CpusetMems":"","Devices":[],"Dns":[],"DnsOptions":[],"DnsSearch":[],"ExtraHosts":nu
ll,"GroupAdd":null,"IpcMode":"","Isolation":"","KernelMemory":0,"Links":null,"LogConfig":{"Config":{},"Type":""},"Memory":0,"MemoryReser
vation":0,"MemorySwap":0,"MemorySwappiness":-1,"NetworkMode":"default","OomKillDisable":false,"OomScoreAdj":0,"PidMode":"","PidsLimit":0
,"PortBindings":{},"Privileged":false,"PublishAllPorts":false,"ReadonlyRootfs":false,"RestartPolicy":{"MaximumRetryCount":0,"Name":"no"}
,"SecurityOpt":null,"ShmSize":0,"UTSMode":"","Ulimits":null,"VolumeDriver":"","VolumesFrom":["samba_config"]},"Hostname":"","Image":"ubu
ntu","Labels":{},"NetworkingConfig":{"EndpointsConfig":{}},"OnBuild":null,"OpenStdin":false,"StdinOnce":false,"StopSignal":"SIGTERM","Tt
y":false,"User":"","Volumes":{},"WorkingDir":""} 
DEBU[0996] container mounted via layerStore: /var/lib/docker/btrfs/subvolumes/7592834c248ed4c5fc37825d83aa8380755d1faa81af6361f0d760fbc4
ffdab2 
DEBU[0996] Calling POST /v1.22/containers/d8ee91c2bbc4f53e042871abd94312176e08ddc9fcfd86100da426ae84cfa851/start 
DEBU[0996] POST /v1.22/containers/d8ee91c2bbc4f53e042871abd94312176e08ddc9fcfd86100da426ae84cfa851/start 
DEBU[0996] container mounted via layerStore: /var/lib/docker/btrfs/subvolumes/7592834c248ed4c5fc37825d83aa8380755d1faa81af6361f0d760fbc4
ffdab2 
DEBU[0996] Assigning addresses for endpoint stoic_allen's interface on network bridge 
DEBU[0996] RequestAddress(LocalDefault/172.17.0.0/16, <nil>, map[]) 
DEBU[0997] Assigning addresses for endpoint stoic_allen's interface on network bridge 
DEBU[1000] Calling GET /v1.22/containers/json           
DEBU[1000] GET /v1.22/containers/json                   
DEBU[1000] Releasing addresses for endpoint stoic_allen's interface on network bridge 
DEBU[1000] ReleaseAddress(LocalDefault/172.17.0.0/16, 172.17.0.2)

I am running Docker on Archlinux with systemd-networkd controlling the network setup:

~ $ cat /etc/systemd/network/bridge.netdev 
[NetDev]
Name=br0
Kind=bridge
~ $ cat /etc/systemd/network/bridge.network 
[Match]
Name=br0

[Network]
Address=192.168.0.101/24
Gateway=192.168.0.1
DNS=8.8.8.8
DNS=8.8.4.4

~ $ cat /etc/systemd/network/eno1.network 
[Match]
Name=en*

[Network]
Bridge=br0

~ $ docker info
Containers: 11
 Running: 0
 Paused: 0
 Stopped: 11
Images: 193
Server Version: 1.10.0
Storage Driver: btrfs
 Build Version: Btrfs v4.3.1
 Library Version: 101
Execution Driver: native-0.2
Logging Driver: json-file
Plugins: 
 Volume: local
 Network: bridge null host
Kernel Version: 4.4.1-2-ARCH
Operating System: Arch Linux
OSType: linux
Architecture: x86_64
CPUs: 4
Total Memory: 7.602 GiB
Name: homeserver
ID: FV23:MQ6K:JGLR:LYAC:GIQX:QTI6:A3IR:37L5:FR6Y:5MXE:YAV6:4RAH
Debug mode (server): true
 File Descriptors: 11
 Goroutines: 27
 System Time: 2016-02-06T11:57:25.515835606-08:00
 EventsListeners: 0
 Init SHA1: 79ca7bc83376a859533464da88086c44b22513bc
 Init Path: /usr/lib/docker/dockerinit
 Docker Root Dir: /var/lib/docker
~ $ docker version
Client:
 Version:      1.10.0
 API version:  1.22
 Go version:   go1.5.3
 Git commit:   590d5108
 Built:        Sat Feb  6 18:04:28 2016
 OS/Arch:      linux/amd64

Server:
 Version:      1.10.0
 API version:  1.22
 Go version:   go1.5.3
 Git commit:   590d5108
 Built:        Sat Feb  6 18:04:28 2016
 OS/Arch:      linux/amd64

I’m really stumped here! My entire Docker container stack is unusable right now…

Thanks, Scott

About this issue

Original URL
State: closed
Created 8 years ago
Comments: 24 (10 by maintainers)

Most upvoted comments

In the interest of moving forward in my setup (Ubuntu 14.04), I used the python script below to copy data from my non-working data-only volumes into newly created ones. This appears to allow it to work.

While it worked for me, it may not for you and there is most likely a better way… But perhaps someone else will find this useful.

# save the python script below as copy.py
docker rename <non-working-data-container-name> <saved-container-name>
python copy.py <saved-container-name> <non-working-data-container-name>
# check everything to make sure things work for you and then optionally cleanup the old
# data with the following commands...
docker rm <saved-container-name>
docker volume rm $(docker volume ls -q --filter=dangling=true)

#!/usr/bin/python
import sys
import os
import json
import subprocess

VAR_DOCKER = "/var/lib/docker"

def call(command):
    process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
    process.wait()
    if process.returncode != 0:
        raise Exception("Failed call to '" + command + "'")
    return process.stdout.read()

def main():
    if len(sys.argv) < 3:
        sys.stdout.write("Usage: %s <old_container_name> <new_container_name>\n" % (sys.argv[0]))
        return 1

    source_container_name = sys.argv[1]
    new_container_name = sys.argv[2]

    container_id = call("docker inspect --format='{{.Id}}' " + source_container_name)

    old_config_path = os.path.join(VAR_DOCKER, "containers", container_id.strip(), "config.json")
    if not os.path.exists(old_config_path):
        sys.stderr.write("Could not find " + old_config_path + ". Perhaps this is not an old container?\n")
        return 1

    old_config = json.load(open(old_config_path))

    volumes = old_config["Volumes"]

    create_command = "docker create"
    for k in volumes.keys():
        create_command += " -v "
        create_command += k
    create_command += " --name " + new_container_name
    create_command += " "
    create_command += old_config["Config"]["Image"]
    create_command += " "
    create_command += " ".join(old_config["Config"]["Cmd"])
    call(create_command)

    new_config = json.loads(call("docker inspect " + new_container_name))
    for mount in new_config[0]["Mounts"]:
        source_dir = volumes[mount["Destination"]]
        destination_dir = mount["Source"]
        rsync_command = "rsync -avc " + source_dir + "/ " + destination_dir
        subprocess.call(rsync_command, shell=True)


if __name__ == '__main__':
    sys.exit(main())

sruffell on Feb 9, 2016

@louissalin when running in production it’s always recommended to “pin” to a specific version, e.g. if you’re using apt, to sudo apt-get install docker-engine=1.12.0-0~xenial (or, e.g. dpkg --set-selections). Upgrading versions should be a deliberate action, i.e., first test, then roll out updates

thaJeztah on Jan 19, 2017