containerd: Windows rootfs.size-gb label is not expanding volume to specified size
Description
We are using containerd 1.6.1 as a CRI for our windows kubernetes nodes (using RKE2).
Our workload needs a rather large C:\ drive and thus in the past when we used docker we used the storage-opts to increase the size to 100GB as described here.
With containerd the only equivalent that I found was this:
and the possibility to label a namespace through https://github.com/containerd/containerd/issues/2285.
After labeling the namespace the container still only has a 20GB root disk.
> ctr ns ls
NAME LABELS
k8s.io
> ctr ns label k8s.io containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb=100
> ctr ns ls
NAME LABELS
k8s.io containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb=100
Steps to reproduce the issue
- Label k8s containerd namespace:
> ctr ns ls
NAME LABELS
k8s.io
> ctr ns label k8s.io containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb=100
> ctr ns ls
NAME LABELS
k8s.io containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb=100
- Start k8s pod
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
name: debug-test
spec:
containers:
- name: debug-test
image: mcr.microsoft.com/windows/servercore:1809
command: ["powershell.exe"]
args:
- -c
- Start-Sleep
- -s
- "10000"
EOF
4.Check size of C: drive:
kubectl exec -it pod/debug-test -- powershell.exe -c Get-PSDrive C
Name Used (GB) Free (GB) Provider Root CurrentLocation
---- --------- --------- -------- ---- ---------------
Alias Alias
C 0.12 19.75 FileSystem C:\
Cert Certificate \
Env Environment
Function Function
HKCU Registry HKEY_CURRENT_USER
HKLM Registry HKEY_LOCAL_MACHINE
Variable Variable
WSMan WSMan
Describe the results you received and expected
I would expect containerd to expand the default 20G disk layer to 100G as specified through the label
What version of containerd are you using?
containerd github.com/containerd/containerd v1.6.1 10f428dac7cec44c864e1b830a4623af27a9fc70
Any other relevant information
> C:\var\lib\rancher\rke2\bin\crictl.exe --runtime-endpoint npipe:////./pipe/containerd-containerd-1.6.1 info
{
"status": {
"conditions": [
{
"type": "RuntimeReady",
"status": true,
"reason": "",
"message": ""
},
{
"type": "NetworkReady",
"status": true,
"reason": "",
"message": ""
}
]
},
"cniconfig": {
"PluginDirs": [
"c:\\var\\lib\\rancher\\rke2\\bin"
],
"PluginConfDir": "c:\\var\\lib\\rancher\\rke2\\agent\\etc\\cni",
"PluginMaxConfNum": 1,
"Prefix": "eth",
"Networks": [
{
"Config": {
"Name": "Calico",
"CNIVersion": "0.3.1",
"Plugins": [
{
"Network": {
"cniVersion": "0.3.1",
"name": "Calico",
"type": "calico",
"capabilities": {
"dns": true
},
"ipam": {
"type": "calico-ipam"
},
"dns": {
"nameservers": [
"10.43.0.10"
],
"search": [
"svc.cluster.local"
]
}
},
"Source": "{\"DNS\":{\"Nameservers\":[\"10.43.0.10\"],\"Search\":[\"svc.cluster.local\"]},\"capabilities\":{\"dns\":true},\"cniVersion\":\"0.3.1\",\"datastore_type\":\"kubernetes\",\"etcd_ca_cert_file\":\"\",\"etcd_cert_file\":\"\",\"etcd_endpoints\":\"\",\"etcd_key_file\":\"\",\"ipam\":{\"subnet\":\"usePodCidr\",\"type\":\"calico-ipam\"},\"kubernetes\":{\"kubeconfig\":\"c:\\\\var\\\\lib\\\\rancher\\\\rke2\\\\agent\\\\calico.kubeconfig\"},\"log_level\":\"info\",\"mode\":\"vxlan\",\"name\":\"Calico\",\"nodename_file\":\"c:\\\\var\\\\lib\\\\rancher\\\\rke2\\\\agent\\\\calico_node_name\",\"policies\":[{\"Name\":\"EndpointPolicy\",\"Value\":{\"ExceptionList\":[\"10.43.0.0/16\"],\"Type\":\"OutBoundNAT\"}},{\"Name\":\"EndpointPolicy\",\"Value\":{\"DestinationPrefix\":\"10.43.0.0/16\",\"NeedEncap\":true,\"Type\":\"SDNROUTE\"}}],\"policy\":{\"type\":\"k8s\"},\"type\":\"calico\",\"vxlan_mac_prefix\":\"0E-2A\",\"vxlan_vni\":4096,\"windows_use_single_network\":true}"
}
],
"Source": "{\"cniVersion\":\"0.3.1\",\"name\":\"Calico\",\"plugins\":[{\"DNS\":{\"Nameservers\":[\"10.43.0.10\"],\"Search\":[\"svc.cluster.local\"]},\"capabilities\":{\"dns\":true},\"cniVersion\":\"0.3.1\",\"datastore_type\":\"kubernetes\",\"etcd_ca_cert_file\":\"\",\"etcd_cert_file\":\"\",\"etcd_endpoints\":\"\",\"etcd_key_file\":\"\",\"ipam\":{\"subnet\":\"usePodCidr\",\"type\":\"calico-ipam\"},\"kubernetes\":{\"kubeconfig\":\"c:\\\\var\\\\lib\\\\rancher\\\\rke2\\\\agent\\\\calico.kubeconfig\"},\"log_level\":\"info\",\"mode\":\"vxlan\",\"name\":\"Calico\",\"nodename_file\":\"c:\\\\var\\\\lib\\\\rancher\\\\rke2\\\\agent\\\\calico_node_name\",\"policies\":[{\"Name\":\"EndpointPolicy\",\"Value\":{\"ExceptionList\":[\"10.43.0.0/16\"],\"Type\":\"OutBoundNAT\"}},{\"Name\":\"EndpointPolicy\",\"Value\":{\"DestinationPrefix\":\"10.43.0.0/16\",\"NeedEncap\":true,\"Type\":\"SDNROUTE\"}}],\"policy\":{\"type\":\"k8s\"},\"type\":\"calico\",\"vxlan_mac_prefix\":\"0E-2A\",\"vxlan_vni\":4096,\"windows_use_single_network\":true}]}"
},
"IFName": "eth0"
}
]
},
"config": {
"containerd": {
"snapshotter": "windows",
"defaultRuntimeName": "runhcs-wcow-process",
"defaultRuntime": {
"runtimeType": "",
"runtimePath": "",
"runtimeEngine": "",
"PodAnnotations": [],
"ContainerAnnotations": [],
"runtimeRoot": "",
"options": {},
"privileged_without_host_devices": false,
"baseRuntimeSpec": "",
"cniConfDir": "c:\\var\\lib\\rancher\\rke2\\agent\\etc\\cni",
"cniMaxConfNum": 0
},
"untrustedWorkloadRuntime": {
"runtimeType": "",
"runtimePath": "",
"runtimeEngine": "",
"PodAnnotations": [],
"ContainerAnnotations": [],
"runtimeRoot": "",
"options": {},
"privileged_without_host_devices": false,
"baseRuntimeSpec": "",
"cniConfDir": "c:\\var\\lib\\rancher\\rke2\\agent\\etc\\cni",
"cniMaxConfNum": 0
},
"runtimes": {
"runhcs-wcow-process": {
"runtimeType": "io.containerd.runhcs.v1",
"runtimePath": "",
"runtimeEngine": "",
"PodAnnotations": [],
"ContainerAnnotations": [],
"runtimeRoot": "",
"options": {},
"privileged_without_host_devices": false,
"baseRuntimeSpec": "",
"cniConfDir": "c:\\var\\lib\\rancher\\rke2\\agent\\etc\\cni",
"cniMaxConfNum": 0
}
},
"noPivot": false,
"disableSnapshotAnnotations": false,
"discardUnpackedLayers": false,
"ignoreRdtNotEnabledErrors": false
},
"cni": {
"binDir": "c:\\var\\lib\\rancher\\rke2\\bin",
"confDir": "c:\\var\\lib\\rancher\\rke2\\agent\\etc\\cni",
"maxConfNum": 1,
"confTemplate": "",
"ipPref": ""
},
"registry": {
"configPath": "",
"mirrors": {
"docker.io": {
"endpoint": [
"https://<removed>"
]
}
},
"configs": {},
"auths": {},
"headers": {
"User-Agent": [
"containerd/v1.6.1"
]
}
},
"imageDecryption": {
"keyModel": "node"
},
"disableTCPService": true,
"streamServerAddress": "127.0.0.1",
"streamServerPort": "0",
"streamIdleTimeout": "4h0m0s",
"enableSelinux": false,
"selinuxCategoryRange": 0,
"sandboxImage": "index.docker.io/rancher/pause:3.6",
"statsCollectPeriod": 10,
"systemdCgroup": false,
"enableTLSStreaming": false,
"x509KeyPairStreaming": {
"tlsCertFile": "",
"tlsKeyFile": ""
},
"maxContainerLogSize": 16384,
"disableCgroup": false,
"disableApparmor": false,
"restrictOOMScoreAdj": false,
"maxConcurrentDownloads": 3,
"disableProcMount": false,
"unsetSeccompProfile": "",
"tolerateMissingHugetlbController": false,
"disableHugetlbController": false,
"device_ownership_from_security_context": false,
"ignoreImageDefinedVolumes": false,
"netnsMountsUnderStateDir": false,
"enableUnprivilegedPorts": false,
"enableUnprivilegedICMP": false,
"containerdRootDir": "C:\\ProgramData\\containerd\\root",
"containerdEndpoint": "\\\\.\\pipe\\containerd-containerd-1.6.1",
"rootDir": "C:\\ProgramData\\containerd\\root\\io.containerd.grpc.v1.cri",
"stateDir": "C:\\ProgramData\\containerd\\state\\io.containerd.grpc.v1.cri"
},
"golang": "go1.17.2",
"lastCNILoadStatus": "OK",
"lastCNILoadStatus.default": "OK",
"lastCNILoadStatus.runhcs-wcow-process": "OK"
}
Show configuration if it is related to CRI plugin.
imports = []
version = 2
root = "C:\\ProgramData\\containerd\\root"
state = "C:\\ProgramData\\containerd\\state"
plugin_dir = ""
disabled_plugins = []
required_plugins = []
oom_score = 0
temp = ""
[grpc]
address = "\\\\.\\pipe\\containerd-containerd-1.6.1"
tcp_address = ""
tcp_tls_ca = ""
tcp_tls_cert = ""
tcp_tls_key = ""
uid = 0
gid = 0
max_recv_message_size = 16777216
max_send_message_size = 16777216
[ttrpc]
address = ""
uid = 0
gid = 0
[debug]
address = ""
format = ""
uid = 0
gid = 0
level = ""
[metrics]
address = ""
grpc_histogram = false
[cgroup]
path = ""
[plugins]
[plugins."io.containerd.gc.v1.scheduler"]
pause_threshold = 0.02
deletion_threshold = 0
mutation_threshold = 100
schedule_delay = "0s"
startup_delay = "100ms"
[plugins."io.containerd.grpc.v1.cri"]
disable_tcp_service = true
stream_server_address = "127.0.0.1"
stream_server_port = "0"
stream_idle_timeout = "4h0m0s"
enable_selinux = false
selinux_category_range = 0
sandbox_image = "index.docker.io/rancher/pause:3.6"
stats_collect_period = 10
systemd_cgroup = false
enable_tls_streaming = false
max_container_log_line_size = 16384
disable_cgroup = false
disable_apparmor = false
restrict_oom_score_adj = false
max_concurrent_downloads = 3
disable_proc_mount = false
unset_seccomp_profile = ""
tolerate_missing_hugetlb_controller = false
disable_hugetlb_controller = false
ignore_image_defined_volumes = false
device_ownership_from_security_context = false
enable_unprivileged_icmp = false
enable_unprivileged_ports = false
netns_mounts_under_state_dir = false
[plugins."io.containerd.grpc.v1.cri".containerd]
snapshotter = "windows"
default_runtime_name = "runhcs-wcow-process"
no_pivot = false
disable_snapshot_annotations = false
discard_unpacked_layers = false
ignore_rdt_not_enabled_errors = false
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
runtime_type = ""
runtime_path = ""
base_runtime_spec = ""
cni_conf_dir = "c:\\var\\lib\\rancher\\rke2\\agent\\etc\\cni"
cni_max_conf_num = 0
container_annotations = []
privileged_without_host_devices = false
pod_annotations = []
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
runtime_type = ""
runtime_engine = ""
runtime_path = ""
runtime_root = ""
privileged_without_host_devices = false
base_runtime_spec = ""
cni_conf_dir = "c:\\var\\lib\\rancher\\rke2\\agent\\etc\\cni"
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process]
runtime_type = "io.containerd.runhcs.v1"
runtime_engine = ""
runtime_path = ""
runtime_root = ""
base_runtime_spec = ""
privileged_without_host_devices = false
cni_conf_dir = "c:\\var\\lib\\rancher\\rke2\\agent\\etc\\cni"
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process.options]
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "c:\\var\\lib\\rancher\\rke2\\bin"
conf_dir = "c:\\var\\lib\\rancher\\rke2\\agent\\etc\\cni"
max_conf_num = 1
conf_template = ""
ip_pref = ""
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = ""
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["<removed>"]
[plugins."io.containerd.grpc.v1.cri".registry.auths]
[plugins."io.containerd.grpc.v1.cri".registry.configs]
[plugins."io.containerd.grpc.v1.cri".registry.headers]
[plugins."io.containerd.grpc.v1.cri".image_decryption]
key_model = "node"
[plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
tls_cert_file = ""
tls_key_file = ""
[plugins."io.containerd.internal.v1.opt"]
path = "C:\\ProgramData\\containerd\\root\\opt"
[plugins."io.containerd.internal.v1.restart"]
interval = "10s"
[plugins."io.containerd.internal.v1.tracing"]
sampling_ratio = 1.0
service_name = "containerd"
[plugins."io.containerd.metadata.v1.bolt"]
content_sharing_policy = "shared"
[plugins."io.containerd.runtime.v2.task"]
platforms = ["windows/amd64", "linux/amd64"]
sched_core = false
[plugins."io.containerd.service.v1.diff-service"]
default = ["windows", "windows-lcow"]
[plugins."io.containerd.service.v1.tasks-service"]
rdt_config_file = ""
[plugins."io.containerd.tracing.processor.v1.otlp"]
endpoint = ""
insecure = false
protocol = ""
[proxy_plugins]
[stream_processors]
[stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
args = ["--decryption-keys-path", "C:\\Program Files\\containerd\\ocicrypt\\keys"]
env = ["OCICRYPT_KEYPROVIDER_CONFIG=C:\\Program Files\\containerd\\ocicrypt\\ocicrypt_keyprovider.conf"]
path = "ctd-decoder"
returns = "application/vnd.oci.image.layer.v1.tar"
[stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
args = ["--decryption-keys-path", "C:\\Program Files\\containerd\\ocicrypt\\keys"]
env = ["OCICRYPT_KEYPROVIDER_CONFIG=C:\\Program Files\\containerd\\ocicrypt\\ocicrypt_keyprovider.conf"]
path = "ctd-decoder"
returns = "application/vnd.oci.image.layer.v1.tar+gzip"
[timeouts]
"io.containerd.timeout.bolt.open" = "0s"
"io.containerd.timeout.shim.cleanup" = "5s"
"io.containerd.timeout.shim.load" = "5s"
"io.containerd.timeout.shim.shutdown" = "3s"
"io.containerd.timeout.task.state" = "2s"
About this issue
- Original URL
- State: closed
- Created 2 years ago
- Comments: 16 (10 by maintainers)
@dcantah Merged the PR above. Can we close this issue now?
@dcantah - There are tests for no default storage size right? I’m surprised to see that this didn’t work