containerd: config.toml SystemdCgroup not work

Description

/etc/containerd/config.toml

          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            SystemdCgroup = true

the whole config.toml:

version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
plugin_dir = ""
disabled_plugins = []
required_plugins = []
oom_score = 0

[grpc]
  address = "/run/containerd/containerd.sock"
  tcp_address = ""
  tcp_tls_cert = ""
  tcp_tls_key = ""
  uid = 0
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216

[ttrpc]
  address = ""
  uid = 0
  gid = 0

[debug]
  address = ""
  uid = 0
  gid = 0
  level = ""

[metrics]
  address = ""
  grpc_histogram = false

[cgroup]
  path = ""

[timeouts]
  "io.containerd.timeout.shim.cleanup" = "5s"
  "io.containerd.timeout.shim.load" = "5s"
  "io.containerd.timeout.shim.shutdown" = "3s"
  "io.containerd.timeout.task.state" = "2s"

[plugins]
  [plugins."io.containerd.gc.v1.scheduler"]
    pause_threshold = 0.02
    deletion_threshold = 0
    mutation_threshold = 100
    schedule_delay = "0s"
    startup_delay = "100ms"
  [plugins."io.containerd.grpc.v1.cri"]
    disable_tcp_service = true
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    stream_idle_timeout = "4h0m0s"
    enable_selinux = false
    selinux_category_range = 1024
    sandbox_image = "k8s.gcr.io/pause:3.2"
    stats_collect_period = 10
    systemd_cgroup = false
    enable_tls_streaming = false
    max_container_log_line_size = 16384
    disable_cgroup = false
    disable_apparmor = false
    restrict_oom_score_adj = false
    max_concurrent_downloads = 3
    disable_proc_mount = false
    unset_seccomp_profile = ""
    tolerate_missing_hugetlb_controller = true
    disable_hugetlb_controller = true
    ignore_image_defined_volumes = false
    [plugins."io.containerd.grpc.v1.cri".containerd]
      snapshotter = "overlayfs"
      default_runtime_name = "runc"
      no_pivot = false
      disable_snapshot_annotations = true
      discard_unpacked_layers = false
      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
        privileged_without_host_devices = false
        base_runtime_spec = ""
      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
        privileged_without_host_devices = false
        base_runtime_spec = ""
      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          runtime_type = "io.containerd.runc.v2"
          runtime_engine = ""
          runtime_root = ""
          privileged_without_host_devices = false
          base_runtime_spec = ""
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            SystemdCgroup = true
    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "/opt/cni/bin"
      conf_dir = "/etc/cni/net.d"
      max_conf_num = 1
      conf_template = ""
    [plugins."io.containerd.grpc.v1.cri".registry]
      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://registry-1.docker.io"]
    [plugins."io.containerd.grpc.v1.cri".image_decryption]
      key_model = ""
    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""
  [plugins."io.containerd.internal.v1.opt"]
    path = "/opt/containerd"
  [plugins."io.containerd.internal.v1.restart"]
    interval = "10s"
  [plugins."io.containerd.metadata.v1.bolt"]
    content_sharing_policy = "shared"
  [plugins."io.containerd.monitor.v1.cgroups"]
    no_prometheus = false
  [plugins."io.containerd.runtime.v1.linux"]
    shim = "containerd-shim"
    runtime = "runc"
    runtime_root = ""
    no_shim = false
    shim_debug = false
  [plugins."io.containerd.runtime.v2.task"]
    platforms = ["linux/amd64"]
  [plugins."io.containerd.service.v1.diff-service"]
    default = ["walking"]
  [plugins."io.containerd.snapshotter.v1.devmapper"]
    root_path = ""
    pool_name = ""
    base_image_size = ""
    async_remove = false

systemctl daemon-reload systemctl restart containerd

Describe the results you received:

  1. crictl info: “systemdCgroup”: false,
  2. containerd config dump | grep SystemdCgroup: not get any result about cgroup

Describe the results you expected:

  1. crictl info: “systemdCgroup”: true,
  2. containerd config dump | grep SystemdCgroup: get SystemdCgroup = true

Output of containerd --version: both 1.3.9 and 1.4.3

About this issue

  • Original URL
  • State: closed
  • Created 3 years ago
  • Comments: 15 (6 by maintainers)

Most upvoted comments

@cyent This config option can not be verified by crictl info or config dump. I am afraid currently there is no way to “directly see” it in live configuration. Instead, you can verify it by the behavior.

For example, once you have systemdCgroup = true set in [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] You could see containerd-shim spawn in containerd.service cgroup, and Kubernetes pods spawn in the systemd root cgroup as kubepod.slice

Run systemctl status containerd. You will see a lot of containerd-shim-runc-v2 xxxx.

● containerd.service - containerd container runtime
   Loaded: loaded (/usr/lib/systemd/system/containerd.service; disabled; vendor preset: disabled)
   Active: active (running) since Fri 2021-01-08 10:29:34 CST; 20min ago
     Docs: https://containerd.io
  Process: 31034 ExecStartPre=/sbin/modprobe overlay (code=exited, status=0/SUCCESS)
 Main PID: 31038 (containerd)
    Tasks: 154
   Memory: 142.7M
   CGroup: /system.slice/containerd.service
           ├─31038 /opt/dev/bin/containerd --config=/opt/dev/cfg/config.toml
           ├─31524 /opt/dev/bin/containerd-shim-runc-v2 -namespace k8s.io -id 2a4230da69cf31db282d5c667693e7dd70efc7c4b9562e7074dc9fe7b5b0ab0f -address /run/containerd/container...
           ├─31548 /opt/dev/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4f163fe301930a50705c3cd73e9b5719affcd1f24df61ff30648d20fdcd06afd -address /run/containerd/container...
           ├─31565 /opt/dev/bin/containerd-shim-runc-v2 -namespace k8s.io -id 633a981b2ef5902f4c8b63954f36f48b32a598d45148f4e40ed1ad11add9abb7 -address /run/containerd/container...
           ├─31589 /opt/dev/bin/containerd-shim-runc-v2 -namespace k8s.io -id 609a087b09d6ca4332a29fe0e81051be9cf1b7da4e30ee5bd77603dac16c0d29 -address /run/containerd/container...
           ├─32196 /opt/dev/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e9ff74e5b3cb6b694b21420138169fa2d824f4945385c8a925ea27312b7d3c4 -address /run/containerd/container...

And you can verify the workload by running systemd-cgls to get a graph of cgroup trees. You will see all container process under the kubepod.slice at the systemd root cgroup instead of containerd.service or kubepod at the root.

├─1 /usr/lib/systemd/systemd --switched-root --system --deserialize 22
├─kubepods.slice
│ ├─kubepods-pod6dea264d_c2ad_4233_a681_53de224e0015.slice
│ │ ├─cri-containerd-b9881732f78c6c71cd61497e2fcd33eccb0724a31c3589c2e692202a01972bf5.scope
│ │ │ └─521 /opt/bin/flanneld
│ │ └─cri-containerd-58d8be75ff8d64f2c9ccdad321cf7604d1fe21749b0734cbf511eaa88225a111.scope
│ │   └─32411 /pause
...

After reading several blogs and trying a lot of things I got to find a workaround on a Chinese blog post:

You may use this configuration for etc/containerd/contig.toml:

version = 2

[plugins]
  [plugins."io.containerd.grpc.v1.cri"]
    systemd_cgroup = true
    [plugins."io.containerd.grpc.v1.cri".containerd]
      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        runtime_type = "io.containerd.runtime.v1.linux"
    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "/usr/lib/cni"
      conf_dir = "/etc/cni/net.d"
  [plugins."io.containerd.internal.v1.opt"]
    path = "/var/lib/containerd/opt"

The important values are systemd_cgroup as true and the default_runtime with runtime_type with the old version using io.containerd.runtime.v1.linux … the other values may be different for your environment, please use it only as a reference.

Once the configuration file is updated run systemctl restart containerd and check the new systemdCgroup as true running the command crictl info

I hope this is useful to someone else.

Tried on a Debian GNU/Linux 11 Bullseye AMD64 with containerd version 1.4.13~ds1 and cri-tools 1.0.0-beta.1-00