ceph-csi: ceph-fuse command hangs

logs from one of the plugin pod

i-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       561     1  0 17:51 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       563   561  0 17:51 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       580     1  0 17:51 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       582   580  0 17:51 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       600     1  0 17:52 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       604   600  0 17:52 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       621     1  0 17:52 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       623   621  0 17:52 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       640     1  0 17:53 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       642   640  0 17:53 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       659     1  0 17:53 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       663   659  0 17:53 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       680     1  0 17:53 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       682   680  0 17:53 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       714     1  0 17:53 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty
root       717   714  0 17:53 ?        00:00:00 ceph-fuse /var/lib/kubelet/plugins_registry/csi-cephfsplugin/controller/volumes/root-csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f -c /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.conf -n client.admin --keyring /etc/ceph/ceph.share.csi-cephfs-pvc-279b619c-1e6d-11e9-8331-52540035463f.client.admin.keyring -r / -o nonempty

I think https://github.com/ceph/ceph-csi/blob/07dbea766e6b9b3fad48d0ccef9ead73fdd3f09a/pkg/cephfs/util.go#L44 cmd.CombinedOutput() can hang and cause this issue.

About this issue

  • Original URL
  • State: closed
  • Created 5 years ago
  • Comments: 18 (7 by maintainers)

Most upvoted comments

It should’ve exited after the error you were getting (I’m guessing this is related to https://github.com/ceph/ceph-csi/pull/133 ?) . We could add a timeout for such cases when we encounter a run-away process like this one.