csharp: `tar` process hangs when reading `stdin` in `NamespacedPodExecAsync`

I am trying to implement cp command by starting tar process - the same principe as in kubectl.

Here is th code.

        public async Task<int> CopyFileToPodAsync(string name, string @namespace, string container, string sourceFilePath, string destinationFilePath, CancellationToken cancellationToken = default(CancellationToken))
        {
            // All other parameters are being validated by MuxedStreamNamespacedPodExecAsync called by NamespacedPodExecAsync
            ValidatePathParameters(sourceFilePath, destinationFilePath);

            // The callback which processes the standard input, standard output and standard error of exec method
            var handler = new ExecAsyncCallback(async (stdIn, stdOut, stdError) =>
            {
                var fileInfo = new FileInfo(destinationFilePath);
                try
                {
                    using (var outputStream = new MemoryStream())
                    {
                        using (var inputFileStream = File.OpenRead(sourceFilePath))
                        using (var gZipOutputStream = new GZipOutputStream(outputStream))
                        using (var tarOutputStream = new TarOutputStream(gZipOutputStream))
                        {
                            // To avoid gZipOutputStream to close the memoryStream
                            gZipOutputStream.IsStreamOwner = false;

                            var fileSize = inputFileStream.Length;
                            var entry = TarEntry.CreateTarEntry(fileInfo.Name);
                            entry.Size = fileSize;

                            tarOutputStream.PutNextEntry(entry);
                            inputFileStream.CopyTo(tarOutputStream);
                            tarOutputStream.CloseEntry();
                        }
                        outputStream.Position = 0;

                        await outputStream.CopyToAsync(stdIn);
                        await outputStream.FlushAsync();
                    }
                }
                catch (Exception ex)
                {
                    throw new IOException($"Copy command failed: {ex.Message}");
                }

                using (System.IO.StreamReader streamReader = new System.IO.StreamReader(stdError))
                {
                    while (streamReader.EndOfStream == false)
                    {
                        string error = await streamReader.ReadToEndAsync();
                        throw new IOException($"Copy command failed: {error}");

                    }
                }


            });

            var destinationFolder = GetFolderName(destinationFilePath);

            return await Kubernetes.NamespacedPodExecAsync(
                name,
                @namespace,
                container,
                new string[] { "sh", "-c", $"tar -xzmf - -C {destinationFolder}" },
                false,
                handler,
                cancellationToken);
        }

uplaod looks ok, but the process stays blocked by tar.

when it is blocked, the size of file is not the same as after clossing the program

While in blocked state:

root@upload-demo--1-bfjxl:/# ls -l /home/
total 65472
-rwx------ 1 root root 38338048 Jul 13 16:13 input_file_large

After I close the program:

root@upload-demo--1-bfjxl:/# ls -l /home/
total 38872
-rwx------ 1 root root 39803980 Jul 13 16:14 input_file_large

About this issue

  • Original URL
  • State: closed
  • Created 2 years ago
  • Reactions: 1
  • Comments: 16 (16 by maintainers)

Most upvoted comments

could you please send a pr to example folder which would help others as well

thank you

@tg123 yes, it works! The size of input stream chunk was a problem. Thanks a lot! here is the full example

public async Task<int> CopyFileToPodAsync(string name, string @namespace, string container, string sourceFilePath, string destinationFilePath, CancellationToken cancellationToken = default(CancellationToken))
{
    // All other parameters are being validated by MuxedStreamNamespacedPodExecAsync called by NamespacedPodExecAsync
    ValidatePathParameters(sourceFilePath, destinationFilePath);

    // The callback which processes the standard input, standard output and standard error of exec method
    var handler = new ExecAsyncCallback(async (stdIn, stdOut, stdError) =>
    {
        var fileInfo = new FileInfo(destinationFilePath);
        try
        {
            using (var memoryStream = new MemoryStream())
            {
                using (var inputFileStream = File.OpenRead(sourceFilePath))
                using (var tarOutputStream = new TarOutputStream(memoryStream)) // optionally set encoding
                {
                    tarOutputStream.IsStreamOwner = false;

                    var fileSize = inputFileStream.Length;
                    var entry = TarEntry.CreateTarEntry(fileInfo.Name);

                    entry.Size = fileSize;

                    tarOutputStream.PutNextEntry(entry);
                    await inputFileStream.CopyToAsync(tarOutputStream);
                    tarOutputStream.CloseEntry();
                }

                memoryStream.Position = 0;

                const int bufferSize = 31 * 1024 * 1024; // must be lower than 32 * 1024 * 1024
                byte[] localBuffer = new byte[bufferSize];
                while (true)
                {
                    int numRead = await memoryStream.ReadAsync(localBuffer, 0, localBuffer.Length);
                    if (numRead <= 0)
                    {
                        break;
                    }
                    await stdIn.WriteAsync(localBuffer, 0, numRead);
                }
                await stdIn.FlushAsync();
            }

        }
        catch (Exception ex)
        {
            throw new IOException($"Copy command failed: {ex.Message}");
        }

        using StreamReader streamReader = new StreamReader(stdError);
        while (streamReader.EndOfStream == false)
        {
            string error = await streamReader.ReadToEndAsync();
            throw new IOException($"Copy command failed: {error}");
        }
    });

    string destinationFolder = GetFolderName(destinationFilePath);

    return await Kubernetes.NamespacedPodExecAsync(
        name,
        @namespace,
        container,
        new string[] { "sh", "-c", $"tar xmf - -C {destinationFolder}" },
        false,
        handler,
        cancellationToken);
}