runtime: HttpWebResponse returns different "Set-Cookie" header value than under .NET Framework

Under .NET Framework, HttpWebResponse.Headers can deliver the Set-Cookie header value as multiple values, where each value represents one cookie. HttpWebResponse.Headers is a WebHeaderCollection and invoking GetValues("Set-Cookie") returns an array of strings where each string is a single cookie. In .NET Core, however, the same returns the entire header as a single string; that is GetValues("Set-Cookie") always returns an array of one string with comma-separated cookies. This seems to be a compatibility bug that yields different results at run-time when the same code is executed under .NET Framework and .NET Core.

I have created a self-contained program to demonstrate the issue:

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;

#if !NETFX
using Microsoft.AspNetCore;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
#endif

static class Program
{
    static async Task<int> Main(string[] args)
    {
        try
        {
            Console.WriteLine((RuntimeInformation.FrameworkDescription + " ").PadRight(70, '-'));
            await Wain(args);
            return 0;
        }
        catch (Exception e)
        {
            Console.Error.WriteLine(e);
            return 1;
        }
    }

    // The following client code is identical between .NET Core and .NET
    // Framework versions.

    static class Client
    {
        public static void Run(Uri url)
        {
            var request = WebRequest.CreateHttp(url);

            using var response = (HttpWebResponse) request.GetResponse();
            Console.WriteLine((int)response.StatusCode + " " + response.StatusDescription);

            var headers =
                from i in Enumerable.Range(0, response.Headers.Count)
                select (Name: response.Headers.GetKey(i), Values: response.Headers.GetValues(i)) into h
                from v in h.Values
                select (h.Name, v);

            foreach (var (name, value) in headers)
                Console.WriteLine(name + ": " + value);

            Console.WriteLine();

            using var stream = response.GetResponseStream();
            using var reader = new StreamReader(stream);
            Console.WriteLine(reader.ReadToEnd());
        }
    }

#if NETFX

    // Under .NET Framework, just run the client.

    static Task Wain(string[] args)
    {
        if (args.Length == 0)
            throw new Exception("Missing URL argument.");
        Client.Run(new Uri(args[0]));
        return Task.CompletedTask;
    }

#else

    // The server that responds with a plain text message and two cookies.

    static class Server
    {
        public static IWebHost Build(string[] args) =>
            WebHost
                .CreateDefaultBuilder(args)
                .Configure(app =>
                {
                    app.Run(async (context) =>
                    {
                        var response = context.Response;
                        var cookies = response.Cookies;
                        cookies.Append("foo", "bar");
                        cookies.Append("bar", "baz");
                        response.ContentType = "text/plain";
                        await response.WriteAsync("Hello World!\n");
                    });
                })
                .Build();
    }

    // Under .NET Core, runs:
    // - the web server
    // - then the .NET Core client
    // - then the .NET Framework client indirectly via `dotnet run`

    static async Task Wain(string[] args)
    {
        var host = Server.Build(args);
        host.Start();

        try
        {
            var addresses = host.ServerFeatures.Get<IServerAddressesFeature>();
            var url = addresses.Addresses
                               .Select(addr => new Uri(addr))
                               .First(url => url.Scheme == Uri.UriSchemeHttp);

            Client.Run(url);

            // Find the project directory and run the .NET Framework version
            // via `dotnet run`, re-directing standard output and error here.

            var appDir = new DirectoryInfo(AppContext.BaseDirectory);
            var projectDir = appDir.Ascendants().First(dir => dir.EnumerateFiles("*.csproj").Any());

            var psi = new ProcessStartInfo("dotnet", "run --framework net471 " + url)
            {
                CreateNoWindow = true,
                UseShellExecute = false,
                WorkingDirectory = projectDir.FullName,
                RedirectStandardOutput = true,
                RedirectStandardError = true,
            };

            using var process = Process.Start(psi);

            static DataReceivedEventHandler CreateDataReceiverFor(TextWriter writer) => (_, e) =>
            {
                if (e.Data is string line)
                    writer.WriteLine(line);
            };

            process.OutputDataReceived += CreateDataReceiverFor(Console.Out);
            process.ErrorDataReceived  += CreateDataReceiverFor(Console.Error);

            process.BeginOutputReadLine();
            process.BeginErrorReadLine();
            process.WaitForExit();

            if (process.ExitCode != 0)
                throw new Exception($"The .NET Framework version of the program exited with a non-zero code of {process.ExitCode}.");
        }
        finally
        {
            // Stop the web server.

            var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
            await host.StopAsync(cts.Token);
        }
    }

    static IEnumerable<DirectoryInfo> Ascendants(this DirectoryInfo dir)
    {
        for (var parent = dir.Parent; parent != null; parent = parent.Parent)
            yield return parent;
    }

#endif
}

When run as a .NET Core 2.2 application, this program will do the following:

It will run a web server (Kestrel) that responds with a plain text message (that reads “Hello World!”) and two cookies (foo=bar and bar=baz).
It will then issue an HTTP request and dump the response status, headers and body as text.
It will do the same as 2, but under .NET Framework. This step is done by running the same project via dotnet run but with the --framework net471 option.

The output of the program shows the difference in behaviour:

.NET Core 4.6.27817.03 -----------------------------------------------
200 OK
Date: Tue, 16 Jul 2019 12:04:59 GMT
Server: Kestrel
Transfer-Encoding: chunked
Set-Cookie: foo=bar; path=/, bar=baz; path=/
Content-Type: text/plain

Hello World!

.NET Framework 4.7.3416.0 --------------------------------------------
200 OK
Transfer-Encoding: chunked
Content-Type: text/plain
Date: Tue, 16 Jul 2019 12:05:01 GMT
Set-Cookie: foo=bar; path=/
Set-Cookie: bar=baz; path=/
Server: Kestrel

Hello World!

Specifically, under .NET Core, we see a single Set-Cookie line with both cookies:

Set-Cookie: foo=bar; path=/, bar=baz; path=/

whereas under .NET Framework, we see two, one per cookie:

Set-Cookie: foo=bar; path=/
Set-Cookie: bar=baz; path=/

I have uploaded a ZIP archive with the full project:

📎 CookiesBugDemo.zip

Simply unzip and execute the run.cmd batch script included.

More Information

dotnet --info says:

.NET Core SDK (reflecting any global.json):
 Version:   2.2.204
 Commit:    8757db13ec

Runtime Environment:
 OS Name:     Windows
 OS Version:  10.0.17763
 OS Platform: Windows
 RID:         win10-x64
 Base Path:   C:\Program Files\dotnet\sdk\2.2.204\

Host (useful for support):
  Version: 3.0.0-preview7-27902-19
  Commit:  fbe9466ddd

.NET Core SDKs installed:
  1.1.13 [C:\Program Files\dotnet\sdk]
  1.1.14 [C:\Program Files\dotnet\sdk]
  2.1.101 [C:\Program Files\dotnet\sdk]
  2.1.103 [C:\Program Files\dotnet\sdk]
  2.1.104 [C:\Program Files\dotnet\sdk]
  2.1.200 [C:\Program Files\dotnet\sdk]
  2.1.201 [C:\Program Files\dotnet\sdk]
  2.1.202 [C:\Program Files\dotnet\sdk]
  2.1.300 [C:\Program Files\dotnet\sdk]
  2.1.400 [C:\Program Files\dotnet\sdk]
  2.1.402 [C:\Program Files\dotnet\sdk]
  2.1.403 [C:\Program Files\dotnet\sdk]
  2.1.500 [C:\Program Files\dotnet\sdk]
  2.1.502 [C:\Program Files\dotnet\sdk]
  2.1.505 [C:\Program Files\dotnet\sdk]
  2.1.507 [C:\Program Files\dotnet\sdk]
  2.1.508 [C:\Program Files\dotnet\sdk]
  2.1.602 [C:\Program Files\dotnet\sdk]
  2.1.604 [C:\Program Files\dotnet\sdk]
  2.1.700 [C:\Program Files\dotnet\sdk]
  2.1.701 [C:\Program Files\dotnet\sdk]
  2.2.101 [C:\Program Files\dotnet\sdk]
  2.2.202 [C:\Program Files\dotnet\sdk]
  2.2.204 [C:\Program Files\dotnet\sdk]
  2.2.300 [C:\Program Files\dotnet\sdk]
  2.2.301 [C:\Program Files\dotnet\sdk]
  3.0.100-preview7-012802 [C:\Program Files\dotnet\sdk]

.NET Core runtimes installed:
  Microsoft.AspNetCore.All 2.1.0 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.All]
  Microsoft.AspNetCore.All 2.1.2 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.All]
  Microsoft.AspNetCore.All 2.1.4 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.All]
  Microsoft.AspNetCore.All 2.1.5 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.All]
  Microsoft.AspNetCore.All 2.1.6 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.All]
  Microsoft.AspNetCore.All 2.1.9 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.All]
  Microsoft.AspNetCore.All 2.1.11 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.All]
  Microsoft.AspNetCore.All 2.1.12 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.All]
  Microsoft.AspNetCore.All 2.2.0 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.All]
  Microsoft.AspNetCore.All 2.2.3 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.All]
  Microsoft.AspNetCore.All 2.2.5 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.All]
  Microsoft.AspNetCore.All 2.2.6 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.All]
  Microsoft.AspNetCore.App 2.1.0 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.AspNetCore.App 2.1.2 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.AspNetCore.App 2.1.4 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.AspNetCore.App 2.1.5 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.AspNetCore.App 2.1.6 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.AspNetCore.App 2.1.9 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.AspNetCore.App 2.1.11 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.AspNetCore.App 2.1.12 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.AspNetCore.App 2.2.0 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.AspNetCore.App 2.2.3 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.AspNetCore.App 2.2.5 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.AspNetCore.App 2.2.6 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.AspNetCore.App 3.0.0-preview7.19353.9 [C:\Program Files\dotnet\shared\Microsoft.AspNetCore.App]
  Microsoft.NETCore.App 1.0.15 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 1.0.16 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 1.1.12 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 1.1.13 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.0.6 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.0.7 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.0.9 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.1.0 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.1.2 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.1.4 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.1.5 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.1.6 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.1.9 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.1.11 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.1.12 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.2.0 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.2.3 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.2.5 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 2.2.6 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.NETCore.App 3.0.0-preview7-27902-19 [C:\Program Files\dotnet\shared\Microsoft.NETCore.App]
  Microsoft.WindowsDesktop.App 3.0.0-preview7-27902-19 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]

To install additional .NET Core runtimes or SDKs:
  https://aka.ms/dotnet-download

About this issue

Original URL
State: open
Created 5 years ago
Comments: 16 (9 by maintainers)

Commits related to this issue

Workaround cookie header regression between .NET Framework & Core https://github.com/dotnet/corefx/issues/39527 — committed to atifaziz/WebLinq by atifaziz 5 years ago

Most upvoted comments

Apparently this issue breaks Azure Function Proxies: https://github.com/Azure/azure-functions-host/issues/4486 Browsers (e.g. currently latest Chrome and Firefox) ignore all other cookies but the first one. This is not a small issue.

nikoudel on Feb 11, 2021

I don’t think we’re trying hard to keep it the same way – it is more that we try hard to not make changes without fully understanding their scope.

The current implementation is clearly broken, so I think we need to make some change that will break anyone depending on a comma being there.

I’m leaning towards reverting to the old behavior of returning separately.

scalablecory on Sep 17, 2019

@davidsh Thanks for reconsidering this.

The delimiter in that case should be a semicolon.

This won’t be helpful as semi-colon (;) is already taken in the cookie syntax to delimit attribute-value pairs (per section 4.1.1 of RFC 6252):

set-cookie-string = cookie-pair *( ";" SP cookie-av )

We will investigate if we can at least correct the delimiter problem even if we still have to have a single ‘Set-Cookie’ header.

Why try so hard to return a single header when GetValues(header) does the right thing already and returns each Set-Cookie header separately as an array of strings? It’s just GetValues(index) that’s the problem. Even if the docs add a compatibility note (thanks @KathleenDollard and @mairaw for taking note), no one in their right mind would use the overload with the regression.

atifaziz on Sep 17, 2019

Just in case, as a workaround: Implementation of HttpResponseMessage doesn’t have this issue and can be used instead of HttpWebResponse. So you have two choices:

Use HttpClient.GetAsync() instead of HttpWebRequest.GetResponseAsync(). Something like this:

            HttpClient client = new HttpClient();
            var resp = await client.GetAsync(your_url);
            var headerValues = resp.Headers.GetValues("Set-Cookie");

Retrieve HttpResponseMessage from the HttpWebResponse private field. It’s ugly but someone may use it as temporally solution. Like this:

        internal static IEnumerable<string> GetSetCookieHeaderValues(HttpWebResponse response)
        {
            if (response == null || !response.Headers.AllKeys.Contains("set-cookie", StringComparer.OrdinalIgnoreCase))
            {
                return null;
            }
            string headerName = response.Headers.AllKeys.First(k => string.Equals(k, "set-cookie", StringComparison.OrdinalIgnoreCase));
            BindingFlags bindFlags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic
                | BindingFlags.Static;
            FieldInfo field = response.GetType().GetField("_httpResponseMessage", bindFlags);
            return (field.GetValue(response) as HttpResponseMessage)?.Headers.GetValues(headerName);
        }

vfedonkin on Jan 10, 2020

Do we need the investigation to be concluded before we update the docs?

@mairaw Yes. Let’s wait on any doc changes for now until we finish investigating. We will open a separate doc issue in the dotnet/dotnet-api-docs repo once that is done.

davidsh on Sep 16, 2019

Returning the Set-Cookie header as a single string is, in fact incorrect, because the header value syntax is invalid per RFC 6265.

and separated by commas

You are correct that using comma as the delimiter between cookies in the single ‘Set-Cookie’ header is incorrect. The delimiter in that case should be a semicolon.

We will investigate if we can at least correct the delimiter problem even if we still have to have a single ‘Set-Cookie’ header.

davidsh on Sep 16, 2019

How is it semantically correct?

According to RFC 6265, there is no order dependencies of cookies received by ‘Set-Cookie’ headers. So, a user-agent (client) should process the cookies the same way regardless of the order they appear within one or more ‘Set-Cookie’ response headers.

This seems like a regression from the fact the implementation of HttpWebRequest is different.

In .NET Core, the HttpWebRequest API is built on top of the HttpClient API. HttpClient coalesces all the ‘Set-Cookie’ response headers into a single array of cookies. And that is why it appears as a single ‘Set-Cookie’ header as viewed by the HttpWebRequest API. This is different from the .NET Framework implementation of HttpWebRequest. But in practice, we haven’t seen any broken applications due to this since according to the RFC, a client shouldn’t expect the cookies to be ordered in any particular way from the server.

Add a compatibility note to the HttpWebResponse.Headers documentation. /cc @KathleenDollard

This is a good point. Feel free to open an issue in https://github.com/dotnet/dotnet-api-docs/issues. Or you can even submit a PR to change the documentation to add more info about this. The ‘Remarks’ section of the API docs is where we currently put compatibility notes like this.

davidsh on Sep 12, 2019

This seems like a regression from the fact the implementation of HttpWebRequest is different. It is technical breaking change, but semantically the behavior is correct. Given we have only 1 complaint, this does not seem to be high-value enough to fix it. Closing.

karelz on Sep 12, 2019