aspnetcore: At random intervals the application fails to set RouteValues

Is there an existing issue for this?

• I have searched the existing issues

Describe the bug

At random intervals (6-24 hours) the application fails to set RouteValues, which raises an exception. Parallel or subsequent requests completes successfully, it only happens once in a blue moon.

Technical details: .NET 6.0.5 Windows Server 2019, IIS 10 YARP (Reverse Proxy) 1.1.0 Application Insights 2.20.0

The bug happened with Ocelot as well as YARP, so i don’t think either of them is the reason.

System.NullReferenceException: Object reference not set to an instance of an object.
   at Microsoft.AspNetCore.Http.DefaultHttpRequest.set_RouteValues(RouteValueDictionary value)
   at Microsoft.AspNetCore.Routing.Matching.DfaMatcher.MatchAsync(HttpContext httpContext)
   at Microsoft.AspNetCore.Routing.EndpointRoutingMiddleware.Invoke(HttpContext httpContext)
   at Microsoft.AspNetCore.Authentication.AuthenticationMiddleware.Invoke(HttpContext context)
   at Microsoft.AspNetCore.Server.IIS.Core.IISHttpContextOfT`1.ProcessRequestAsync()

appsettings.json

{
  "Logging": {
    "LogLevel": {
      "Default": "Warning",
      "Microsoft": "Warning",
      "Microsoft.Hosting.Lifetime": "Warning"
    }
  },
  "Serilog": {
    "Using": [ "Serilog.Sinks.File", "Serilog.Sinks.ApplicationInsights" ],
    "MinimumLevel": "Warning",
    "WriteTo": [
      {
        "Name": "Async",
        "Args": {
          "configure": [
            {
              "Name": "File",
              "Args": {
                "path": "Logs/serilog.txt",
                "rollingInterval": "Day",
                "rollOnFileSizeLimit": true,
                "retainedFileCountLimit": 20,
                "fileSizeLimitBytes": 104857600
              }
            }
          ]
        }
      },
      {
        "Name": "ApplicationInsights",
        "Args": {
          "InstrumentationKey": "xxx",
          "restrictedToMinimumLevel": "Information",
          "telemetryConverter": "Serilog.Sinks.ApplicationInsights.Sinks.ApplicationInsights.TelemetryConverters.TraceTelemetryConverter, Serilog.Sinks.ApplicationInsights"
        }
      }
    ],
    "Enrich": [ "FromLogContext", "WithMachineName", "WithThreadId" ]
  },
  "AllowedHosts": "*",
  "ConnectionStrings": {
    "CmsAdmin": "Data Source=xxx;User ID=xxx;Password=xxx;Initial Catalog=xxx;MultipleActiveResultSets=True"
  },
  "InternalIPWhiteList": [
    "10.0.0.0/8",
    "172.16.0.0/12",
    "192.168.0.0/16",
    "::1",
    "127.0.0.1"
  ],
  "ApplicationInsights": {
    "ConnectionString": "xxx",
    "DeveloperMode": false,
    "EnableRequestTrackingTelemetryModule": true,
    "EnableAdaptiveSampling": true,
    "EnableActiveTelemetryConfigurationSetup": true
  },
  "ReverseProxy": {
    "Routes": {
      "route1": {
        "ClusterId": "cluster1",
        "AuthorizationPolicy": "default",
        "Match": {
          "Path": "{**catch-all}"
        },
        "AllowAnonymous": false
      }
    },
    "Clusters": {
      "cluster1": {
        "Destinations": {
          "destination1": {
            "Address": "http://192.168.197.13"
          },
          "destination2": {
            "Address": "http://192.168.197.14"
          }
        }
      }
    },
    "LoadBalancingPolicy": "PowerOfTwoChoices"
  }
}

Startup.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using xx.Api.Gateway.Services;
using xx.Api.Handlers;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.HttpOverrides;
using Microsoft.AspNetCore.HttpsPolicy;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Hosting.Internal;
using Microsoft.Extensions.Logging;
using Serilog;
using Yarp.ReverseProxy.Transforms;

namespace xx.Api.Gateway
{
    public class Startup
    {
        public Startup(IConfiguration configuration, IWebHostEnvironment env)
        {
            Configuration = configuration;
            _env = env;
        }

        public IConfiguration Configuration { get; }

        private readonly IWebHostEnvironment _env;

        // This method gets called by the runtime. Use this method to add services to the container.
        public void ConfigureServices(IServiceCollection services)
        {
            services.AddCors();
            services.AddOptions();
            services.AddMemoryCache();
            services.AddHttpContextAccessor();
            
            // configure basic authentication 
            services.AddAuthentication("BasicAuthentication")
                .AddScheme<AuthenticationSchemeOptions, BasicAuthenticationHandler>("BasicAuthentication", null);

            // configure DI for application services
            services.AddScoped<IAccountService, AccountService>();
            services.AddApplicationInsightsTelemetry();
            services.AddSingleton<IPService>();

            var proxyBuilder = services.AddReverseProxy();
            // Initialize the reverse proxy from the "ReverseProxy" section of configuration
            proxyBuilder.LoadFromConfig(Configuration.GetSection("ReverseProxy"));
            proxyBuilder.AddTransforms(builderContext =>
            {
                // Conditionally add a transform for routes that require auth.
                if (!string.IsNullOrEmpty(builderContext.Route.AuthorizationPolicy))
                {
                    builderContext.AddRequestTransform(transformContext =>
                    {
                        var user = transformContext.HttpContext.User;
                        var version = user.Claims.FirstOrDefault(x => x.Type.Equals("ApiVersion"))?.Value;
                        var hostname = user.Claims.FirstOrDefault(x => x.Type.Equals("ApiHostname"))?.Value;
                        var urlVersion = user.Claims.FirstOrDefault(x => x.Type.Equals("ApiUrlVersion"))?.Value;

                        var environment = _env.EnvironmentName == "Production" ? "" : "-development";
                        if (!string.IsNullOrEmpty(version) && !string.IsNullOrEmpty(hostname) && !string.IsNullOrEmpty(urlVersion))
                        {
                            transformContext.ProxyRequest.Headers.Remove("X-Api-Version");
                            transformContext.ProxyRequest.Headers.Remove("X-Api-Hostname");
                            transformContext.ProxyRequest.Headers.Add("X-Api-Version", version);
                            transformContext.ProxyRequest.Headers.Add("X-Api-Hostname", hostname);
                            transformContext.Path = string.Format("/v{0}{1}{2}", urlVersion, environment, transformContext.Path);
                        }

                        return default;
                    });
                }
            });
        }

        // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
        public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
        {
            // global cors policy
            app.UseCors(x => x
                .AllowAnyOrigin()
                .AllowAnyMethod()
                .AllowAnyHeader());

            //app.UseHttpsRedirection();

            app.UseAuthentication();         
            app.UseRouting();
            app.UseAuthorization();
            app.UseEndpoints(endpoints =>
            {
                endpoints.MapReverseProxy();
            });

        }
    }
}

BasicAuthenticationHandler:

using System;
using System.Net.Http.Headers;
using System.Security.Claims;
using System.Text;
using System.Text.Encodings.Web;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authentication;
using Microsoft.Extensions.Options;
using xx.Api.Gateway.Entities;
using xx.Api.Gateway.Services;
using Microsoft.AspNetCore.HttpOverrides;
using Microsoft.Extensions.Logging;
using Microsoft.ApplicationInsights;

namespace xx.Api.Handlers
{
    public class BasicAuthenticationHandler : AuthenticationHandler<AuthenticationSchemeOptions>
    {
        private readonly IAccountService _userService;
        private readonly IPService _ipService;
        private readonly TelemetryClient _telemetry;

        public BasicAuthenticationHandler(
            IOptionsMonitor<AuthenticationSchemeOptions> options,
            ILoggerFactory logger,
            UrlEncoder encoder,
            ISystemClock clock,
            IAccountService userService,
            IPService ipService,
            TelemetryClient telemetry)
            : base(options, logger, encoder, clock)
        {
            _userService = userService;
            _ipService = ipService;
            _telemetry = telemetry;
        }

        protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
        {
            var remoteIp = Request.HttpContext.Connection.RemoteIpAddress;
            Logger.LogInformation(remoteIp != null ? "Remote ip is: " + remoteIp.ToString() : "Remote ip is null");

            if (Request.Headers.ContainsKey("Authorization"))
            {
                Logger.LogDebug("Logging in with autorization");
                return await AuthenticateWithAuthorization();
            }
            
            if(remoteIp != null && _ipService.IsIPWhiteListed(remoteIp))
            {
                Logger.LogDebug("Logging in with hostname");
                return await AuthenticateWithApiHost();
            }

            return AuthenticateResult.Fail("Unknown authentication method.");
        }

        private async Task<AuthenticateResult> AuthenticateWithApiHost()
        {
            Account account = null;
            try
            {
                string apiHost;

                if(Request.Headers.ContainsKey("X-Api-Hostname"))
                {
                    apiHost = Request.Headers["X-Api-Hostname"];
                    Logger.LogInformation("X-Api-Hostname " + apiHost);
                }
                else
                {
                    var hosts = Request.Host.Host.Split(".");
                    if (hosts.Length < 3)
                    {
                        return AuthenticateResult.Fail("No hostname");
                    }

                    apiHost = hosts[0];
                    Logger.LogInformation("Parsed hostname " + apiHost);
                }

                account = await _userService.AuthenticateHost(apiHost);
            }
            catch(Exception ex)
            {
                _telemetry.TrackException(ex);
                return AuthenticateResult.Fail("Invalid hostname");
            }

            if (account == null)
                return AuthenticateResult.Fail("Invalid hostname");

            return CreateTicket(account);
        }

        protected async Task<AuthenticateResult> AuthenticateWithAuthorization()
        {
            Account account = null;
            try
            {
                var authHeader = AuthenticationHeaderValue.Parse(Request.Headers["Authorization"]);
                var credentialBytes = Convert.FromBase64String(authHeader.Parameter);
                var credentials = Encoding.UTF8.GetString(credentialBytes).Split(new[] { ':' }, 2);
                var username = credentials[0];
                var password = credentials[1];
                if(Request.Headers.ContainsKey("X-Api-Verify-Token"))
                {
                    account = await _userService.AuthenticateAccessToken(username, password);
                }
                else
                {
                    account = await _userService.Authenticate(username, password);
                }
                
            }
            catch(Exception ex)
            {
                _telemetry.TrackException(ex);
                return AuthenticateResult.Fail("Invalid Authorization Header");
            }

            if (account == null)
                return AuthenticateResult.Fail("Invalid Username or Password");

            return CreateTicket(account);
        }

        protected AuthenticateResult CreateTicket(Account account)
        {
            Version apiVersion = new Version(1,0);
            int urlVersion = 2;
            if(Request.Headers.ContainsKey("X-Api-Version"))
            {
                Version v;
                if(Version.TryParse(Request.Headers["X-Api-Version"].ToString(), out v))
                {
                    urlVersion = v.Major > 1 ? v.Major : 2;
                    apiVersion = v;
                }
            }
            Logger.LogInformation(string.Format("Api hostname: {0}, Api version: {1}, Url version: {2}", account.ApiHost, apiVersion, urlVersion));

            var name = string.IsNullOrEmpty(account.Username) ? account.ApiHost : account.Username;

            var claims = new[] {
                new Claim(ClaimTypes.NameIdentifier, account.Id.ToString()),
                new Claim(ClaimTypes.Name, name),
                new Claim("ApiHostname", account.ApiHost),
                new Claim("ApiUrlVersion", urlVersion.ToString()),
                new Claim("ApiVersion", apiVersion.ToString())
            };
            var identity = new ClaimsIdentity(claims, Scheme.Name);
            var principal = new ClaimsPrincipal(identity);
            var ticket = new AuthenticationTicket(principal, Scheme.Name);

            return AuthenticateResult.Success(ticket);
        }
    }
}

Expected Behavior

No exception

Steps To Reproduce

Sadly i can’t reproduce it, since i don’t know what’s actually wrong.

Exceptions (if any)

System.NullReferenceException: Object reference not set to an instance of an object. at Microsoft.AspNetCore.Http.DefaultHttpRequest.set_RouteValues(RouteValueDictionary value) at Microsoft.AspNetCore.Routing.Matching.DfaMatcher.MatchAsync(HttpContext httpContext) at Microsoft.AspNetCore.Routing.EndpointRoutingMiddleware.Invoke(HttpContext httpContext) at Microsoft.AspNetCore.Authentication.AuthenticationMiddleware.Invoke(HttpContext context) at Microsoft.AspNetCore.Server.IIS.Core.IISHttpContextOfT`1.ProcessRequestAsync()

.NET Version

6.0.5

Anything else?

.NET 6.0.5 Windows Server 2019, IIS 10 YARP (Reverse Proxy) 1.1.0 Application Insights 2.20.0