runtime: The SSL connection could not be established after bypassing validation

Team,

Hosted a proxy api in .net core 3.1 on Azure stack as http(not https) app service. Internally, this api calls a secure https vault api which validates my api via mutual TLS authentication certificate. The backend servers of Azure stack doesn’t have updated TLS cipher suits and very soon the patch will be applied. But, till then, I thought to bypass the https negotiation validation check. However, even after bypassing the check, we’re still facing SSL connection issue intermittently.

This is the code of the api:

Startup.cs

 ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true;
    services.AddHttpClient(Configuration["ADClient"], c =>
            {
                c.BaseAddress = new Uri(Configuration["ADApi"]);
                c.DefaultRequestHeaders.Add(Configuration["Headers:Authentication"], Configuration["Token"]);
            }).ConfigureHttpMessageHandlerBuilder(builder =>
            {
                builder.PrimaryHandler = new HttpClientHandler
                {
                    ServerCertificateCustomValidationCallback = (m, c, ch, e) => true,
                    SslProtocols = System.Security.Authentication.SslProtocols.Tls12
            };
            })
            
            services.AddHttpClient(Configuration["VaultClient"], c =>
            {
                c.BaseAddress = new Uri(Configuration["VaultApi"]);
            }).ConfigurePrimaryHttpMessageHandler<CertificateRegistryHttpHandler>().AddPolicyHandlerFromRegistry("CircuitFallBackWrap"); 
    
     service.AddSingleton<IHttpClient, HttpClient>(); //My custom http client class 
    
     service.AddTransient<CertificateRegistryHttpHandler>();  // Here life time doesn't matter because Http Client is singleton

The code for HTTPClient is:

       public interface IHttpClient
    {
        Task<string> Get(string url, string clientName);
    }
    public class HttpClient : IHttpClient
    {
        private readonly IHttpClientFactory _clientFactory;
        public HttpClient(IHttpClientFactory clientFactory)
        {
            _clientFactory = clientFactory;
        }
       
        public async Task<string> Get(string url, string clientName)
        {
            string data = String.Empty;
            using (var request = new HttpRequestMessage(HttpMethod.Get, url))
            {
                using (var client = _clientFactory.CreateClient(clientName))
                {
                    using (var response = await client.SendAsync(request))
                    {
                        if (response != null)
                        {
                            data= await response.Content.ReadAsStringAsync();
                        }                        
                    }
                }
            }
            return data;
        }
    }

The code for certificate handler is:

    public class CertificateRegistryHttpHandler : HttpClientHandler, IDisposable
        {
            private readonly IConfiguration _configuration;
            private readonly Dictionary<string, X509Certificate2> _certMap;
            private bool disposedValue;
            public CertificateRegistryHttpHandler(IConfiguration configuration) : base()
            {
                _configuration = configuration;
                _certMap = new Dictionary<string, X509Certificate2>() { { configuration["DefaultCertificateName"], new X509Certificate2(fileName: Path.Combine(configuration["CertificatePath"], configuration["DefaultCertificateName"]), password: _configuration[configuration["DefaultCertificateName"]]) } };
            SslProtocols = System.Security.Authentication.SslProtocols.Tls12;
            ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => { return true; };            
            }
    
            protected override async Task<HttpResponseMessage> SendAsync(
                    HttpRequestMessage request,
                    CancellationToken cancellationToken)
            {
                var appId = (string)Microsoft.AspNetCore.WebUtilities.QueryHelpers.ParseQuery(request.RequestUri.Query)["myID"];
                string certName = _configuration[appId];
                if (!string.IsNullOrEmpty(certName))
                {
                    string certPass = _configuration[certName];
                    if (!string.IsNullOrEmpty(certPass))
                    {
                        if (!_certMap.ContainsKey(certName))
                        {
                            string certPath = Path.Combine(_configuration["CertificatePath"], certName);
                            if (File.Exists(certPath))
                            {
                                _certMap.Add(certName, new X509Certificate2(fileName: certPath, password: certPass));
                                AddCertificate(certName);
                            }
                            else
                            {
                                AddCertificate(_configuration["DefaultCertificateName"]);
                            }
                        }
                        else
                        {
                            AddCertificate(certName);
                        }
    
                    }
                    else
                    {
                        AddCertificate(_configuration["DefaultCertificateName"]);
                    }
                }
                else
                {
                    AddCertificate(_configuration["DefaultCertificateName"]);
                }
                return await base.SendAsync(request, cancellationToken);
            }
            private void AddCertificate(string certName)
            {
                if (_certMap[certName]!=null && !ClientCertificates.Contains(_certMap[certName]))
                {
                    ClientCertificates.Clear();
                    ClientCertificates.Add(_certMap[certName]);
                }
            }
            protected override void Dispose(bool disposing)
            {
                if (!disposedValue)
                {
                    if (disposing)
                    {
                        _certMap.Values.All(x =>
                        {
                            x.Dispose();
                            return true;
                        });
                        _certMap.Clear();
                    }
                    base.Dispose();
                    disposedValue = true;
                }
            }
            public new void Dispose()
            {
                Dispose(disposing: true);
                GC.SuppressFinalize(this);
            }
        }
    }

I only receive error from Vault Client and not from AD Api client because AD services are hosted on azure stack itself so, it never makes any issue. Still, I thought to by pass https check for both of them.

Tried SSL protocols selected as TLS12 and none both but, no luck with any of the. These are the two exceptions coming regularly like a flood. Sometime they come for 1 hour or 2 hours and sometimes at different intervals. But ultimately impacting the production disastrously:

Exception 1

System.AggregateException: One or more errors occurred. (The SSL connection could not be established, see inner exception.) ---> System.Net.Http.HttpRequestException: The SSL connection could not be established, see inner exception. ---> System.ArgumentNullException: Value cannot be null. (Parameter 'value') at System.Collections.CollectionBase.OnValidate(Object value) at System.Security.Cryptography.X509Certificates.X509CertificateCollection.OnValidate(Object value) at System.Collections.CollectionBase.System.Collections.IList.Add(Object value) at System.Security.Cryptography.X509Certificates.X509CertificateCollection.AddRange(X509CertificateCollection value) at System.Net.Security.CertificateHelper.GetEligibleClientCertificate(X509CertificateCollection candidateCerts) at System.Net.Security.SslStream.UserCertSelectionCallbackWrapper(String targetHost, X509CertificateCollection localCertificates, X509Certificate remoteCertificate, String[] acceptableIssuers) at System.Net.Security.SecureChannel.AcquireClientCredentials(Byte[]& thumbPrint) at System.Net.Security.SecureChannel.GenerateToken(Byte[] input, Int32 offset, Int32 count, Byte[]& output) at System.Net.Security.SecureChannel.NextMessage(Byte[] incoming, Int32 offset, Int32 count) at System.Net.Security.SslStream.StartSendBlob(Byte[] incoming, Int32 count, AsyncProtocolRequest asyncRequest) at System.Net.Security.SslStream.ForceAuthentication(Boolean receiveFirst, Byte[] buffer, AsyncProtocolRequest asyncRequest) at System.Net.Security.SslStream.ProcessAuthentication(LazyAsyncResult lazyResult, CancellationToken cancellationToken) at System.Net.Security.SslStream.BeginAuthenticateAsClient(SslClientAuthenticationOptions sslClientAuthenticationOptions, CancellationToken cancellationToken, AsyncCallback asyncCallback, Object asyncState) at System.Net.Security.SslStream.<>c.b__65_0(SslClientAuthenticationOptions arg1, CancellationToken arg2, AsyncCallback callback, Object state) at System.Threading.Tasks.TaskFactory`1.FromAsyncImpl[TArg1,TArg2](Func`5 beginMethod, Func`2 endFunction, Action`1 endAction, TArg1 arg1, TArg2 arg2, Object state, TaskCreationOptions creationOptions) at System.Net.Security.SslStream.AuthenticateAsClientAsync(SslClientAuthenticationOptions sslClientAuthenticationOptions, CancellationToken cancellationToken) at System.Net.Http.ConnectHelper.EstablishSslConnectionAsyncCore(Stream stream, SslClientAuthenticationOptions sslOptions, CancellationToken cancellationToken) --- End of inner exception stack trace --- at System.Net.Http.ConnectHelper.EstablishSslConnectionAsyncCore(Stream stream, SslClientAuthenticationOptions sslOptions, CancellationToken cancellationToken) at System.Net.Http.HttpConnectionPool.ConnectAsync(HttpRequestMessage request, Boolean allowHttp2, CancellationToken cancellationToken) at System.Net.Http.HttpConnectionPool.CreateHttp11ConnectionAsync(HttpRequestMessage request, CancellationToken cancellationToken) at System.Net.Http.HttpConnectionPool.GetHttpConnectionAsync(HttpRequestMessage request, CancellationToken cancellationToken) at System.Net.Http.HttpConnectionPool.SendWithRetryAsync(HttpRequestMessage request, Boolean doRequestAuth, CancellationToken cancellationToken) at System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) at System.Net.Http.DiagnosticsHandler.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) at Middlewares.CertificateRegistryHttpHandler.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) at Microsoft.Extensions.Http.Logging.LoggingHttpMessageHandler.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) at Polly.CircuitBreaker.AsyncCircuitBreakerEngine.ImplementationAsync[TResult](Func`3 action, Context context, CancellationToken cancellationToken, Boolean continueOnCapturedContext, ExceptionPredicates shouldHandleExceptionPredicates, ResultPredicates`1 shouldHandleResultPredicates, ICircuitController`1 breakerController) at Polly.AsyncPolicy`1.ExecuteAsync(Func`3 action, Context context, CancellationToken cancellationToken, Boolean continueOnCapturedContext) at Polly.Wrap.AsyncPolicyWrapEngine.<>c__DisplayClass0_0`1.<b__0>d.MoveNext() --- End of stack trace from previous location where exception was thrown --- at Polly.Fallback.AsyncFallbackEngine.ImplementationAsync[TResult](Func`3 action, Context context, CancellationToken cancellationToken, ExceptionPredicates shouldHandleExceptionPredicates, ResultPredicates`1 shouldHandleResultPredicates, Func`3 onFallbackAsync, Func`4 fallbackAction, Boolean continueOnCapturedContext) at Polly.AsyncPolicy`1.ExecuteAsync(Func`3 action, Context context, CancellationToken cancellationToken, Boolean continueOnCapturedContext) at Polly.Wrap.AsyncPolicyWrapEngine.ImplementationAsync[TResult](Func`3 func, Context context, CancellationToken cancellationToken, Boolean continueOnCapturedContext, IAsyncPolicy`1 outerPolicy, IAsyncPolicy`1 innerPolicy) at Polly.AsyncPolicy`1.ExecuteAsync(Func`3 action, Context context, CancellationToken cancellationToken, Boolean continueOnCapturedContext) at Microsoft.Extensions.Http.PolicyHttpMessageHandler.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) at Microsoft.Extensions.Http.Logging.LoggingScopeHttpMessageHandler.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) at System.Net.Http.HttpClient.FinishSendAsyncBuffered(Task`1 sendTask, HttpRequestMessage request, CancellationTokenSource cts, Boolean disposeCts) at Services.HttpClient.Get(String url, String clientName) --- End of inner exception stack trace --- at System.Threading.Tasks.Task`1.GetResultCore(Boolean waitCompletionNotification) at Services.SerlializeService.GetData[T](String queryStringParam, String clientName) at Myproject.Vault.VaultResponse.RequestVault_Credentials(String objectName, String safe, String user) at VaultGateway.Controllers.VaultCredentialsController.Credentials(String safe, String objectname) at Microsoft.AspNetCore.Mvc.Infrastructure.ActionMethodExecutor.TaskOfIActionResultExecutor.Execute(IActionResultTypeMapper mapper, ObjectMethodExecutor executor, Object controller, Object[] arguments) at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.g__Awaited|12_0(ControllerActionInvoker invoker, ValueTask`1 actionResultValueTask) at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.g__Awaited|10_0(ControllerActionInvoker invoker, Task lastTask, State next, Scope scope, Object state, Boolean isCompleted) at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.Rethrow(ActionExecutedContextSealed context) at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.Next(State& next, Scope& scope, Object& state, Boolean& isCompleted) at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.InvokeInnerFilterAsync() --- End of stack trace from previous location where exception was thrown --- at Microsoft.AspNetCore.Mvc.Infrastructure.ResourceInvoker.g__Awaited|19_0(ResourceInvoker invoker, Task lastTask, State next, Scope scope, Object state, Boolean isCompleted) at Microsoft.AspNetCore.Mvc.Infrastructure.ResourceInvoker.g__Awaited|17_0(ResourceInvoker invoker, Task task, IDisposable scope) at Microsoft.AspNetCore.Routing.EndpointMiddleware.g__AwaitRequestTask|6_0(Endpoint endpoint, Task requestTask, ILogger logger) at Microsoft.AspNetCore.Authorization.AuthorizationMiddleware.Invoke(HttpContext context) at Microsoft.AspNetCore.Diagnostics.ExceptionHandlerMiddleware.g__Awaited|6_0(ExceptionHandlerMiddleware middleware, HttpContext context, Task task)

Exception 2:

System.AggregateException: One or more errors occurred. (The SSL connection could not be established, see inner exception.) ---> System.Net.Http.HttpRequestException: The SSL connection could not be established, see inner exception. ---> System.IndexOutOfRangeException: Index was outside the bounds of the array. at System.Collections.ArrayList.get_Item(Int32 index) at System.Security.Cryptography.X509Certificates.X509CertificateCollection.get_Item(Int32 index) at System.Security.Cryptography.X509Certificates.X509CertificateCollection.AddRange(X509CertificateCollection value) at System.Net.Security.CertificateHelper.GetEligibleClientCertificate(X509CertificateCollection candidateCerts) at System.Net.Security.SslStream.UserCertSelectionCallbackWrapper(String targetHost, X509CertificateCollection localCertificates, X509Certificate remoteCertificate, String[] acceptableIssuers) at System.Net.Security.SecureChannel.AcquireClientCredentials(Byte[]& thumbPrint) at System.Net.Security.SecureChannel.GenerateToken(Byte[] input, Int32 offset, Int32 count, Byte[]& output) at System.Net.Security.SecureChannel.NextMessage(Byte[] incoming, Int32 offset, Int32 count) at System.Net.Security.SslStream.StartSendBlob(Byte[] incoming, Int32 count, AsyncProtocolRequest asyncRequest) at System.Net.Security.SslStream.ForceAuthentication(Boolean receiveFirst, Byte[] buffer, AsyncProtocolRequest asyncRequest) at System.Net.Security.SslStream.ProcessAuthentication(LazyAsyncResult lazyResult, CancellationToken cancellationToken) at System.Net.Security.SslStream.BeginAuthenticateAsClient(SslClientAuthenticationOptions sslClientAuthenticationOptions, CancellationToken cancellationToken, AsyncCallback asyncCallback, Object asyncState) at System.Net.Security.SslStream.<>c.b__65_0(SslClientAuthenticationOptions arg1, CancellationToken arg2, AsyncCallback callback, Object state) at System.Threading.Tasks.TaskFactory`1.FromAsyncImpl[TArg1,TArg2](Func`5 beginMethod, Func`2 endFunction, Action`1 endAction, TArg1 arg1, TArg2 arg2, Object state, TaskCreationOptions creationOptions) at System.Net.Security.SslStream.AuthenticateAsClientAsync(SslClientAuthenticationOptions sslClientAuthenticationOptions, CancellationToken cancellationToken) at System.Net.Http.ConnectHelper.EstablishSslConnectionAsyncCore(Stream stream, SslClientAuthenticationOptions sslOptions, CancellationToken cancellationToken) --- End of inner exception stack trace --- at System.Net.Http.ConnectHelper.EstablishSslConnectionAsyncCore(Stream stream, SslClientAuthenticationOptions sslOptions, CancellationToken cancellationToken) at System.Net.Http.HttpConnectionPool.ConnectAsync(HttpRequestMessage request, Boolean allowHttp2, CancellationToken cancellationToken) at System.Net.Http.HttpConnectionPool.CreateHttp11ConnectionAsync(HttpRequestMessage request, CancellationToken cancellationToken) at System.Net.Http.HttpConnectionPool.GetHttpConnectionAsync(HttpRequestMessage request, CancellationToken cancellationToken) at System.Net.Http.HttpConnectionPool.SendWithRetryAsync(HttpRequestMessage request, Boolean doRequestAuth, CancellationToken cancellationToken) at System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) at System.Net.Http.DiagnosticsHandler.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) at Middlewares.CertificateRegistryHttpHandler.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) at Microsoft.Extensions.Http.Logging.LoggingHttpMessageHandler.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) at Polly.CircuitBreaker.AsyncCircuitBreakerEngine.ImplementationAsync[TResult](Func`3 action, Context context, CancellationToken cancellationToken, Boolean continueOnCapturedContext, ExceptionPredicates shouldHandleExceptionPredicates, ResultPredicates`1 shouldHandleResultPredicates, ICircuitController`1 breakerController) at Polly.AsyncPolicy`1.ExecuteAsync(Func`3 action, Context context, CancellationToken cancellationToken, Boolean continueOnCapturedContext) at Polly.Wrap.AsyncPolicyWrapEngine.<>c__DisplayClass0_0`1.<b__0>d.MoveNext() --- End of stack trace from previous location where exception was thrown --- at Polly.Fallback.AsyncFallbackEngine.ImplementationAsync[TResult](Func`3 action, Context context, CancellationToken cancellationToken, ExceptionPredicates shouldHandleExceptionPredicates, ResultPredicates`1 shouldHandleResultPredicates, Func`3 onFallbackAsync, Func`4 fallbackAction, Boolean continueOnCapturedContext) at Polly.AsyncPolicy`1.ExecuteAsync(Func`3 action, Context context, CancellationToken cancellationToken, Boolean continueOnCapturedContext) at Polly.Wrap.AsyncPolicyWrapEngine.ImplementationAsync[TResult](Func`3 func, Context context, CancellationToken cancellationToken, Boolean continueOnCapturedContext, IAsyncPolicy`1 outerPolicy, IAsyncPolicy`1 innerPolicy) at Polly.AsyncPolicy`1.ExecuteAsync(Func`3 action, Context context, CancellationToken cancellationToken, Boolean continueOnCapturedContext) at Microsoft.Extensions.Http.PolicyHttpMessageHandler.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) at Microsoft.Extensions.Http.Logging.LoggingScopeHttpMessageHandler.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) at System.Net.Http.HttpClient.FinishSendAsyncBuffered(Task`1 sendTask, HttpRequestMessage request, CancellationTokenSource cts, Boolean disposeCts) at Services.HttpClient.Get(String url, String clientName) --- End of inner exception stack trace --- at System.Threading.Tasks.Task`1.GetResultCore(Boolean waitCompletionNotification) at Services.SerlializeService.GetData[T](String queryStringParam, String clientName) at MyProject.Vault.VaultResponse.RequestVault_Credentials(String objectName, String safe, String user) at VaultGateway.Controllers.VaultCredentialsController.Credentials(String safe, String objectname) at Microsoft.AspNetCore.Mvc.Infrastructure.ActionMethodExecutor.TaskOfIActionResultExecutor.Execute(IActionResultTypeMapper mapper, ObjectMethodExecutor executor, Object controller, Object[] arguments) at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.g__Awaited|12_0(ControllerActionInvoker invoker, ValueTask`1 actionResultValueTask) at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.g__Awaited|10_0(ControllerActionInvoker invoker, Task lastTask, State next, Scope scope, Object state, Boolean isCompleted) at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.Rethrow(ActionExecutedContextSealed context) at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.Next(State& next, Scope& scope, Object& state, Boolean& isCompleted) at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.InvokeInnerFilterAsync() --- End of stack trace from previous location where exception was thrown --- at Microsoft.AspNetCore.Mvc.Infrastructure.ResourceInvoker.g__Awaited|19_0(ResourceInvoker invoker, Task lastTask, State next, Scope scope, Object state, Boolean isCompleted) at Microsoft.AspNetCore.Mvc.Infrastructure.ResourceInvoker.g__Awaited|17_0(ResourceInvoker invoker, Task task, IDisposable scope) at Microsoft.AspNetCore.Routing.EndpointMiddleware.g__AwaitRequestTask|6_0(Endpoint endpoint, Task requestTask, ILogger logger) at Microsoft.AspNetCore.Authorization.AuthorizationMiddleware.Invoke(HttpContext context) at Microsoft.AspNetCore.Diagnostics.ExceptionHandlerMiddleware.g__Awaited|6_0(ExceptionHandlerMiddleware middleware, HttpContext context, Task task)

I would be grateful, if someone can suggest me the correct procedure to bypass this validation as we’re facing this issue since 1 month leads to lot of challenges in the production.

Regards