runtime: [Discussion] Fail Tasks stuck in Wait() on ThreadPool when starvation occurs?

ThreadPool starvation can be problematic and will never recover if the rate of ThreadPool threads blocking is higher than the rate of ThreadPool thread injection.

Suggestion

If .Wait() and its associated permutations e.g. .Result etc are done on a ThreadPool thread; this could trigger the spawning of a “finalizer” type thread that fails (cancels?) the Tasks and releases the .Waits if ThreadPool starvation occurs.

The Tasks would need to register themselves before waiting if on the CurrentThread.IsThreadPoolThread so they are available to be cancelled and unblocked.

Would likely want to operate with some ThreadPool heuristics so it doesn’t trigger too easily when natural thread injection would resolve it.

Drawbacks

Tasks would start getting randomly cancelled if the ThreadPool starts getting starved which might be an unexpected behaviour; but may be better than entering an unrecoverable state?

Example registration code

Task.cs

+ private static Dictionary<SetOnInvokeMres, Task>? s_threadPoolWaitRegistrations;
+ private static object? s_registrationLock;
  
  private bool SpinThenBlockingWait(int millisecondsTimeout, CancellationToken cancellationToken)
  {
      bool infiniteWait = millisecondsTimeout == Timeout.Infinite;
      uint startTimeTicks = infiniteWait ? 0 : (uint)Environment.TickCount;
      bool returnValue = SpinWait(millisecondsTimeout);
      if (!returnValue)
      {
          var mres = new SetOnInvokeMres();
+         var isWaitRegistered = false;
          try
          {
              AddCompletionAction(mres, addBeforeOthers: true);
+             // If running on ThreadPool register this Task and Wait so its available
+             //  to be Failed/Cancelled if ThreadPool starvation occurs.
+             if (Thread.CurrentThread.IsThreadPoolThread)
+             {
+                 lock (ThreadPoolWaitRegistrationLock)
+                 {
+                     ThreadPoolWaitRegistrations.Add(mres, this);
+                     isWaitRegistered = true;
+                 }
+             }
  
              if (infiniteWait)
              {
                  returnValue = mres.Wait(Timeout.Infinite, cancellationToken);
              }
              else
              {
                  uint elapsedTimeTicks = ((uint)Environment.TickCount) - startTimeTicks;
                  if (elapsedTimeTicks < millisecondsTimeout)
                  {
                      returnValue = mres.Wait((int)(millisecondsTimeout - elapsedTimeTicks), cancellationToken);
                  }
              }
          }
          finally
          {
+             if (isWaitRegistered)
+             {
+                 lock (ThreadPoolWaitRegistrationLock)
+                 {
+                     // Remove the wait registration.
+                     ThreadPoolWaitRegistrations.Remove(mres);
+                 }
+             }
  
              if (!IsCompleted) RemoveContinuation(mres);
              // Don't Dispose of the MRES, because the continuation off of this task may
              // still be running.  This is ok, however, as we never access the MRES' WaitHandle,
              // and thus no finalizable resources are actually allocated.
          }
      }
      return returnValue;
  }

+ private static Dictionary<SetOnInvokeMres, Task> ThreadPoolWaitRegistrations
+     => s_threadPoolWaitRegistrations ?? CreateThreadPoolRegistrations();
+ 
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ private static Dictionary<SetOnInvokeMres, Task> CreateThreadPoolRegistrations()
+ {
+     Interlocked.CompareExchange(ref s_threadPoolWaitRegistrations, new Dictionary<SetOnInvokeMres, Task>(), null);
+     return s_threadPoolWaitRegistrations;
+ }
+ 
+ private static object ThreadPoolWaitRegistrationLock
+     => s_registrationLock ?? CreateThreadPoolWaitRegistrationLock();
+ 
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ private static object CreateThreadPoolWaitRegistrationLock()
+ {
+     Interlocked.CompareExchange(ref s_registrationLock, new object(), null);
+     return s_registrationLock;
+ }

/cc @stephentoub @davidfowl @jkotas @kouvel