AssemblyRunner can stay in executing forever depending on when Cancel is called

[gtcooke94]

I've been experimenting with making sure I can cancel AssemblyRunner, and I've run weirdness when cancelling the runner quickly after calling Start or quickly after it reaches AssemblyRunnerStatus.Executing.

I'm looking for best practices on making sure I can deterministically cancel and dispose of the runner.

1. This snippet will run OnExecutionComplete, but the state is forever Executing

using System;
using System.Reflection;
using Xunit.Runners;
using System.Threading;
using System.Threading.Tasks;
using System.Text;
using System.Collections.Generic;

namespace XUnitExploration
{
    public class XUnitRunner
    {
        object consoleLock = new object();

        ManualResetEvent finished = new ManualResetEvent(false);

        StringBuilder results = new StringBuilder();

        public XUnitRunner()
        {
        }

        public async Task<string> Run()
        {
            Console.WriteLine("Running");
            var testAssembly = Assembly.Load("ExtraXUnit");
            using var runner = AssemblyRunner.WithoutAppDomain(testAssembly.Location);
            runner.TestCaseFilter = (test) => {
                return true;
            };

            runner.OnDiscoveryComplete = OnDiscoveryComplete;
            runner.OnExecutionComplete = OnExecutionComplete;
            runner.OnTestFailed = OnTestFailed;
            runner.Start();
            runner.Cancel();

            finished.WaitOne();

            while (runner.Status != AssemblyRunnerStatus.Idle)
            {
                await Task.Delay(1000);
                Console.WriteLine(runner.Status);
            }
            finished.Dispose();

            return results.ToString();
        }

        void OnDiscoveryComplete(DiscoveryCompleteInfo info)
        {
            Console.WriteLine($"Running {info.TestCasesToRun} of {info.TestCasesDiscovered} tests...");
            results.AppendLine($"Running {info.TestCasesToRun} of {info.TestCasesDiscovered} tests...");
        }

        void OnExecutionComplete(ExecutionCompleteInfo info)
        {
            Console.WriteLine($"Finished: {info.TotalTests} tests in {info.ExecutionTime}s ({info.TestsFailed} failed, {info.TestsSkipped} skipped)");
            results.AppendLine($"Finished: {info.TotalTests} tests in {info.ExecutionTime}s ({info.TestsFailed} failed, {info.TestsSkipped} skipped)");
            finished.Set();
        }

        void OnTestFailed(TestFailedInfo info)
        {
            Console.WriteLine($"[FAIL] {info.TestDisplayName}: {info.ExceptionMessage}");
            results.AppendLine($"[FAIL] {info.TestDisplayName}: {info.ExceptionMessage}");
            if (info.ExceptionStackTrace != null)
            {
                Console.WriteLine(info.ExceptionStackTrace);
                results.AppendLine(info.ExceptionStackTrace);
            }
        }
    }
}

2. If we take a very similar snipped but wait for the Status to be Executing before calling Cancel, it sometimes never calls OnExecutionComplete, therefore the finished.WaitOne() is sometimes never completed.

using System;
using System.Reflection;
using Xunit.Runners;
using System.Threading;
using System.Threading.Tasks;
using System.Text;
using System.Collections.Generic;

namespace XUnitExploration
{
    public class XUnitRunner
    {
        object consoleLock = new object();

        ManualResetEvent finished = new ManualResetEvent(false);

        StringBuilder results = new StringBuilder();

        public XUnitRunner()
        {
        }

        public async Task<string> Run()
        {
            Console.WriteLine("Running");
            var testAssembly = Assembly.Load("ExtraXUnit");
            using var runner = AssemblyRunner.WithoutAppDomain(testAssembly.Location);
            runner.TestCaseFilter = (test) => {
                return true;
            };

            runner.OnDiscoveryComplete = OnDiscoveryComplete;
            runner.OnExecutionComplete = OnExecutionComplete;
            runner.OnTestFailed = OnTestFailed;
            runner.Start();

            while (runner.Status != AssemblyRunnerStatus.Executing)
            {
                await Task.Delay(10);
                Console.WriteLine(runner.Status);
            }

            runner.Cancel();

            finished.WaitOne();

            while (runner.Status != AssemblyRunnerStatus.Idle)
            {
                await Task.Delay(1000);
                Console.WriteLine(runner.Status);
            }
            finished.Dispose();

            return results.ToString();
        }

        void OnDiscoveryComplete(DiscoveryCompleteInfo info)
        {
            Console.WriteLine($"Running {info.TestCasesToRun} of {info.TestCasesDiscovered} tests...");
            results.AppendLine($"Running {info.TestCasesToRun} of {info.TestCasesDiscovered} tests...");
        }

        void OnExecutionComplete(ExecutionCompleteInfo info)
        {
            Console.WriteLine($"Finished: {info.TotalTests} tests in {info.ExecutionTime}s ({info.TestsFailed} failed, {info.TestsSkipped} skipped)");
            results.AppendLine($"Finished: {info.TotalTests} tests in {info.ExecutionTime}s ({info.TestsFailed} failed, {info.TestsSkipped} skipped)");
            finished.Set();
        }

        void OnTestFailed(TestFailedInfo info)
        {
            Console.WriteLine($"[FAIL] {info.TestDisplayName}: {info.ExceptionMessage}");
            results.AppendLine($"[FAIL] {info.TestDisplayName}: {info.ExceptionMessage}");
            if (info.ExceptionStackTrace != null)
            {
                Console.WriteLine(info.ExceptionStackTrace);
                results.AppendLine(info.ExceptionStackTrace);
            }
        }
    }
}