hot_reload_suite.dart

// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'dart:math';

import 'package:_fe_analyzer_shared/src/util/relativize.dart' as fe_shared;
import 'package:args/args.dart';
import 'package:collection/collection.dart';
import 'package:dev_compiler/dev_compiler.dart' as ddc_names
    show libraryUriToJsIdentifier;
import 'package:front_end/src/compute_platform_binaries_location.dart' as fe;
import 'package:reload_test/ddc_helpers.dart' as ddc_helpers;
import 'package:reload_test/frontend_server_controller.dart';
import 'package:reload_test/hot_reload_memory_filesystem.dart';
import 'package:reload_test/test_helpers.dart';

// Set an arbitrary cap on generations.
final globalMaxGenerations = 100;

final testTimeoutSeconds = 10;

// The separator between a test file and its inlined diff.
//
// All contents after this separator are considered are diff comments.
final testDiffSeparator = '/** DIFF **/';

final argParser = ArgParser()
  ..addOption('runtime',
      abbr: 'r',
      defaultsTo: 'd8',
      allowed: RuntimePlatforms.values.map((v) => v.text),
      help: 'runtime platform used to run tests.')
  ..addOption('named-configuration',
      abbr: 'n',
      defaultsTo: 'no-configuration',
      help: 'configuration name to use for emitting test result files.')
  ..addOption('output-directory', help: 'directory to emit test results files.')
  ..addOption('diff',
      allowed: ['check', 'write', 'ignore'],
      allowedHelp: {
        'check': 'Validate that reload test diffs are generated and correct.',
        'write': 'Write diffs for reload tests.',
        'ignore': 'Ignore reload diffs.',
      },
      defaultsTo: 'check',
      help:
          'Selects whether test diffs should be checked, written, or ignored.')
  ..addFlag('debug',
      abbr: 'd',
      defaultsTo: false,
      negatable: true,
      help: 'enables additional debug behavior and logging.')
  ..addFlag('verbose',
      abbr: 'v',
      defaultsTo: true,
      negatable: true,
      help: 'enables verbose logging.');

late final bool verbose;
late final bool debug;

Future<void> main(List<String> args) async {
  final argResults = argParser.parse(args);
  final runtimePlatform =
      RuntimePlatforms.values.byName(argResults['runtime'] as String);
  debug = argResults['debug'] as bool;
  verbose = argResults['verbose'] as bool;

  // Used to communicate individual test failures to our test bots.
  final emitTestResultsJson = argResults['output-directory'] != null;
  final buildRootUri = fe.computePlatformBinariesLocation(forceBuildDir: true);
  // We can use the outline instead of the full SDK dill here.
  final ddcPlatformDillUri = buildRootUri.resolve('ddc_outline.dill');
  final vmPlatformDillUri = buildRootUri.resolve('vm_platform_strong.dill');

  final sdkRoot = Platform.script.resolve('../../../');
  final packageConfigUri = sdkRoot.resolve('.dart_tool/package_config.json');
  final allTestsUri = sdkRoot.resolve('tests/hot_reload/');
  final soundStableDartSdkJsUri =
      buildRootUri.resolve('gen/utils/ddc/stable/sdk/ddc/dart_sdk.js');
  final ddcModuleLoaderJsUri =
      sdkRoot.resolve('pkg/dev_compiler/lib/js/ddc/ddc_module_loader.js');
  final d8Config = ddc_helpers.D8Configuration(sdkRoot);

  // Contains generated code for all tests.
  final generatedCodeDir = Directory.systemTemp.createTempSync();
  final generatedCodeUri = generatedCodeDir.uri;
  _debugPrint('See generated hot reload framework code in $generatedCodeUri');

  // The snapshot directory is a staging area the test framework uses to
  // construct a compile-able test app across reload/restart generations.
  final snapshotDir = Directory.fromUri(generatedCodeUri.resolve('.snapshot/'));
  snapshotDir.createSync();
  final snapshotUri = snapshotDir.uri;

  // Contains files emitted from Frontend Server compiles and recompiles.
  final frontendServerEmittedFilesDirUri = generatedCodeUri.resolve('.fes/');
  Directory.fromUri(frontendServerEmittedFilesDirUri).createSync();
  final outputDillUri = frontendServerEmittedFilesDirUri.resolve('output.dill');
  final outputIncrementalDillUri =
      frontendServerEmittedFilesDirUri.resolve('output_incremental.dill');

  // TODO(markzipan): Support custom entrypoints.
  final snapshotEntrypointUri = snapshotUri.resolve('main.dart');
  final filesystemRootUri = snapshotUri;
  final filesystemScheme = 'hot-reload-test';
  final snapshotEntrypointLibraryName = fe_shared.relativizeUri(
      filesystemRootUri, snapshotEntrypointUri, fe_shared.isWindows);
  final snapshotEntrypointWithScheme =
      '$filesystemScheme:///$snapshotEntrypointLibraryName';

  _print('Initializing the Frontend Server.');
  HotReloadFrontendServerController controller;
  final commonArgs = [
    '--incremental',
    '--filesystem-root=${snapshotUri.toFilePath()}',
    '--filesystem-scheme=$filesystemScheme',
    '--output-dill=${outputDillUri.toFilePath()}',
    '--output-incremental-dill=${outputIncrementalDillUri.toFilePath()}',
    '--packages=${packageConfigUri.toFilePath()}',
    '--sdk-root=${sdkRoot.toFilePath()}',
    '--verbosity=${verbose ? 'all' : 'info'}',
  ];
  switch (runtimePlatform) {
    case RuntimePlatforms.d8:
      final ddcPlatformDillFromSdkRoot = fe_shared.relativizeUri(
          sdkRoot, ddcPlatformDillUri, fe_shared.isWindows);
      final fesArgs = [
        ...commonArgs,
        '--dartdevc-module-format=ddc',
        '--platform=$ddcPlatformDillFromSdkRoot',
        '--target=dartdevc',
      ];
      controller = HotReloadFrontendServerController(fesArgs);
      break;
    case RuntimePlatforms.chrome:
      throw Exception('Unsupported platform: $runtimePlatform');
    case RuntimePlatforms.vm:
      final vmPlatformDillFromSdkRoot = fe_shared.relativizeUri(
          sdkRoot, vmPlatformDillUri, fe_shared.isWindows);
      final fesArgs = [
        ...commonArgs,
        '--platform=$vmPlatformDillFromSdkRoot',
        '--target=vm',
      ];
      controller = HotReloadFrontendServerController(fesArgs);
      break;
  }
  controller.start();

  Future<void> shutdown() async {
    // Persist the temp directory for debugging.
    await controller.stop();
    _print('Frontend Server has shut down.');
    if (!debug) {
      generatedCodeDir.deleteSync(recursive: true);
    }
  }

  final testOutcomes = <TestResultOutcome>[];
  final validTestSourceName = RegExp(r'.*[a-zA-Z0-9]+.[0-9]+.dart');
  for (var testDir in Directory.fromUri(allTestsUri).listSync()) {
    if (testDir is! Directory) {
      if (testDir is File) {
        // Ignore Dart source files, which may be imported as helpers
        continue;
      }
      throw Exception('Non-directory or file entity found in '
          '${allTestsUri.toFilePath()}: $testDir');
    }
    final testDirParts = testDir.uri.pathSegments;
    final testName = testDirParts[testDirParts.length - 2];

    var outcome = TestResultOutcome(
      configuration: argResults['named-configuration'] as String,
      testName: testName,
    );
    var stopwatch = Stopwatch()..start();

    // Report results for this test's execution.
    Future<void> reportTestOutcome(String testOutput, bool testPassed) async {
      stopwatch.stop();
      outcome.elapsedTime = stopwatch.elapsed;
      outcome.testOutput = testOutput;
      outcome.matchedExpectations = testPassed;
      testOutcomes.add(outcome);
      if (testPassed) {
        _print('PASSED with:\n  $testOutput', label: testName);
      } else {
        _print('FAILED with:\n  $testOutput', label: testName);
      }
    }

    // Report results for this test's sources' diff validations.
    void reportDiffOutcome(Uri fileUri, String testOutput, bool testPassed) {
      final filePath = fileUri.path;
      var outcome = TestResultOutcome(
        configuration: argResults['named-configuration'] as String,
        testName: '$filePath-diff',
        testOutput: testOutput,
      );
      outcome.elapsedTime = stopwatch.elapsed;
      outcome.matchedExpectations = testPassed;
      testOutcomes.add(outcome);
      if (testPassed) {
        _debugPrint('PASSED (diff on $filePath) with:\n  $testOutput',
            label: testName);
      } else {
        _debugPrint('FAILED (diff on $filePath) with:\n  $testOutput',
            label: testName);
      }
    }

    final tempUri = generatedCodeUri.resolve('$testName/');
    Directory.fromUri(tempUri).createSync();

    _print('Generating test assets.', label: testName);
    _debugPrint('Emitting JS code to ${tempUri.toFilePath()}.',
        label: testName);

    var filesystem = HotReloadMemoryFilesystem(tempUri);

    // Perform checks on this test's files. Checks include:
    // 1) Count the number of generations and ensure they're capped.
    // 2) Validate or generate diffs if specified
    //
    // Assumes all files are named like '$name.$integer.dart', where 0 is the
    // first generation.
    //
    // TODO(markzipan): Account for subdirectories.
    var maxGenerations = 0;
    late ReloadTestConfiguration testConfig;
    // All files in this test clustered by file name - in generation order.
    final filesByGeneration = <String, PriorityQueue<(int, Uri)>>{};

    for (final file in testDir.listSync()) {
      if (file is File) {
        final fileName = file.uri.pathSegments.last;
        // Process config files.
        if (fileName == 'config.json') {
          testConfig = ReloadTestConfiguration.fromJsonFile(file.uri);
        } else if (fileName.endsWith('.dart')) {
          if (!validTestSourceName.hasMatch(fileName)) {
            throw Exception('Invalid test source file name: $fileName\n'
                'Valid names look like "file_name.10.dart".');
          }
          final strippedName =
              fileName.substring(0, fileName.length - '.dart'.length);
          final generationIndex = strippedName.lastIndexOf('.');
          final generationId =
              int.parse(strippedName.substring(generationIndex + 1));
          maxGenerations = max(maxGenerations, generationId);
          final basename = strippedName.substring(0, generationIndex);
          filesByGeneration
              .putIfAbsent(
                  basename,
                  () => PriorityQueue(
                      ((int, Uri) a, (int, Uri) b) => a.$1 - b.$1))
              .add((generationId, file.uri));
        }
      }
    }
    if (maxGenerations > globalMaxGenerations) {
      throw Exception('Too many generations specified in test '
          '(requested: $maxGenerations, max: $globalMaxGenerations).');
    }

    var diffMode = argResults['diff']!;
    if (fe_shared.isWindows && diffMode != 'ignore') {
      _print("Diffing isn't supported on Windows. Defaulting to 'ignore'.",
          label: testName);
      diffMode = 'ignore';
    }
    switch (diffMode) {
      case 'check':
        _print('Checking source file diffs.', label: testName);
        filesByGeneration.forEach((basename, filesQueue) {
          final files = filesQueue.toList();
          _debugPrint('Checking source file diffs for $files.',
              label: testName);
          files.forEachIndexed((i, (int, Uri) element) {
            var (_, file) = element;
            if (i == 0) {
              // Check that the first file does not have a diff.
              !File.fromUri(file).readAsStringSync().contains(testDiffSeparator)
                  ? reportDiffOutcome(
                      file, 'First generation does not have a diff', true)
                  : reportDiffOutcome(file,
                      'First generation should not have any diffs', false);
            } else {
              // Check that exactly one diff exists.
              final currentText = File.fromUri(file).readAsStringSync();
              final diffCount =
                  testDiffSeparator.allMatches(currentText).length;
              if (diffCount == 0) {
                reportDiffOutcome(file, 'No diff found for $file', false);
                return;
              }
              if (diffCount > 1) {
                reportDiffOutcome(
                    file, 'Too many diffs found for $file (expected 1)', false);
                return;
              }
              // Check that the diff is properly generated.
              final (_, previousFile) = files[i - 1];
              final (previousCode, _) = _splitTestByDiff(previousFile);
              final (currentCode, currentDiff) = _splitTestByDiff(file);
              // 'main' is allowed to have empty diffs since the first
              // generation must be specified.
              if (basename != 'main' && previousCode == currentCode) {
                // TODO(markzipan): Should we make this an error?
                _print(
                    'Extraneous file detected. $file is identical to '
                    '$previousFile and can be removed.',
                    label: testName);
              }
              final previousTempUri = generatedCodeUri.resolve('__previous');
              final currentTempUri = generatedCodeUri.resolve('__current');
              File.fromUri(previousTempUri).writeAsStringSync(previousCode);
              File.fromUri(currentTempUri).writeAsStringSync(currentCode);
              final diffOutput = _diffWithFileUris(
                  previousTempUri, currentTempUri,
                  label: testName);
              File.fromUri(previousTempUri).deleteSync();
              File.fromUri(currentTempUri).deleteSync();
              if (diffOutput != currentDiff) {
                reportDiffOutcome(
                    file,
                    'Unexpected diff found for $file:\n'
                    '-- Expected --\n$diffOutput\n'
                    '-- Actual --\n$currentDiff',
                    false);
                return;
              }
              reportDiffOutcome(file, 'Correct diff found for $file', true);
              return;
            }
          });
        });
        break;
      case 'write':
        _print('Generating source file diffs.', label: testName);
        filesByGeneration.forEach((basename, filesQueue) {
          final files = filesQueue.toList();
          _debugPrint('Generating source file diffs for $files.',
              label: testName);
          files.forEachIndexed((i, (int, Uri) element) {
            final (_, file) = element;
            final (currentCode, currentDiff) = _splitTestByDiff(file);
            // Don't generate a diff for the first file of any generation,
            // and delete any diffs encountered.
            if (i == 0) {
              if (currentDiff.isNotEmpty) {
                _print('Removing extraneous diff from $file', label: testName);
                File.fromUri(file).writeAsStringSync(currentCode);
              }
              return;
            }
            final (_, previousFile) = files[i - 1];
            final (previousCode, _) = _splitTestByDiff(previousFile);
            final previousTempUri = generatedCodeUri.resolve('__previous');
            final currentTempUri = generatedCodeUri.resolve('__current');
            File.fromUri(previousTempUri).writeAsStringSync(previousCode);
            File.fromUri(currentTempUri).writeAsStringSync(currentCode);
            final diffOutput = _diffWithFileUris(
                previousTempUri, currentTempUri,
                label: testName);
            File.fromUri(previousTempUri).deleteSync();
            File.fromUri(currentTempUri).deleteSync();
            final newCurrentText =
                '$currentCode${currentCode.endsWith('\n') ? '' : '\n'}$diffOutput\n';
            File.fromUri(file).writeAsStringSync(newCurrentText);
            _print('Writing updated diff to $file', label: testName);
            _debugPrint('Updated diff:\n$diffOutput', label: testName);
            reportDiffOutcome(file, 'diff updated for $file', true);
          });
        });
        break;
      case 'ignore':
        _print('Ignoring source file diffs.', label: testName);
        filesByGeneration.forEach((basename, filesQueue) {
          filesQueue.unorderedElements.forEach(((int, Uri) element) {
            final (_, file) = element;
            reportDiffOutcome(file, 'Ignoring diff for $file', true);
          });
        });
        break;
    }

    // Skip this test directory if this platform is excluded.
    if (testConfig.excludedPlaforms.contains(runtimePlatform)) {
      _print('Skipping test on platform: ${runtimePlatform.text}',
          label: testName);
      continue;
    }

    // TODO(markzipan): replace this with a test-configurable main entrypoint.
    final mainDartFilePath = testDir.uri.resolve('main.dart').toFilePath();
    _debugPrint('Test entrypoint: $mainDartFilePath', label: testName);
    _print('Generating code over ${maxGenerations + 1} generations.',
        label: testName);

    var hasCompileError = false;
    // Generate hot reload/restart generations as subdirectories in a loop.
    var currentGeneration = 0;
    while (currentGeneration <= maxGenerations) {
      _debugPrint('Entering generation $currentGeneration', label: testName);
      var updatedFilesInCurrentGeneration = <String>[];

      // Copy all files in this generation to the snapshot directory with their
      // names restored (e.g., path/to/main' from 'path/to/main.0.dart).
      // TODO(markzipan): support subdirectories.
      _debugPrint(
          'Copying Dart files to snapshot directory: '
          '${snapshotUri.toFilePath()}',
          label: testName);
      for (var file in testDir.listSync()) {
        // Convert a name like `/path/foo.bar.25.dart` to `/path/foo.bar.dart`.
        if (file is File && file.path.endsWith('.dart')) {
          final baseName = file.uri.pathSegments.last;
          final parts = baseName.split('.');
          final generationId = int.parse(parts[parts.length - 2]);
          if (generationId == currentGeneration) {
            // Reconstruct the name of the file without generation indicators.
            parts.removeLast(); // Remove `.dart`.
            parts.removeLast(); // Remove the generation id.
            parts.add('.dart'); // Re-add `.dart`.
            final restoredName = parts.join();
            final fileSnapshotUri = snapshotUri.resolve(restoredName);
            final relativeSnapshotPath = fe_shared.relativizeUri(
                filesystemRootUri, fileSnapshotUri, fe_shared.isWindows);
            final snapshotPathWithScheme =
                '$filesystemScheme:///$relativeSnapshotPath';
            updatedFilesInCurrentGeneration.add(snapshotPathWithScheme);
            file.copySync(fileSnapshotUri.toFilePath());
          }
        }
      }
      _print(
          'Updated files in generation $currentGeneration: '
          '[${updatedFilesInCurrentGeneration.join(', ')}]',
          label: testName);

      // The first generation calls `compile`, but subsequent ones call
      // `recompile`.
      // Likewise, use the incremental output directory for `recompile` calls.
      String outputDirectoryPath;
      _print(
          'Compiling generation $currentGeneration with the Frontend Server.',
          label: testName);
      CompilerOutput compilerOutput;
      if (currentGeneration == 0) {
        _debugPrint(
            'Compiling snapshot entrypoint: $snapshotEntrypointWithScheme',
            label: testName);
        outputDirectoryPath = outputDillUri.toFilePath();
        compilerOutput =
            await controller.sendCompile(snapshotEntrypointWithScheme);
      } else {
        _debugPrint(
            'Recompiling snapshot entrypoint: $snapshotEntrypointWithScheme',
            label: testName);
        outputDirectoryPath = outputIncrementalDillUri.toFilePath();
        // TODO(markzipan): Add logic to reject bad compiles.
        compilerOutput = await controller.sendRecompile(
            snapshotEntrypointWithScheme,
            invalidatedFiles: updatedFilesInCurrentGeneration);
      }
      // Frontend Server reported compile errors. Fail if they weren't
      // expected, and do not run tests.
      if (compilerOutput.errorCount > 0) {
        hasCompileError = true;
        await controller.sendReject();
        // TODO(markzipan): Determine if 'contains' is good enough to determine
        // compilation error correctness.
        if (testConfig.expectedError != null &&
            compilerOutput.outputText.contains(testConfig.expectedError!)) {
          await reportTestOutcome(
              'Expected error found during compilation: '
              '${testConfig.expectedError}',
              true);
        } else {
          await reportTestOutcome(
              'Test failed with compile error: ${compilerOutput.outputText}',
              false);
        }
      } else {
        controller.sendAccept();
      }

      // Stop processing further generations if compilation failed.
      if (hasCompileError) break;

      _debugPrint(
          'Frontend Server successfully compiled outputs to: '
          '$outputDirectoryPath',
          label: testName);

      if (runtimePlatform.emitsJS) {
        // Update the memory filesystem with the newly-created JS files
        _print(
            'Loading generation $currentGeneration files '
            'into the memory filesystem.',
            label: testName);
        final codeFile = File('$outputDirectoryPath.sources');
        final manifestFile = File('$outputDirectoryPath.json');
        final sourcemapFile = File('$outputDirectoryPath.map');
        filesystem.update(
          codeFile,
          manifestFile,
          sourcemapFile,
          generation: '$currentGeneration',
        );

        // Write JS files and sourcemaps to their respective generation.
        _print('Writing generation $currentGeneration assets.',
            label: testName);
        _debugPrint('Writing JS assets to ${tempUri.toFilePath()}',
            label: testName);
        filesystem.writeToDisk(tempUri, generation: '$currentGeneration');
      } else {
        final dillOutputDir =
            Directory.fromUri(tempUri.resolve('generation$currentGeneration'));
        dillOutputDir.createSync();
        final dillOutputUri = dillOutputDir.uri.resolve('$testName.dill');
        File(outputDirectoryPath).copySync(dillOutputUri.toFilePath());
        // Write dills their respective generation.
        _print('Writing generation $currentGeneration assets.',
            label: testName);
        _debugPrint('Writing dill to ${dillOutputUri.toFilePath()}',
            label: testName);
      }
      currentGeneration++;
    }

    // Skip to the next test and avoid execution if we encountered a
    // compilation error.
    if (hasCompileError) {
      _print('Did not emit all assets due to compilation error.',
          label: testName);
      continue;
    }

    _print('Finished emitting assets.', label: testName);

    final testOutputStreamController = StreamController<List<int>>();
    final testOutputBuffer = StringBuffer();
    testOutputStreamController.stream
        .transform(utf8.decoder)
        .listen(testOutputBuffer.write);
    var testPassed = false;
    switch (runtimePlatform) {
      case RuntimePlatforms.d8:
        // Run the compiled JS generations with D8.
        // TODO(markzipan): Add logic for evaluating with Chrome or the VM.
        _print('Creating D8 hot reload test suite.', label: testName);

        final d8Suite = D8SuiteRunner(
          config: d8Config,
          bootstrapJsUri: tempUri.resolve('generation0/bootstrap.js'),
          entrypointLibraryExportName:
              ddc_names.libraryUriToJsIdentifier(snapshotEntrypointUri),
          dartSdkJsUri: soundStableDartSdkJsUri,
          ddcModuleLoaderJsUri: ddcModuleLoaderJsUri,
          outputSink: IOSink(testOutputStreamController.sink),
        );
        await d8Suite.setupTest(
          testName: testName,
          scriptDescriptors: filesystem.scriptDescriptorForBootstrap,
          generationToModifiedFiles: filesystem.generationsToModifiedFilePaths,
        );
        final d8ExitCode = await d8Suite.runTest(testName: testName);
        testPassed = d8ExitCode == 0;
        await d8Suite.teardownTest(testName: testName);
        break;
      case RuntimePlatforms.chrome:
      case RuntimePlatforms.vm:
        final firstGenerationDillUri =
            tempUri.resolve('generation0/$testName.dill');
        // Start the VM at generation 0.
        final vmArgs = [
          '--enable-vm-service=0', // 0 avoids port collisions.
          '--disable-service-auth-codes',
          '--disable-dart-dev',
          firstGenerationDillUri.toFilePath(),
        ];
        final vm = await Process.start(Platform.executable, vmArgs);
        _debugPrint(
            'Starting VM with command: '
            '${Platform.executable} ${vmArgs.join(" ")}',
            label: testName);
        vm.stdout
            .transform(utf8.decoder)
            .transform(LineSplitter())
            .listen((String line) {
          _debugPrint('VM stdout: $line', label: testName);
          testOutputBuffer.writeln(line);
        });
        vm.stderr
            .transform(utf8.decoder)
            .transform(LineSplitter())
            .listen((String err) {
          _debugPrint('VM stderr: $err', label: testName);
          testOutputBuffer.writeln(err);
        });
        _print('Executing VM test.', label: testName);
        final vmExitCode = await vm.exitCode
            .timeout(Duration(seconds: testTimeoutSeconds), onTimeout: () {
          final timeoutText =
              'Test timed out after $testTimeoutSeconds seconds.';
          _print(timeoutText, label: testName);
          testOutputBuffer.writeln(timeoutText);
          vm.kill();
          return 1;
        });
        testPassed = vmExitCode == 0;
    }

    await reportTestOutcome(testOutputBuffer.toString(), testPassed);
  }

  await shutdown();
  _print('Testing complete.');

  if (emitTestResultsJson) {
    final testOutcomeResults = testOutcomes.map((o) => o.toRecordJson());
    final testOutcomeLogs = testOutcomes.map((o) => o.toLogJson());
    final testResultsOutputDir =
        Uri.directory(argResults['output-directory'] as String);
    _print('Saving test results to ${testResultsOutputDir.toFilePath()}.');

    // Test outputs must have one JSON blob per line and be newline-terminated.
    final testResultsUri = testResultsOutputDir.resolve('results.json');
    final testResultsSink = File.fromUri(testResultsUri).openWrite();
    testOutcomeResults.forEach(testResultsSink.writeln);
    await testResultsSink.flush();
    await testResultsSink.close();

    final testLogsUri = testResultsOutputDir.resolve('logs.json');
    if (Platform.isWindows) {
      // TODO(55297): Logs are disabled on windows until this but is fixed.
      _print('Logs are not written on Windows. '
          'See: https://github.com/dart-lang/sdk/issues/55297');
    } else {
      final testLogsSink = File.fromUri(testLogsUri).openWrite();
      testOutcomeLogs.forEach(testLogsSink.writeln);
      await testLogsSink.flush();
      await testLogsSink.close();
    }
    _print('Emitted logs to ${testResultsUri.toFilePath()} '
        'and ${testLogsUri.toFilePath()}.');
  }

  // Report failed tests.
  var failedTests =
      testOutcomes.where((outcome) => !outcome.matchedExpectations);
  if (failedTests.isNotEmpty) {
    print('Some tests failed:');
    failedTests.forEach((outcome) {
      print('${outcome.testName} failed with:\n  ${outcome.testOutput}');
    });
    exit(1);
  }
}

/// Runs the [command] with [args] in [environment].
///
/// Will echo the commands to the console before running them when running in
/// `verbose` mode.
Future<Process> startProcess(String name, String command, List<String> args,
    {Map<String, String> environment = const {},
    ProcessStartMode mode = ProcessStartMode.normal}) {
  if (verbose) {
    print('Running $name:\n$command ${args.join(' ')}\n');
    if (environment.isNotEmpty) {
      var environmentVariables =
          environment.entries.map((e) => '${e.key}: ${e.value}').join('\n');
      print('With environment:\n$environmentVariables\n');
    }
  }
  return Process.start(command, args, mode: mode, environment: environment);
}

/// Prints messages if 'verbose' mode is enabled.
void _print(String message, {String? label}) {
  if (verbose) {
    final labelText = label == null ? '' : '($label)';
    print('hot_reload_test$labelText: $message');
  }
}

/// Prints messages if 'debug' mode is enabled.
void _debugPrint(String message, {String? label}) {
  if (debug) {
    final labelText = label == null ? '' : '($label)';
    print('DEBUG$labelText: $message');
  }
}

/// Returns the diff'd output between two files.
///
/// These diffs are appended at the end of updated file generations for better
/// test readability.
///
/// If [commented] is set, the output will be wrapped in multiline comments
/// and the diff separator.
///
/// If [trimHeaders] is set, the leading '+++' and '---' file headers will be
/// removed.
String _diffWithFileUris(Uri file1, Uri file2,
    {String label = '', bool commented = true, bool trimHeaders = true}) {
  final file1Path = file1.toFilePath();
  final file2Path = file2.toFilePath();
  final diffArgs = [
    '-du',
    '--width=120',
    '--expand-tabs',
    file1Path,
    file2Path
  ];
  _debugPrint("Running diff with 'diff ${diffArgs.join(' ')}'.", label: label);
  final diffProcess = Process.runSync('diff', diffArgs);
  final errOutput = diffProcess.stderr as String;
  if (errOutput.isNotEmpty) {
    throw Exception('diff failed with:\n$errOutput');
  }
  var output = diffProcess.stdout as String;
  if (trimHeaders) {
    // Skip the first two lines.
    // TODO(markzipan): Add support for Windows-style line endings.
    output = output.split('\n').skip(2).join('\n');
  }
  return commented ? '$testDiffSeparator\n/*\n$output*/' : output;
}

/// Returns the code and diff portions of [file].
(String, String) _splitTestByDiff(Uri file) {
  final text = File.fromUri(file).readAsStringSync();
  final diffIndex = text.indexOf(testDiffSeparator);
  final diffSplitIndex = diffIndex == -1 ? text.length - 1 : diffIndex;
  final codeText = text.substring(0, diffSplitIndex);
  final diffText = text.substring(diffSplitIndex, text.length - 1);
  // Avoid 'No newline at end of file' messages in the output by appending a
  // newline if one is not already trailing.
  return ('$codeText${codeText.endsWith('\n') ? '' : '\n'}', diffText);
}

abstract class HotReloadSuiteRunner {
  final String entrypointModuleName;
  final String entrypointLibraryExportName;
  final Uri dartSdkJsUri;
  final Uri ddcModuleLoaderJsUri;
  final StreamSink<List<int>> outputSink;

  HotReloadSuiteRunner({
    required this.entrypointModuleName,
    required this.entrypointLibraryExportName,
    required this.dartSdkJsUri,
    required this.ddcModuleLoaderJsUri,
    required this.outputSink,
  });

  /// Logic that needs to be run before every test begins.
  ///
  /// [scriptDescriptors] and [generationToModifiedFiles] are only used for
  /// DDC-based execution environments.
  Future<void> setupTest(
      {String? testName,
      List<Map<String, String?>>? scriptDescriptors,
      Map<String, List<String>>? generationToModifiedFiles});

  /// Executes a test.
  Future<int> runTest({String? testName});

  /// Logic that needs to be run after every test completes.
  Future<void> teardownTest({String? testName});
}

class D8SuiteRunner implements HotReloadSuiteRunner {
  final ddc_helpers.D8Configuration config;
  final Uri bootstrapJsUri;
  @override
  final String entrypointModuleName;
  @override
  final String entrypointLibraryExportName;
  @override
  final Uri dartSdkJsUri;
  @override
  final Uri ddcModuleLoaderJsUri;
  @override
  final StreamSink<List<int>> outputSink;

  D8SuiteRunner._({
    required this.config,
    required this.bootstrapJsUri,
    required this.entrypointModuleName,
    required this.entrypointLibraryExportName,
    required this.dartSdkJsUri,
    required this.ddcModuleLoaderJsUri,
    required this.outputSink,
  });

  factory D8SuiteRunner({
    required ddc_helpers.D8Configuration config,
    required Uri bootstrapJsUri,
    String entrypointModuleName = 'main.dart',
    String entrypointLibraryExportName = 'main',
    required Uri dartSdkJsUri,
    required Uri ddcModuleLoaderJsUri,
    StreamSink<List<int>>? outputSink,
  }) {
    return D8SuiteRunner._(
      config: config,
      entrypointModuleName: entrypointModuleName,
      entrypointLibraryExportName: entrypointLibraryExportName,
      bootstrapJsUri: bootstrapJsUri,
      dartSdkJsUri: dartSdkJsUri,
      ddcModuleLoaderJsUri: ddcModuleLoaderJsUri,
      outputSink: outputSink ?? stdout,
    );
  }

  String _generateBootstrapper({
    required List<Map<String, String?>> scriptDescriptors,
    required Map<String, List<String>> generationToModifiedFiles,
  }) {
    return ddc_helpers.generateD8Bootstrapper(
      ddcModuleLoaderJsPath: escapedString(ddcModuleLoaderJsUri.toFilePath()),
      dartSdkJsPath: escapedString(dartSdkJsUri.toFilePath()),
      entrypointModuleName: escapedString(entrypointModuleName),
      entrypointLibraryExportName: escapedString(entrypointLibraryExportName),
      scriptDescriptors: scriptDescriptors,
      modifiedFilesPerGeneration: generationToModifiedFiles,
    );
  }

  @override
  Future<void> setupTest({
    String? testName,
    List<Map<String, String?>>? scriptDescriptors,
    Map<String, List<String>>? generationToModifiedFiles,
  }) async {
    _print('Preparing to run D8 test.', label: testName);
    if (scriptDescriptors == null || generationToModifiedFiles == null) {
      throw ArgumentError('D8SuiteRunner requires that "scriptDescriptors" '
          'and "generationToModifiedFiles" be provided during setup.');
    }
    final d8BootstrapJS = _generateBootstrapper(
        scriptDescriptors: scriptDescriptors,
        generationToModifiedFiles: generationToModifiedFiles);
    File.fromUri(bootstrapJsUri).writeAsStringSync(d8BootstrapJS);
    _debugPrint('Writing D8 bootstrapper: $bootstrapJsUri', label: testName);
  }

  @override
  Future<int> runTest({String? testName}) async {
    final process = await startProcess('D8', config.binary.toFilePath(), [
      config.sealNativeObjectScript.toFilePath(),
      config.preamblesScript.toFilePath(),
      bootstrapJsUri.toFilePath()
    ]);
    unawaited(process.stdout.pipe(outputSink));
    return process.exitCode;
  }

  @override
  Future<void> teardownTest({String? testName}) async {}
}