Skip to content

Commit f3b160f

Browse files
kotlarmilosCopilot
kotlarmilos
and
Copilot
authored
Add NativeAOT xunit test runner with threaded execution (#1560)
* Add NativeAOT xunit test runner with threaded execution Introduce NativeAotXunitTestRunner that extends ThreadlessXunitTestRunner with parallel test execution support and file-based result output. ThreadlessXunitTestRunner uses reflection-based discovery (NativeAOT-safe) but forces single-threaded execution. The new runner overrides the configuration to allow parallel test collections since NativeAOT has threads available. - Add NativeAotXunitTestRunner extending ThreadlessXunitTestRunner with configurable MaxParallelThreads and ParallelizeTestCollections - Extract virtual CreateConfiguration() and RunnerDisplayName from ThreadlessXunitTestRunner for extensibility - Revert PR #1554 changes to ThreadlessXunitTestRunner, keeping it WASM-only as originally intended - Update iOS and Android entry points to use NativeAotXunitTestRunner when RuntimeFeature.IsDynamicCodeSupported is false Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Refactor: extract CustomXunitTestRunner abstract base, rename to ReflectionBasedXunitTestRunner Address review feedback: - Rename NativeAotXunitTestRunner to ReflectionBasedXunitTestRunner - Extract shared reflection-based discovery logic into abstract CustomXunitTestRunner base class - Make ThreadlessXunitTestRunner and ReflectionBasedXunitTestRunner concrete implementations of CustomXunitTestRunner Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 09774ba commit f3b160f

5 files changed

Lines changed: 225 additions & 129 deletions

File tree

‎src/Microsoft.DotNet.XHarness.TestRunners.Xunit/AndroidApplicationEntryPoint.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5+
using System.Runtime.CompilerServices;
56
using Microsoft.DotNet.XHarness.TestRunners.Common;
67

78
#nullable enable
@@ -13,6 +14,13 @@ public abstract class AndroidApplicationEntryPoint : AndroidApplicationEntryPoin
1314

1415
protected override TestRunner GetTestRunner(LogWriter logWriter)
1516
{
17+
if (!RuntimeFeature.IsDynamicCodeSupported)
18+
{
19+
var reflectionRunner = new ReflectionBasedXunitTestRunner(logWriter) { MaxParallelThreads = MaxParallelThreads };
20+
ConfigureRunnerFilters(reflectionRunner, ApplicationOptions.Current);
21+
return reflectionRunner;
22+
}
23+
1624
var runner = new XUnitTestRunner(logWriter) { MaxParallelThreads = MaxParallelThreads };
1725
ConfigureRunnerFilters(runner, ApplicationOptions.Current);
1826
return runner;

‎src/Microsoft.DotNet.XHarness.TestRunners.Xunit/CustomXunitTestRunner.cs‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
using System.Collections.Generic;
7+
using System.Diagnostics;
8+
using System.Linq;
9+
using System.Threading.Tasks;
10+
using System.Xml.Linq;
11+
using Microsoft.DotNet.XHarness.TestRunners.Common;
12+
using Xunit;
13+
using Xunit.Abstractions;
14+
15+
#nullable enable
16+
namespace Microsoft.DotNet.XHarness.TestRunners.Xunit;
17+
18+
/// <summary>
19+
/// Abstract xunit test runner that uses reflection-based discovery (NativeAOT-safe).
20+
/// Concrete implementations define configuration (parallelism) and result output behavior.
21+
/// </summary>
22+
internal abstract class CustomXunitTestRunner : XunitTestRunnerBase
23+
{
24+
protected CustomXunitTestRunner(LogWriter logger) : base(logger)
25+
{
26+
ShowFailureInfos = false;
27+
}
28+
29+
protected abstract string RunnerDisplayName { get; }
30+
31+
private protected XElement? _assembliesElement;
32+
33+
internal XElement ConsumeAssembliesElement()
34+
{
35+
Debug.Assert(_assembliesElement != null, "ConsumeAssembliesElement called before Run() or after ConsumeAssembliesElement() was already called.");
36+
var res = _assembliesElement;
37+
_assembliesElement = null;
38+
FailureInfos.Clear();
39+
return res!;
40+
}
41+
42+
protected abstract TestAssemblyConfiguration CreateConfiguration();
43+
44+
public override async Task Run(IEnumerable<TestAssemblyInfo> testAssemblies)
45+
{
46+
OnInfo($"Using {RunnerDisplayName}");
47+
48+
_assembliesElement = new XElement("assemblies");
49+
50+
var configuration = CreateConfiguration();
51+
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
52+
var discoverySink = new TestDiscoverySink();
53+
var diagnosticSink = new ConsoleDiagnosticMessageSink(Logger);
54+
var testOptions = TestFrameworkOptions.ForExecution(configuration);
55+
var testSink = new TestMessageSink();
56+
57+
var totalSummary = new ExecutionSummary();
58+
foreach (var testAsmInfo in testAssemblies)
59+
{
60+
string assemblyFileName = testAsmInfo.FullPath;
61+
var controller = new YieldingXunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);
62+
63+
discoveryOptions.SetSynchronousMessageReporting(true);
64+
testOptions.SetSynchronousMessageReporting(true);
65+
66+
OnInfo($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})");
67+
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(testAsmInfo.Assembly);
68+
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
69+
70+
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
71+
var testCasesToRun = discoverySink.TestCases.Where(t => !_filters.IsExcluded(t)).ToList();
72+
OnInfo($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)");
73+
74+
var summaryTaskSource = new TaskCompletionSource<ExecutionSummary>();
75+
var resultsXmlAssembly = new XElement("assembly");
76+
#pragma warning disable CS0618 // Delegating*Sink types are marked obsolete, but we can't move to ExecutionSink yet: https://github.com/dotnet/arcade/issues/14375
77+
var resultsSink = new DelegatingXmlCreationSink(new DelegatingExecutionSummarySink(testSink), resultsXmlAssembly);
78+
#pragma warning restore
79+
var completionSink = new CompletionCallbackExecutionSink(resultsSink, summary => summaryTaskSource.SetResult(summary));
80+
81+
if (EnvironmentVariables.IsLogTestStart())
82+
{
83+
testSink.Execution.TestStartingEvent += args => { OnInfo($"[STRT] {args.Message.Test.DisplayName}"); };
84+
}
85+
testSink.Execution.TestPassedEvent += args =>
86+
{
87+
OnDebug($"[PASS] {args.Message.Test.DisplayName}");
88+
PassedTests++;
89+
};
90+
testSink.Execution.TestSkippedEvent += args =>
91+
{
92+
OnDebug($"[SKIP] {args.Message.Test.DisplayName}");
93+
SkippedTests++;
94+
};
95+
testSink.Execution.TestFailedEvent += args =>
96+
{
97+
OnError($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}");
98+
FailedTests++;
99+
};
100+
testSink.Execution.TestFinishedEvent += args => ExecutedTests++;
101+
102+
testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); };
103+
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); };
104+
105+
await controller.RunTestsAsync(testCasesToRun, MessageSinkAdapter.Wrap(completionSink), testOptions);
106+
107+
totalSummary = Combine(totalSummary, await summaryTaskSource.Task);
108+
109+
_assembliesElement.Add(resultsXmlAssembly);
110+
}
111+
TotalTests = totalSummary.Total;
112+
}
113+
114+
private ExecutionSummary Combine(ExecutionSummary aggregateSummary, ExecutionSummary assemblySummary)
115+
{
116+
return new ExecutionSummary
117+
{
118+
Total = aggregateSummary.Total + assemblySummary.Total,
119+
Failed = aggregateSummary.Failed + assemblySummary.Failed,
120+
Skipped = aggregateSummary.Skipped + assemblySummary.Skipped,
121+
Errors = aggregateSummary.Errors + assemblySummary.Errors,
122+
Time = aggregateSummary.Time + assemblySummary.Time
123+
};
124+
}
125+
}

‎src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ReflectionBasedXunitTestRunner.cs‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
using System.IO;
7+
using System.Threading.Tasks;
8+
using System.Xml;
9+
using Microsoft.DotNet.XHarness.Common;
10+
using Microsoft.DotNet.XHarness.TestRunners.Common;
11+
using Xunit;
12+
13+
#nullable enable
14+
namespace Microsoft.DotNet.XHarness.TestRunners.Xunit;
15+
16+
/// <summary>
17+
/// Xunit test runner using reflection-based discovery (NativeAOT-safe)
18+
/// with parallel test execution support and file-based result output.
19+
/// </summary>
20+
internal class ReflectionBasedXunitTestRunner : CustomXunitTestRunner
21+
{
22+
public int? MaxParallelThreads { get; set; }
23+
24+
public ReflectionBasedXunitTestRunner(LogWriter logger) : base(logger)
25+
{
26+
}
27+
28+
protected override string RunnerDisplayName => "reflection-based Xunit runner (threaded execution)";
29+
30+
private string _resultsFileName = "TestResults.xUnit.xml";
31+
protected override string ResultsFileName { get => _resultsFileName; set => _resultsFileName = value; }
32+
33+
protected override TestAssemblyConfiguration CreateConfiguration()
34+
{
35+
int maxThreads = MaxParallelThreads ?? Environment.ProcessorCount;
36+
return new TestAssemblyConfiguration()
37+
{
38+
ShadowCopy = false,
39+
ParallelizeAssembly = false,
40+
ParallelizeTestCollections = RunInParallel,
41+
MaxParallelThreads = RunInParallel ? maxThreads : 1,
42+
PreEnumerateTheories = false,
43+
};
44+
}
45+
46+
public override Task<string> WriteResultsToFile(XmlResultJargon xmlResultJargon)
47+
{
48+
if (_assembliesElement is null)
49+
return Task.FromResult(string.Empty);
50+
51+
string outputFilePath = GetResultsFilePath();
52+
var settings = new XmlWriterSettings { Indent = true };
53+
using (var xmlWriter = XmlWriter.Create(outputFilePath, settings))
54+
{
55+
_assembliesElement.Save(xmlWriter);
56+
}
57+
58+
return Task.FromResult(outputFilePath);
59+
}
60+
61+
public override Task WriteResultsToFile(TextWriter writer, XmlResultJargon jargon)
62+
{
63+
if (_assembliesElement is null)
64+
return Task.CompletedTask;
65+
66+
var settings = new XmlWriterSettings { Indent = true };
67+
using (var xmlWriter = XmlWriter.Create(writer, settings))
68+
{
69+
_assembliesElement.Save(xmlWriter);
70+
}
71+
72+
return Task.CompletedTask;
73+
}
74+
}

‎src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ThreadlessXunitTestRunner.cs‎

Lines changed: 15 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,10 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System;
6-
using System.Collections.Generic;
76
using System.Diagnostics;
87
using System.IO;
98
using System.Linq;
109
using System.Threading.Tasks;
11-
using System.Xml;
12-
using System.Xml.Linq;
1310
using Microsoft.DotNet.XHarness.Common;
1411
using Microsoft.DotNet.XHarness.TestRunners.Common;
1512
using Xunit;
@@ -18,146 +15,38 @@
1815
#nullable enable
1916
namespace Microsoft.DotNet.XHarness.TestRunners.Xunit;
2017

21-
internal class ThreadlessXunitTestRunner : XunitTestRunnerBase
18+
internal class ThreadlessXunitTestRunner : CustomXunitTestRunner
2219
{
2320
public ThreadlessXunitTestRunner(LogWriter logger) : base(logger)
2421
{
25-
ShowFailureInfos = false;
2622
}
2723

28-
private string _resultsFileName = "TestResults.xUnit.xml";
29-
protected override string ResultsFileName { get => _resultsFileName; set => _resultsFileName = value; }
24+
protected override string RunnerDisplayName => "threadless Xunit runner";
3025

31-
private XElement? _assembliesElement;
26+
protected override string ResultsFileName { get => string.Empty; set => throw new InvalidOperationException("This runner outputs its results to stdout."); }
3227

33-
internal XElement ConsumeAssembliesElement()
28+
protected override TestAssemblyConfiguration CreateConfiguration()
3429
{
35-
Debug.Assert(_assembliesElement != null, "ConsumeAssembliesElement called before Run() or after ConsumeAssembliesElement() was already called.");
36-
var res = _assembliesElement;
37-
_assembliesElement = null;
38-
FailureInfos.Clear();
39-
return res!;
40-
}
41-
42-
public override async Task Run(IEnumerable<TestAssemblyInfo> testAssemblies)
43-
{
44-
OnInfo("Using threadless Xunit runner");
45-
46-
_assembliesElement = new XElement("assemblies");
47-
48-
var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false };
49-
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
50-
var discoverySink = new TestDiscoverySink();
51-
var diagnosticSink = new ConsoleDiagnosticMessageSink(Logger);
52-
var testOptions = TestFrameworkOptions.ForExecution(configuration);
53-
var testSink = new TestMessageSink();
54-
55-
var totalSummary = new ExecutionSummary();
56-
foreach (var testAsmInfo in testAssemblies)
57-
{
58-
string assemblyFileName = testAsmInfo.FullPath;
59-
var controller = new YieldingXunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);
60-
61-
discoveryOptions.SetSynchronousMessageReporting(true);
62-
testOptions.SetSynchronousMessageReporting(true);
63-
64-
OnInfo($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})");
65-
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(testAsmInfo.Assembly);
66-
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
67-
68-
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
69-
var testCasesToRun = discoverySink.TestCases.Where(t => !_filters.IsExcluded(t)).ToList();
70-
OnInfo($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)");
71-
72-
var summaryTaskSource = new TaskCompletionSource<ExecutionSummary>();
73-
var resultsXmlAssembly = new XElement("assembly");
74-
#pragma warning disable CS0618 // Delegating*Sink types are marked obsolete, but we can't move to ExecutionSink yet: https://github.com/dotnet/arcade/issues/14375
75-
var resultsSink = new DelegatingXmlCreationSink(new DelegatingExecutionSummarySink(testSink), resultsXmlAssembly);
76-
#pragma warning restore
77-
var completionSink = new CompletionCallbackExecutionSink(resultsSink, summary => summaryTaskSource.SetResult(summary));
78-
79-
if (EnvironmentVariables.IsLogTestStart())
80-
{
81-
testSink.Execution.TestStartingEvent += args => { OnInfo($"[STRT] {args.Message.Test.DisplayName}"); };
82-
}
83-
testSink.Execution.TestPassedEvent += args =>
84-
{
85-
OnDebug($"[PASS] {args.Message.Test.DisplayName}");
86-
PassedTests++;
87-
};
88-
testSink.Execution.TestSkippedEvent += args =>
89-
{
90-
OnDebug($"[SKIP] {args.Message.Test.DisplayName}");
91-
SkippedTests++;
92-
};
93-
testSink.Execution.TestFailedEvent += args =>
94-
{
95-
OnError($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}");
96-
FailedTests++;
97-
};
98-
testSink.Execution.TestFinishedEvent += args => ExecutedTests++;
99-
100-
testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); };
101-
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); };
102-
103-
await controller.RunTestsAsync(testCasesToRun, MessageSinkAdapter.Wrap(completionSink), testOptions);
104-
105-
totalSummary = Combine(totalSummary, await summaryTaskSource.Task);
106-
107-
_assembliesElement.Add(resultsXmlAssembly);
108-
}
109-
TotalTests = totalSummary.Total;
110-
}
111-
112-
private ExecutionSummary Combine(ExecutionSummary aggregateSummary, ExecutionSummary assemblySummary)
113-
{
114-
return new ExecutionSummary
30+
return new TestAssemblyConfiguration()
11531
{
116-
Total = aggregateSummary.Total + assemblySummary.Total,
117-
Failed = aggregateSummary.Failed + assemblySummary.Failed,
118-
Skipped = aggregateSummary.Skipped + assemblySummary.Skipped,
119-
Errors = aggregateSummary.Errors + assemblySummary.Errors,
120-
Time = aggregateSummary.Time + assemblySummary.Time
32+
ShadowCopy = false,
33+
ParallelizeAssembly = false,
34+
ParallelizeTestCollections = false,
35+
MaxParallelThreads = 1,
36+
PreEnumerateTheories = false,
12137
};
12238
}
12339

12440
public override async Task<string> WriteResultsToFile(XmlResultJargon xmlResultJargon)
12541
{
126-
if (_assembliesElement is null)
127-
return string.Empty;
128-
129-
if (OperatingSystem.IsBrowser())
130-
{
131-
Debug.Assert(xmlResultJargon == XmlResultJargon.xUnit);
132-
await WriteResultsToFile(Console.Out, xmlResultJargon);
133-
return "";
134-
}
135-
136-
string outputFilePath = GetResultsFilePath();
137-
var settings = new XmlWriterSettings { Indent = true };
138-
using (var xmlWriter = XmlWriter.Create(outputFilePath, settings))
139-
{
140-
_assembliesElement.Save(xmlWriter);
141-
}
142-
143-
return outputFilePath;
42+
Debug.Assert(xmlResultJargon == XmlResultJargon.xUnit);
43+
await WriteResultsToFile(Console.Out, xmlResultJargon);
44+
return "";
14445
}
14546

146-
public override Task WriteResultsToFile(TextWriter writer, XmlResultJargon jargon)
47+
public override async Task WriteResultsToFile(TextWriter writer, XmlResultJargon jargon)
14748
{
148-
if (_assembliesElement is null)
149-
return Task.CompletedTask;
150-
151-
if (OperatingSystem.IsBrowser())
152-
return WasmXmlResultWriter.WriteResultsToFile(ConsumeAssembliesElement());
153-
154-
var settings = new XmlWriterSettings { Indent = true };
155-
using (var xmlWriter = XmlWriter.Create(writer, settings))
156-
{
157-
_assembliesElement.Save(xmlWriter);
158-
}
159-
160-
return Task.CompletedTask;
49+
await WasmXmlResultWriter.WriteResultsToFile(ConsumeAssembliesElement());
16150
}
16251
}
16352

0 commit comments

Comments
 (0)