Skip to content

Add extra test assertions around tests #13536

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Youssef1313 merged 2 commits into from
Apr 14, 2026
Merged

Add extra test assertions around tests #13536

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1e26a4d
Add extra test assertions around tests
Youssef1313 Apr 14, 2026
aac0488
Attempt to simplify test
Youssef1313 Apr 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/Build.OM.UnitTests/Microsoft.Build.Engine.OM.UnitTests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

<AssemblyName>Microsoft.Build.Engine.OM.UnitTests</AssemblyName>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<DefineConstants>$(DefineConstants);IS_OM_TESTS</DefineConstants>
</PropertyGroup>

<PropertyGroup>
Expand Down
133 changes: 60 additions & 73 deletions src/Build.UnitTests/BinaryLogger_Tests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,95 +106,82 @@ public void TestBinaryLoggerRoundtrip(string projectText, BinlogRoundtripTestRep
// See logic around showTargetOutputs.
// In short, this controls whether or not the "Target output items:" part is shown.
// Traits.Instance is weird, it's not always a singleton, depending on whether or not BuildEnvironmentState.s_runningTests is true.
// The s_runningTests check is also not that straightforward.
// Calls to ResetInstance_ForUnitTestsOnly (by tests that ran earlier than this test) might end up setting s_runningTests to false.
// So the current approach is extremely vulnerable to the order of tests being run, and also for test filters.
// When s_runningTests is true, we don't use a singleton, and in this case, one logger shows target outputs while the other doesn't.
// When s_runningTests is false, Traits.Instance is truly a singleton and so both see the same value.
// To stabilize this specific test, we explicitly set EnableTargetOutputLogging to true so we guarantee that both loggers see the same value.
// Long-term, unifying s_runningTests to always be true in unit tests (or even removing it completely, if possible) is the best approach.
var initialEnableTargetOutputLogging = Traits.Instance.EnableTargetOutputLogging;
// In this case s_runningTests is true.
// When s_runningTests is true, we don't use a singleton, and in this case, what matters is the env variable value.
// So we set the env variable to 1 and clear at the end of test.
using var env = TestEnvironment.Create();
try
{
env.SetEnvironmentVariable("MSBUILDTARGETOUTPUTLOGGING", "1");
Traits.Instance.EnableTargetOutputLogging = true;
env.SetEnvironmentVariable("MSBUILDTARGETOUTPUTLOGGING", "1");

var binaryLogger = new BinaryLogger();
var binaryLogger = new BinaryLogger();

binaryLogger.Parameters = _logFile;
binaryLogger.Parameters = _logFile;

var mockLogFromBuild = new MockLogger();
var mockLogFromBuild = new MockLogger();

var parallelFromBuildText = new StringBuilder();
var parallelFromBuild = new ParallelConsoleLogger(Framework.LoggerVerbosity.Diagnostic, t => parallelFromBuildText.Append(t), colorSet: null, colorReset: null);
parallelFromBuild.Parameters = "NOPERFORMANCESUMMARY";
var parallelFromBuildText = new StringBuilder();
var parallelFromBuild = new ParallelConsoleLogger(Framework.LoggerVerbosity.Diagnostic, t => parallelFromBuildText.Append(t), colorSet: null, colorReset: null);
parallelFromBuild.Parameters = "NOPERFORMANCESUMMARY";

// build and log into binary logger, mock logger and parallel console loggers
// no logging on evaluation
using (ProjectCollection collection = new())
{
Project project = ObjectModelHelpers.CreateInMemoryProject(collection, projectText);
project.Build(new ILogger[] { binaryLogger, mockLogFromBuild, parallelFromBuild }).ShouldBeTrue();
}
// build and log into binary logger, mock logger and parallel console loggers
// no logging on evaluation
using (ProjectCollection collection = new())
{
Project project = ObjectModelHelpers.CreateInMemoryProject(collection, projectText);
project.Build(new ILogger[] { binaryLogger, mockLogFromBuild, parallelFromBuild }).ShouldBeTrue();
}

string fileToReplay;
switch (replayMode)
{
case BinlogRoundtripTestReplayMode.NoReplay:
fileToReplay = _logFile;
break;
case BinlogRoundtripTestReplayMode.Structured:
case BinlogRoundtripTestReplayMode.RawEvents:
string fileToReplay;
switch (replayMode)
{
case BinlogRoundtripTestReplayMode.NoReplay:
fileToReplay = _logFile;
break;
case BinlogRoundtripTestReplayMode.Structured:
case BinlogRoundtripTestReplayMode.RawEvents:
{
var logReader = new BinaryLogReplayEventSource();
fileToReplay = _env.ExpectFile(".binlog").Path;
if (replayMode == BinlogRoundtripTestReplayMode.Structured)
{
var logReader = new BinaryLogReplayEventSource();
fileToReplay = _env.ExpectFile(".binlog").Path;
if (replayMode == BinlogRoundtripTestReplayMode.Structured)
{
// need dummy handler to force structured replay
logReader.BuildFinished += (_, _) => { };
}

BinaryLogger outputBinlog = new BinaryLogger()
{
Parameters = fileToReplay
};
outputBinlog.Initialize(logReader);
logReader.Replay(_logFile);
outputBinlog.Shutdown();
// need dummy handler to force structured replay
logReader.BuildFinished += (_, _) => { };
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(replayMode), replayMode, null);
}

var mockLogFromPlayback = new MockLogger();
BinaryLogger outputBinlog = new BinaryLogger()
{
Parameters = fileToReplay
};
outputBinlog.Initialize(logReader);
logReader.Replay(_logFile);
outputBinlog.Shutdown();
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(replayMode), replayMode, null);
}

var parallelFromPlaybackText = new StringBuilder();
var parallelFromPlayback = new ParallelConsoleLogger(Framework.LoggerVerbosity.Diagnostic, t => parallelFromPlaybackText.Append(t), colorSet: null, colorReset: null);
parallelFromPlayback.Parameters = "NOPERFORMANCESUMMARY";
var mockLogFromPlayback = new MockLogger();

var binaryLogReader = new BinaryLogReplayEventSource();
mockLogFromPlayback.Initialize(binaryLogReader);
parallelFromPlayback.Initialize(binaryLogReader);
var parallelFromPlaybackText = new StringBuilder();
var parallelFromPlayback = new ParallelConsoleLogger(Framework.LoggerVerbosity.Diagnostic, t => parallelFromPlaybackText.Append(t), colorSet: null, colorReset: null);
parallelFromPlayback.Parameters = "NOPERFORMANCESUMMARY";

// read the binary log and replay into mockLogger2
binaryLogReader.Replay(fileToReplay);
mockLogFromPlayback.Shutdown();
parallelFromPlayback.Shutdown();
var binaryLogReader = new BinaryLogReplayEventSource();
mockLogFromPlayback.Initialize(binaryLogReader);
parallelFromPlayback.Initialize(binaryLogReader);

// the binlog will have more information than recorded by the text log
mockLogFromPlayback.FullLog.ShouldContainWithoutWhitespace(mockLogFromBuild.FullLog);
// read the binary log and replay into mockLogger2
binaryLogReader.Replay(fileToReplay);
mockLogFromPlayback.Shutdown();
parallelFromPlayback.Shutdown();

var parallelExpected = parallelFromBuildText.ToString();
var parallelActual = parallelFromPlaybackText.ToString();
// the binlog will have more information than recorded by the text log
mockLogFromPlayback.FullLog.ShouldContainWithoutWhitespace(mockLogFromBuild.FullLog);

parallelActual.ShouldContainWithoutWhitespace(parallelExpected);
}
finally
{
Traits.Instance.EnableTargetOutputLogging = initialEnableTargetOutputLogging;
}
var parallelExpected = parallelFromBuildText.ToString();
var parallelActual = parallelFromPlaybackText.ToString();

parallelActual.ShouldContainWithoutWhitespace(parallelExpected);
}

/// <summary>
Expand Down
45 changes: 36 additions & 9 deletions src/Shared/UnitTests/TestAssemblyInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,19 +119,46 @@ public MSBuildTestCase(IXunitTestCase wrapped)

public ValueTask<IReadOnlyCollection<IXunitTest>> CreateTests() => _wrapped.CreateTests();

public void PostInvoke() => _wrapped.PostInvoke();
public void PostInvoke()
{
Assert.True(BuildEnvironmentState.s_runningTests);
_wrapped.PostInvoke();
Assert.True(BuildEnvironmentState.s_runningTests);
}

public void PreInvoke()
{
Assert.True(BuildEnvironmentState.s_runningTests);
_wrapped.PreInvoke();
Assert.True(BuildEnvironmentState.s_runningTests);
Comment thread
Youssef1313 marked this conversation as resolved.
}

public void PreInvoke() => _wrapped.PreInvoke();
public async ValueTask<RunSummary> Run(ExplicitOption explicitOption, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
{
using var _ = TestEnvironment.Create();
return await XunitRunnerHelper.RunXunitTestCase(
this,
messageBus,
cancellationTokenSource,
aggregator.Clone(),
explicitOption,
constructorArguments);
#if !IS_OM_TESTS
string initialHandshakeSalt = Framework.Traits.MSBuildNodeHandshakeSalt;
bool initialLogAllEnvVariables = Framework.Traits.LogAllEnvironmentVariables;
#endif
Assert.True(BuildEnvironmentState.s_runningTests);
try
{
return await XunitRunnerHelper.RunXunitTestCase(
this,
messageBus,
cancellationTokenSource,
aggregator,
Comment thread
Youssef1313 marked this conversation as resolved.
explicitOption,
constructorArguments);
}
finally
{
Assert.True(BuildEnvironmentState.s_runningTests);
#if !IS_OM_TESTS
Assert.Equal(initialHandshakeSalt, Framework.Traits.MSBuildNodeHandshakeSalt);
Assert.Equal(initialLogAllEnvVariables, Framework.Traits.LogAllEnvironmentVariables);
#endif
}
}

public void Serialize(IXunitSerializationInfo info)
Expand Down
Loading