Fix flaky tests: realistic timeouts and robust assertions

src/Build.UnitTests/BackEnd/TaskBuilder_Tests.cs

@@ -167,7 +167,7 @@ public void CanceledTasksDoNotLogMSB4181()
                 manager.BeginBuild(_parameters);
                 BuildSubmission asyncResult = manager.PendBuildRequest(data);
                 asyncResult.ExecuteAsync(null, null);
-                int timeoutMilliseconds = 2000;
+                int timeoutMilliseconds = 30_000;
                 bool isCommandExecuted = waitCommandExecuted.WaitOne(timeoutMilliseconds);
                 manager.CancelAllSubmissions();
                 bool isSubmissionComplated = asyncResult.WaitHandle.WaitOne(timeoutMilliseconds);

src/Build.UnitTests/BackEnd/TaskHostFactory_Tests.cs

@@ -86,7 +86,7 @@ public void TaskNodesDieAfterBuild(bool taskHostFactorySpecified, bool envVariab
                     try
                     {
                         Process taskHostNode = Process.GetProcessById(pid);
-                        taskHostNode.WaitForExit(3000).ShouldBeTrue("The process with taskHostNode is still running.");
+                        taskHostNode.WaitForExit(15_000).ShouldBeTrue("The process with taskHostNode is still running.");
                     }
 
                     // We expect the TaskHostNode to exit quickly. If it exits before Process.GetProcessById, it will throw an ArgumentException.
@@ -122,13 +122,13 @@ public void TaskNodesDieAfterBuild(bool taskHostFactorySpecified, bool envVariab
                             // Process not ready yet
                         }
 
-                        Thread.Sleep(2000);
+                        Thread.Sleep(3000);
                         attempts++;
                         taskHostNode.Refresh();
                     }
 
                     // Now wait to ensure it stays alive
-                    bool processExited = taskHostNode.WaitForExit(3000);
+                    bool processExited = taskHostNode.WaitForExit(15_000);
 
                     processExited.ShouldBeFalse(
                         processExited

src/MSBuild.UnitTests/MSBuildServer_Tests.cs

@@ -114,7 +114,7 @@ public void MSBuildServerTest()
             {
                 _output.WriteLine($"The marker file {markerFile.Path} was created. The build task has been started. Ready to kill the server.");
                 // Kill the server
-                Process.GetProcessById(pidOfServerProcess).KillTree(1000);
+                Process.GetProcessById(pidOfServerProcess).KillTree(10_000);
                 _output.WriteLine($"The old server was killed.");
             };
             watcher.Filter = Path.GetFileName(markerFile.Path);
@@ -206,7 +206,7 @@ public void BuildsWhileBuildIsRunningOnServer()
 
             // The server will soon be in use; make sure we don't try to use it before that happens.
             _output.WriteLine("Waiting for the server to be in use.");
-            mre.WaitOne();
+            mre.WaitOne(60_000).ShouldBeTrue("Timed out waiting for marker file creation indicating server is busy.");
             _output.WriteLine("It's OK to go ahead.");
 
             Environment.SetEnvironmentVariable("MSBUILDUSESERVER", "0");
@@ -259,9 +259,7 @@ public void CanShutdownServerProcess(bool byBuildManager)
                 serverIsDown.ShouldBeTrue();
             }
 
-            serverProcess.WaitForExit(10_000);
-
-            serverProcess.HasExited.ShouldBeTrue();
+            serverProcess.WaitForExit(30_000).ShouldBeTrue("Server process did not exit within 30 seconds after shutdown request.");
         }
 
         [Fact]

src/Tasks.UnitTests/DownloadFile_Tests.cs

@@ -48,7 +48,7 @@ public void CanBeCanceled()
 
                 downloadFile.Cancel();
 
-                task.Wait(TimeSpan.FromMilliseconds(1500)).ShouldBeTrue();
+                task.Wait(TimeSpan.FromSeconds(10)).ShouldBeTrue("Download task should have completed within 10 seconds after cancellation.");
 
                 task.Result.ShouldBeFalse();
             }

src/Tasks.UnitTests/Exec_Tests.cs

@@ -166,27 +166,21 @@ public void Timeout()
             exec.Timeout = 5;
             bool result = exec.Execute();
 
-            Assert.False(result);
-            Assert.Equal(expectedExitCode, exec.ExitCode);
-            ((MockEngine)exec.BuildEngine).AssertLogContains("MSB5002");
-            int warningsCount = ((MockEngine)exec.BuildEngine).Warnings;
-            if (warningsCount == 1)
-            {
-                warningsCount.ShouldBe(1,
-                $"Expected 1 warning, encountered {warningsCount}: " + string.Join(",",
-                    ((MockEngine)exec.BuildEngine).WarningEvents.Select(w => w.Message)));
-            }
-            else
+            result.ShouldBeFalse();
+            exec.ExitCode.ShouldBe(expectedExitCode);
+            MockEngine mockEngine = (MockEngine)exec.BuildEngine;
+            mockEngine.AssertLogContains("MSB5002");
+            // At least the timeout warning (MSB5002). Virus checkers may cause additional MSB5018 temp file warnings.
+            mockEngine.Warnings.ShouldBeGreaterThanOrEqualTo(1,
+                $"Expected at least 1 warning, encountered {mockEngine.Warnings}: " + string.Join(",",
+                    mockEngine.WarningEvents.Select(w => w.Message)));
+            if (mockEngine.Warnings > 1)
             {
-                // Occasionally temp files fail to delete because of virus checkers, so generate MSB5018 warning
-                ((MockEngine)exec.BuildEngine).AssertLogContains("MSB5018");
-                warningsCount.ShouldBe(2,
-                $"Expected 2 warnings, encountered {warningsCount}: " + string.Join(",",
-                    ((MockEngine)exec.BuildEngine).WarningEvents.Select(w => w.Message)));
+                mockEngine.AssertLogContains("MSB5018");
             }
 
             // ToolTask does not log an error on timeout.
-            Assert.Equal(0, ((MockEngine)exec.BuildEngine).Errors);
+            mockEngine.Errors.ShouldBe(0);
         }
 
         [Fact]
@@ -200,7 +194,8 @@ public void TimeoutFailsEvenWhenExitCodeIsIgnored()
             result.ShouldBeFalse();
             MockEngine mockEngine = (MockEngine)exec.BuildEngine;
             mockEngine.AssertLogContains("MSB5002");
-            mockEngine.Warnings.ShouldBe(1);
+            // At least the timeout warning; virus checkers may cause additional MSB5018 temp file warnings.
+            mockEngine.Warnings.ShouldBeGreaterThanOrEqualTo(1);
 
             // ToolTask does not log an error on timeout.
             mockEngine.Errors.ShouldBe(0);

src/Tasks.UnitTests/ResourceHandling/GenerateResourceOutOfProc_Tests.cs

@@ -285,8 +285,11 @@ public void ForceOutOfDate()
                 t2.Sources = new ITaskItem[] { new TaskItem(resxFile) };
 
                 DateTime time = File.GetLastWriteTime(t.OutputResources[0].ItemSpec);
-                System.Threading.Thread.Sleep(200);
-                File.SetLastWriteTime(resxFile, DateTime.Now);
+                System.Threading.Thread.Sleep(500);
+                // Use a timestamp slightly in the past so that when ExecuteTask writes the output,
+                // its filesystem timestamp is guaranteed to be >= this value even on Linux ext4
+                // where nanosecond-precision timestamps can cause sub-millisecond ordering inversions.
+                File.SetLastWriteTime(resxFile, DateTime.Now.AddMilliseconds(-100));
 
                 Utilities.ExecuteTask(t2);
 
@@ -335,8 +338,11 @@ public void ForceOutOfDateLinked()
                 t2.Sources = new ITaskItem[] { new TaskItem(resxFile) };
 
                 DateTime time = File.GetLastWriteTime(t.OutputResources[0].ItemSpec);
-                System.Threading.Thread.Sleep(200);
-                File.SetLastWriteTime(bitmap, DateTime.Now);
+                System.Threading.Thread.Sleep(500);
+                // Use a timestamp slightly in the past so that when ExecuteTask writes the output,
+                // its filesystem timestamp is guaranteed to be >= this value even on Linux ext4
+                // where nanosecond-precision timestamps can cause sub-millisecond ordering inversions.
+                File.SetLastWriteTime(bitmap, DateTime.Now.AddMilliseconds(-100));
 
                 Utilities.ExecuteTask(t2);
 
@@ -389,9 +395,12 @@ public void ForceSomeOutOfDate()
                 t2.StateFile = new TaskItem(createResources.StateFile.ItemSpec);
                 t2.Sources = new ITaskItem[] { new TaskItem(firstResx), new TaskItem(secondResx) };
 
-                System.Threading.Thread.Sleep(200);
+                System.Threading.Thread.Sleep(500);
                 _output.WriteLine("Touch one input");
-                File.SetLastWriteTime(firstResx, DateTime.Now);
+                // Use a timestamp slightly in the past so that when ExecuteTask writes the output,
+                // its filesystem timestamp is guaranteed to be >= this value even on Linux ext4
+                // where nanosecond-precision timestamps can cause sub-millisecond ordering inversions.
+                File.SetLastWriteTime(firstResx, DateTime.Now.AddMilliseconds(-100));
 
                 Utilities.ExecuteTask(t2);
 
@@ -463,7 +472,7 @@ public void AllowLinkedNoGenerate()
                 t2.StateFile = new TaskItem(t.StateFile);
                 t2.Sources = new ITaskItem[] { new TaskItem(resxFile) };
 
-                System.Threading.Thread.Sleep(200);
+                System.Threading.Thread.Sleep(500);
 
                 Utilities.ExecuteTask(t2);
 
@@ -522,7 +531,7 @@ public void NothingOutOfDate()
 
                 DateTime time = File.GetLastWriteTime(t.OutputResources[0].ItemSpec);
                 DateTime time2 = File.GetLastWriteTime(t.OutputResources[1].ItemSpec);
-                System.Threading.Thread.Sleep(200);
+                System.Threading.Thread.Sleep(500);
 
                 Utilities.ExecuteTask(t2);
                 // Although everything was up to date, OutputResources and FilesWritten

src/Tasks.UnitTests/ResourceHandling/GenerateResource_Tests.cs

@@ -417,8 +417,11 @@ public void ForceOutOfDateLinked(bool usePreserialized)
                 t2.Sources = new ITaskItem[] { new TaskItem(resxFile) };
 
                 DateTime firstWriteTime = File.GetLastWriteTime(t.OutputResources[0].ItemSpec);
-                System.Threading.Thread.Sleep(200);
-                File.SetLastWriteTime(bitmap, DateTime.Now + TimeSpan.FromSeconds(2));
+                System.Threading.Thread.Sleep(500);
+                // Use a timestamp slightly in the past so that when ExecuteTask writes the output,
+                // its filesystem timestamp is guaranteed to be >= this value even on Linux ext4
+                // where nanosecond-precision timestamps can cause sub-millisecond ordering inversions.
+                File.SetLastWriteTime(bitmap, DateTime.Now.AddMilliseconds(-100));
 
                 Utilities.ExecuteTask(t2);
 
@@ -692,7 +695,7 @@ public void ForceSomeOutOfDate()
             t2.StateFile = new TaskItem(createResources.StateFile.ItemSpec);
             t2.Sources = new ITaskItem[] { new TaskItem(firstResx), new TaskItem(secondResx) };
 
-            System.Threading.Thread.Sleep(200);
+            System.Threading.Thread.Sleep(500);
             if (!NativeMethodsShared.IsWindows)
             {
                 // Must be > 1 sec on some file systems for proper timestamp granularity
@@ -701,10 +704,10 @@ public void ForceSomeOutOfDate()
             }
 
             _output.WriteLine("Touch one input");
-            File.SetLastWriteTime(firstResx, DateTime.Now);
-
-            // Increasing the space between the last write and task execution due to precision on file time
-            System.Threading.Thread.Sleep(1000);
+            // Use a timestamp slightly in the past so that when ExecuteTask writes the output,
+            // its filesystem timestamp is guaranteed to be >= this value even on Linux ext4
+            // where nanosecond-precision timestamps can cause sub-millisecond ordering inversions.
+            File.SetLastWriteTime(firstResx, DateTime.Now.AddMilliseconds(-100));
 
             Utilities.ExecuteTask(t2);
 
@@ -814,7 +817,7 @@ public void NothingOutOfDate()
 
                 DateTime time = File.GetLastWriteTime(t.OutputResources[0].ItemSpec);
                 DateTime time2 = File.GetLastWriteTime(t.OutputResources[1].ItemSpec);
-                System.Threading.Thread.Sleep(200);
+                System.Threading.Thread.Sleep(500);
 
                 Utilities.ExecuteTask(t2);
                 // Although everything was up to date, OutputResources and FilesWritten
@@ -3695,7 +3698,7 @@ GenerateResource ExecuteTask()
                 DateTime initialWriteTime = File.GetLastWriteTime(resourcesFile);
 
                 // fs granularity on HFS is 1 sec!
-                System.Threading.Thread.Sleep(NativeMethodsShared.IsOSX ? 1000 : 100);
+                System.Threading.Thread.Sleep(NativeMethodsShared.IsOSX ? 1500 : 500);
 
                 // Rebuild, it shouldn't regen .resources file since the sources
                 // haven't changed

src/Utilities.UnitTests/ToolTask_Tests.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Diagnostics;
+using System.Globalization;
 using System.IO;
 using System.Resources;
 using System.Text.RegularExpressions;
@@ -1011,8 +1012,10 @@ public void ToolTaskThatTimeoutAndRetry(int repeats, int initialDelay, int follo
         /// </remarks>
         private sealed class ToolTaskThatSleeps : ToolTask
         {
-            // Windows prompt command to sleep:
-            private readonly string _windowsSleep = "/c start /wait timeout {0}";
+            // Windows command to delay using ping to loopback (avoids interactive console requirement
+            // of the 'timeout' command and avoids 'start /wait' overhead of creating a new console window).
+            // ping -n N sends N echo requests at ~1 second intervals, so total delay is ~(N-1) seconds.
+            private readonly string _windowsSleep = "/c ping -n {0} 127.0.0.1 > nul";
 
             // UNIX command to sleep:
             private readonly string _unixSleep = "-c \"sleep {0}\"";
@@ -1062,10 +1065,19 @@ public ToolTaskThatSleeps()
             /// <summary>
             /// Generates a shell command to sleep different amount of time based on repeat counter.
             /// </summary>
-            protected override string GenerateCommandLineCommands() =>
-                NativeMethodsShared.IsUnixLike ?
-                string.Format(_unixSleep, RepeatCount < 2 ? InitialDelay / 1000.0 : FollowupDelay / 1000.0) :
-                string.Format(_windowsSleep, RepeatCount < 2 ? InitialDelay / 1000.0 : FollowupDelay / 1000.0);
+            protected override string GenerateCommandLineCommands()
+            {
+                double delaySec = (RepeatCount < 2 ? InitialDelay : FollowupDelay) / 1000.0;
+                if (NativeMethodsShared.IsUnixLike)
+                {
+                    return string.Format(CultureInfo.InvariantCulture, _unixSleep, delaySec);
+                }
+
+                // ping -n N sends N echo requests; delay between requests is ~1 second,
+                // so total delay is roughly (N-1) seconds.
+                int pingCount = Math.Max(1, (int)delaySec + 1);
+                return string.Format(CultureInfo.InvariantCulture, _windowsSleep, pingCount);
+            }
 
             /// <summary>
             /// Ensures that test parameters make sense.