[6.0] Port Flaky Test Separation

build.proj

@@ -43,6 +43,29 @@
     <NugetPackProperties Condition="'$(TF_BUILD)' == 'true'">$(NugetPackProperties);ContinuousIntegrationBuild=true;</NugetPackProperties>
   </PropertyGroup>
 
+  <!-- Test configuration properties -->
+  <PropertyGroup>
+    <!-- Set up default filters for tests to exclude failing/flaky/interactive tests -->
+    <Filter Condition="'$(Filter)' == ''">category!=failing&amp;category!=flaky&amp;category!=interactive</Filter>
+    <FilterArgument>--filter "$(Filter)"</FilterArgument>
+
+    <!-- Collect code coverage unless explicitly disabled -->
+    <CollectCodeCoverage Condition="'$(CollectCodeCoverage)' == ''">true</CollectCodeCoverage>
+    <CodeCoverageRunSettings>$(TestsPath)/tools/Microsoft.Data.SqlClient.TestUtilities/CodeCoverage.runsettings</CodeCoverageRunSettings>
+    <CollectArgument Condition="'$(CollectCodeCoverate)' == 'true'">
+      --collect "Code coverage"
+      --settings "$(CodeCoverageRunSettings)"
+    </CollectArgument>
+
+    <!-- Set up tests to fail after hangs and report via blame -->
+    <BlameArgument>$(Blame)</BlameArgument>
+    <BlameArgument Condition="'$(BlameArgument)' == ''">
+      --blame-hang
+      --blame-hang-dump-tup full
+      --blame-hang-timeout 10m
+    </BlameArgument>
+  </PropertyGroup>
+
   <!-- Release Build properties must be turned on for Release purposes, and turned off for Code Coverage calculations -->
   <PropertyGroup>
     <BuildForRelease Condition="$(BuildForRelease) == ''">true</BuildForRelease>
@@ -178,18 +201,101 @@
 
   <!-- Tests -->
   <Target Name="RunTests" DependsOnTargets="RunFunctionalTests;RunManualTests"/>
-  <Target Name="RunFunctionalTests">
-    <!-- Windows -->
-    <Exec ConsoleToMsBuild="true" Command="$(DotnetPath)dotnet test &quot;@(FunctionalTestsProj)&quot; -p:Configuration=$(Configuration) -p:Target$(TFGroup)Version=$(TF) -p:ReferenceType=$(ReferenceType) --no-build -v n --collect &quot;Code coverage&quot; -p:TestSet=$(TestSet) --results-directory $(ResultsDirectory) -p:TestTargetOS=Windows$(TargetGroup) --filter &quot;category!=non$(TargetGroup)tests&amp;category!=failing&amp;category!=nonwindowstests&quot; &quot;--logger:trx;LogFilePrefix=Functional-Windows$(TargetGroup)-$(TestSet)&quot;" Condition="'$(IsEnabledWindows)' == 'true'"/>
-    <!-- Unix -->
-    <Exec ConsoleToMsBuild="true" Command="$(DotnetPath)dotnet test &quot;@(FunctionalTestsProj)&quot; -p:Configuration=$(Configuration) -p:TargetNetCoreVersion=$(TF) -p:ReferenceType=$(ReferenceType) --no-build -v n -p:TestSet=$(TestSet) --results-directory $(ResultsDirectory) -p:TestTargetOS=Unixnetcoreapp --collect &quot;Code coverage&quot; --filter &quot;category!=nonnetcoreapptests&amp;category!=failing&amp;category!=nonlinuxtests&amp;category!=nonuaptests&quot; &quot;--logger:trx;LogFilePrefix=Functional-Unixnetcoreapp-$(TestSet)&quot;" Condition="'$(IsEnabledWindows)' != 'true'"/>
-  </Target>
-
-  <Target Name="RunManualTests">
-    <!-- Windows -->
-    <Exec ConsoleToMsBuild="true" Command="$(DotnetPath)dotnet test &quot;@(ManualTestsProj)&quot; -p:Configuration=$(Configuration) -p:Target$(TFGroup)Version=$(TF) -p:ReferenceType=$(ReferenceType) --no-build -l &quot;console;verbosity=normal&quot; --collect &quot;Code coverage&quot; -p:TestSet=$(TestSet) --results-directory $(ResultsDirectory) -p:TestTargetOS=Windows$(TargetGroup) --filter &quot;category!=non$(TargetGroup)tests&amp;category!=failing&amp;category!=nonwindowstests&quot; &quot;--logger:trx;LogFilePrefix=Manual-Windows$(TargetGroup)-$(TestSet)&quot;" Condition="'$(IsEnabledWindows)' == 'true'"/>
-    <!-- Unix -->
-    <Exec ConsoleToMsBuild="true" Command="$(DotnetPath)dotnet test &quot;@(ManualTestsProj)&quot; -p:Configuration=$(Configuration) -p:TargetNetCoreVersion=$(TF) -p:ReferenceType=$(ReferenceType) --no-build -l &quot;console;verbosity=normal&quot; --collect &quot;Code coverage&quot; -p:TestSet=$(TestSet) --results-directory $(ResultsDirectory) -p:TestTargetOS=Unixnetcoreapp --filter &quot;category!=nonnetcoreapptests&amp;category!=failing&amp;category!=nonlinuxtests&amp;category!=nonuaptests&quot; &quot;--logger:trx;LogFilePrefix=Manual-Unixnetcoreapp-$(TestSet)&quot;" Condition="'$(IsEnabledWindows)' != 'true'"/>
+
+  <!-- Functional Tests -->
+  <Target Name="RunFunctionalTests" DependsOnTargets="RunFunctionalTestsWindows;RunFunctionalTestsUnix" />
+  <Target Name="RunFunctionalTestsWindows" Condition="'$(IsEnabledWindows)' == 'true'">
+    <PropertyGroup>
+      <TestCommand>
+        $(DotnetPath)dotnet test "@(FunctionalTestsProj)"
+        --no-build
+        -v n
+        -p:Configuration=$(Configuration)
+        -p:Target$(TFGroup)Version=$(TF)
+        -p:TestTargetOS=Windows$(TargetGroup)
+        -p:ReferenceType=$(ReferenceType)
+        $(FilterArgument)
+        $(BlameArgument)
+        $(CollectArgument)
+        --results-directory "$(ResultsDirectory)"
+        --logger:"trx;LogFilePrefix=Functional-Windows$(TargetGroup)-$(TestSet)"
+      </TestCommand>
+      <!-- Convert more than one whitespace character into one space -->
+      <TestCommand>$([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " "))</TestCommand>
+    </PropertyGroup>
+    <Message Text=">>> Running Functional test for Windows via command: $(TestCommand)" />
+    <Exec ConsoleToMsBuild="true" Command="$(TestCommand)" />
+  </Target>
+  <Target Name="RunFunctionalTestsUnix" Condition="'$(IsEnabledWindows)' != 'true'">
+    <PropertyGroup>
+      <TestCommand>
+        $(DotnetPath)dotnet test "@(FunctionalTestsProj)"
+        --no-build
+        -v n
+        -p:Configuration=$(Configuration)
+        -p:TargetNetCoreVersion=$(TF)
+        -p:TestTargetOS=Unixnetcoreapp
+        -p:ReferenceType=$(ReferenceType)
+        $(FilterArgument)
+        $(BlameArgument)
+        $(CollectArgument)
+        --results-directory "$(ResultsDirectory)"
+        --logger:"trx;LogFilePrefix=Functional-Unixnetcoreapp-$(TestSet)"
+      </TestCommand>
+      <!-- Convert more than one whitespace character into one space -->
+      <TestCommand>$([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " "))</TestCommand>
+    </PropertyGroup>
+    <Message Text=">>> Running Functional test for Unix via command: $(TestCommand)" />
+    <Exec ConsoleToMsBuild="true" Command="$(TestCommand)" />
+  </Target>
+
+  <!-- Manual Tests -->
+  <Target Name="RunManualTests" DependsOnTargets="RunManualTestsWindows;RunManualTestsUnix" />
+  <Target Name="RunManualTestsWindows" Condition="'$(IsEnabledWindows)' == 'true'">
+    <PropertyGroup>
+      <TestCommand>
+        $(DotnetPath)dotnet test "@(ManualTestsProj)"
+        --no-build
+        -l "console;verbosity=normal"
+        -p:Configuration=$(Configuration)
+        -p:Target$(TFGroup)Version=$(TF)
+        -p:TestTargetOS=Windows$(TargetGroup)
+        -p:ReferenceType=$(ReferenceType)
+        -p:TestSet=$(TestSet)
+        $(FilterArgument)
+        $(BlameArgument)
+        $(CollectArgument)
+        --results-directory "$(ResultsDirectory)"
+        --logger:"trx;LogFilePrefix=Manual-Windows$(TargetGroup)-$(TestSet)"
+      </TestCommand>
+      <!-- Convert more than one whitespace character into one space -->
+      <TestCommand>$([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " "))</TestCommand>
+    </PropertyGroup>
+    <Message Text=">>> Running Functional test for Windows via command: $(TestCommand)" />
+    <Exec ConsoleToMsBuild="true" Command="$(TestCommand)" />
+  </Target>
+  <Target Name="RunManualTestsUnix"  Condition="'$(IsEnabledWindows)' != 'true'">
+    <PropertyGroup>
+      <TestCommand>
+        $(DotnetPath)dotnet test "@(ManualTestsProj)"
+        --no-build
+        -l "console;verbosity=normal"
+        -p:Configuration=$(Configuration)
+        -p:TargetNetCoreVersion=$(TF)
+        -p:TestTargetOS=Unixnetcoreapp
+        -p:ReferenceType=$(ReferenceType)
+        -p:TestSet=$(TestSet)
+        $(FilterArgument)
+        $(BlameArgument)
+        $(CollectArgument)
+        --results-directory "$(ResultsDirectory)"
+        --logger:"trx;LogFilePrefix=Manual-Unixnetcoreapp-$(TestSet)"
+      </TestCommand>
+      <!-- Convert more than one whitespace character into one space -->
+      <TestCommand>$([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " "))</TestCommand>
+    </PropertyGroup>
+    <Message Text=">>> Running Functional test for Windows via command: $(TestCommand)" />
+    <Exec ConsoleToMsBuild="true" Command="$(TestCommand)" />
   </Target>
 
   <Target Name="Clean">

eng/pipelines/common/templates/steps/run-all-tests-step.yml

@@ -64,9 +64,23 @@ steps:
         msbuildArguments: '-t:RunFunctionalTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }}'
       ${{ else }}: # x86
         msbuildArguments: '-t:RunFunctionalTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:DotnetPath=${{parameters.dotnetx86RootPath }}'
-    condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+    condition: succeededOrFailed()
     retryCountOnTaskFailure: 1
 
+  - task: MSBuild@1
+    displayName: 'Run Flaky Functional Tests ${{parameters.msbuildArchitecture }}'
+    inputs:
+      solution: build.proj
+      msbuildArchitecture: ${{parameters.msbuildArchitecture }}
+      platform: '${{parameters.platform }}'
+      configuration: '${{parameters.configuration }}'
+      ${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
+        msbuildArguments: '-t:RunFunctionalTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:Filter="category=flaky"'
+      ${{ else }}: # x86
+        msbuildArguments: '-t:RunFunctionalTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:DotnetPath=${{parameters.dotnetx86RootPath }} -p:Filter="category=flaky"'
+    condition: succeededOrFailed()
+    continueOnError: true
+
   - task: MSBuild@1
     displayName: 'Run Manual Tests ${{parameters.msbuildArchitecture }}'
     inputs:
@@ -78,9 +92,23 @@ steps:
         msbuildArguments: '-t:RunManualTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }}'
       ${{ else }}: # x86
         msbuildArguments: '-t:RunManualTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:DotnetPath=${{parameters.dotnetx86RootPath }}'
-    condition: eq(variables['Agent.OS'], 'Windows_NT')
+    condition: succeededOrFailed()
     retryCountOnTaskFailure: 2
 
+  - task: MSBuild@1
+    displayName: 'Run Flaky Manual Tests ${{parameters.msbuildArchitecture }}'
+    inputs:
+      solution: build.proj
+      msbuildArchitecture: ${{parameters.msbuildArchitecture }}
+      platform: '${{parameters.platform }}'
+      configuration: '${{parameters.configuration }}'
+      ${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
+        msbuildArguments: '-t:RunManualTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:Filter="category=flaky"'
+      ${{ else }}: # x86
+        msbuildArguments: '-t:RunManualTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:DotnetPath=${{parameters.dotnetx86RootPath }} -p:Filter="category=flaky"'
+    condition: succeededOrFailed()
+    continueOnError: true
+
 - ${{ else }}: # Linux or macOS
   - task: DotNetCoreCLI@2
     displayName: 'Run Functional Tests'
@@ -92,7 +120,19 @@ steps:
       verbosityRestore: Detailed
       verbosityPack: Detailed
     retryCountOnTaskFailure: 1
-    condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))
+    condition: succeededOrFailed()
+
+  - task: DotNetCoreCLI@2
+    displayName: 'Run Flaky Functional Tests'
+    inputs:
+      command: custom
+      projects: build.proj
+      custom: msbuild
+      arguments: '-t:RunFunctionalTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:platform=${{parameters.platform }} -p:Configuration=${{parameters.configuration }} -p:Filter="category=flaky"'
+      verbosityRestore: Detailed
+      verbosityPack: Detailed
+    condition: succeededOrFailed()
+    continueOnError: true
 
   - task: DotNetCoreCLI@2
     displayName: 'Run Manual Tests'
@@ -104,4 +144,16 @@ steps:
       verbosityRestore: Detailed
       verbosityPack: Detailed
     retryCountOnTaskFailure: 2
-    condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))
+    condition: succeededOrFailed()
+
+  - task: DotNetCoreCLI@2
+    displayName: 'Run Flaky Manual Tests'
+    inputs:
+      command: custom
+      projects: build.proj
+      custom: msbuild
+      arguments: '-t:RunManualTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:platform=${{parameters.platform }} -p:Configuration=${{parameters.configuration }} -p:Filter="category=flaky"'
+      verbosityRestore: Detailed
+      verbosityPack: Detailed
+    condition: succeededOrFailed()
+    continueOnError: true