[main] Source code updates from dotnet/dotnet

eng/common/build.ps1

@@ -6,6 +6,7 @@ Param(
   [string][Alias('v')]$verbosity = "minimal",
   [string] $msbuildEngine = $null,
   [bool] $warnAsError = $true,
+  [string] $warnNotAsError = '',
   [bool] $nodeReuse = $true,
   [switch] $buildCheck = $false,
   [switch][Alias('r')]$restore,
@@ -70,6 +71,7 @@ function Print-Usage() {
   Write-Host "  -excludeCIBinarylog     Don't output binary log (short: -nobl)"
   Write-Host "  -prepareMachine         Prepare machine for CI run, clean up processes after build"
   Write-Host "  -warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
+  Write-Host "  -warnNotAsError <value> Sets a semi-colon delimited list of warning codes that should not be treated as errors"
   Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
   Write-Host "  -excludePrereleaseVS    Set to exclude build engines in prerelease versions of Visual Studio"
   Write-Host "  -nativeToolsOnMachine   Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)"

eng/common/build.sh

@@ -42,6 +42,7 @@ usage()
   echo "  --prepareMachine         Prepare machine for CI run, clean up processes after build"
   echo "  --nodeReuse <value>      Sets nodereuse msbuild parameter ('true' or 'false')"
   echo "  --warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
+  echo "  --warnNotAsError <value> Sets a semi-colon delimited list of warning codes that should not be treated as errors"
   echo "  --buildCheck <value>     Sets /check msbuild parameter"
   echo "  --fromVMR                Set when building from within the VMR"
   echo ""
@@ -78,6 +79,7 @@ ci=false
 clean=false
 
 warn_as_error=true
+warn_not_as_error=''
 node_reuse=true
 build_check=false
 binary_log=false
@@ -176,6 +178,10 @@ while [[ $# -gt 0 ]]; do
       warn_as_error=$2
       shift
       ;;
+    -warnnotaserror)
+      warn_not_as_error=$2
+      shift
+      ;;
     -nodereuse)
       node_reuse=$2
       shift

eng/common/core-templates/job/renovate.yml

@@ -53,6 +53,30 @@ parameters:
   type: boolean
   default: false
 
+# Name of the arcade repository resource in the pipeline.
+# This allows repos which haven't been onboarded to Arcade to still use this 
+# template by checking out the repo as a resource with a custom name and pointing
+# this parameter to it.
+- name: arcadeRepoResource
+  type: string
+  default: self
+
+# Directory name for the self repo under $(Build.SourcesDirectory) in multi-checkout.
+# In multi-checkout (when arcadeRepoResource != 'self'), Azure DevOps checks out the
+# self repo to $(Build.SourcesDirectory)/<repoName>. Set this to match the auto-generated
+# directory name. Using the auto-generated name is necessary rather than explicitly
+# defining a checkout path because container jobs expect repos to live under the agent's
+# workspace ($(Pipeline.Workspace)). On some self-hosted setups the host path
+# (e.g., /mnt/vss/_work) differs from the container path (e.g., /__w), and a custom checkout
+# path can fail validation. Using the default checkout location keeps the paths consistent
+# and avoids this issue.
+- name: selfRepoName
+  type: string
+  default: ''
+- name: arcadeRepoName
+  type: string
+  default: ''
+
 # Pool configuration for the job.
 - name: pool
   type: object
@@ -71,16 +95,36 @@ jobs:
   # Changing the variable name here would require updating the name in https://github.com/dotnet/arcade/blob/main/eng/renovate.json as well.
   - name: renovateVersion
     value: '42'
+    readonly: true
+  - name: renovateLogFilePath
+    value: '$(Build.ArtifactStagingDirectory)/renovate.json'
+    readonly: true
   - name: dryRunArg
+    readonly: true
     ${{ if eq(parameters.dryRun, true) }}:
       value: 'full'
     ${{ else }}:
       value: ''
   - name: recreateWhenArg
+    readonly: true
     ${{ if eq(parameters.forceRecreatePR, true) }}:
       value: 'always'
     ${{ else }}:
       value: ''
+  # In multi-checkout (without custom paths), Azure DevOps places each repo under
+  # $(Build.SourcesDirectory)/<repoName>. selfRepoName must be provided in that case.
+  - name: selfRepoPath
+    readonly: true
+    ${{ if eq(parameters.arcadeRepoResource, 'self') }}:
+      value: '$(Build.SourcesDirectory)'
+    ${{ else }}:
+      value: '$(Build.SourcesDirectory)/${{ parameters.selfRepoName }}'
+  - name: arcadeRepoPath
+    readonly: true
+    ${{ if eq(parameters.arcadeRepoResource, 'self') }}:
+      value: '$(Build.SourcesDirectory)'
+    ${{ else }}:
+      value: '$(Build.SourcesDirectory)/${{ parameters.arcadeRepoName }}'
   pool: ${{ parameters.pool }}
 
   templateContext:
@@ -96,17 +140,34 @@ jobs:
   steps:
   - checkout: self
     fetchDepth: 1
+
+  - ${{ if ne(parameters.arcadeRepoResource, 'self') }}:
+    - checkout: ${{ parameters.arcadeRepoResource }}
+      fetchDepth: 1
 
-  - script: renovate-config-validator $(Build.SourcesDirectory)/${{parameters.renovateConfigPath}}
+  - script: |
+      renovate-config-validator $(selfRepoPath)/${{parameters.renovateConfigPath}} 2>&1 | tee /tmp/renovate-config-validator.out
+      validatorExit=${PIPESTATUS[0]}
+      if grep -q '^ WARN:' /tmp/renovate-config-validator.out; then
+        echo "##vso[task.logissue type=warning]Renovate config validator produced warnings."
+        echo "##vso[task.complete result=SucceededWithIssues]"
+      fi
+      exit $validatorExit
     displayName: Validate Renovate config
     env:
       LOG_LEVEL: info
       LOG_FILE_LEVEL: debug
       LOG_FILE: $(Build.ArtifactStagingDirectory)/renovate-config-validator.json
 
   - script: |
-      . $(Build.SourcesDirectory)/eng/common/renovate.env
-      renovate
+      . $(arcadeRepoPath)/eng/common/renovate.env
+      renovate 2>&1 | tee /tmp/renovate.out
+      renovateExit=${PIPESTATUS[0]}
+      if grep -q '^ WARN:' /tmp/renovate.out; then
+        echo "##vso[task.logissue type=warning]Renovate produced warnings."
+        echo "##vso[task.complete result=SucceededWithIssues]"
+      fi
+      exit $renovateExit
     displayName: Run Renovate
     env:
       RENOVATE_FORK_TOKEN: $(BotAccount-dotnet-renovate-bot-PAT)
@@ -117,13 +178,13 @@ jobs:
       RENOVATE_RECREATE_WHEN: $(recreateWhenArg)
       LOG_LEVEL: info
       LOG_FILE_LEVEL: debug
-      LOG_FILE: $(Build.ArtifactStagingDirectory)/renovate.json
-      RENOVATE_CONFIG_FILE: $(Build.SourcesDirectory)/${{parameters.renovateConfigPath}}
+      LOG_FILE: $(renovateLogFilePath)
+      RENOVATE_CONFIG_FILE: $(selfRepoPath)/${{parameters.renovateConfigPath}}
 
   - script: |
       echo "PRs created by Renovate:"
-      if [ -s "$(Build.ArtifactStagingDirectory)/renovate-log.json" ]; then
-        if ! jq -r 'select(.msg == "PR created" and .pr != null) | "https://github.com/\(.repository)/pull/\(.pr)"' "$(Build.ArtifactStagingDirectory)/renovate-log.json" | sort -u; then
+      if [ -s "$(renovateLogFilePath)" ]; then
+        if ! jq -r 'select(.msg == "PR created" and .pr != null) | "https://github.com/\(.repository)/pull/\(.pr)"' "$(renovateLogFilePath)" | sort -u; then
           echo "##vso[task.logissue type=warning]Failed to parse Renovate log file with jq."
           echo "##vso[task.complete result=SucceededWithIssues]"
         fi

eng/common/core-templates/job/source-index-stage1.yml

@@ -15,6 +15,8 @@ jobs:
   variables:
   - name: BinlogPath
     value: ${{ parameters.binlogPath }}
+  - name: skipComponentGovernanceDetection
+    value: true
   - template: /eng/common/core-templates/variables/pool-providers.yml
     parameters:
       is1ESPipeline: ${{ parameters.is1ESPipeline }}

eng/common/core-templates/post-build/post-build.yml

@@ -50,16 +50,6 @@ parameters:
   type: boolean
   default: false
 
-- name: SDLValidationParameters
-  type: object
-  default:
-    enable: false
-    publishGdn: false
-    continueOnError: false
-    params: ''
-    artifactNames: ''
-    downloadArtifacts: true
-
 - name: isAssetlessBuild
   type: boolean
   displayName: Is Assetless Build
@@ -103,7 +93,7 @@ parameters:
   default: false
 
 stages:
-- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true')) }}:
   - stage: Validate
     dependsOn: ${{ parameters.validateDependsOn }}
     displayName: Validate Build Assets
@@ -206,7 +196,7 @@ stages:
         displayName: Validate
         inputs:
           filePath: eng\common\sdk-task.ps1
-          arguments: -task SigningValidation -restore -msbuildEngine vs
+          arguments: -task SigningValidation -restore
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
             /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt'
             ${{ parameters.signingValidationAdditionalParameters }}
@@ -268,7 +258,7 @@ stages:
 
 - ${{ if ne(parameters.publishAssetsImmediately, 'true') }}:
   - stage: publish_using_darc
-    ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+    ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true')) }}:
       dependsOn: ${{ parameters.publishDependsOn }}
     ${{ else }}:
       dependsOn: ${{ parameters.validateDependsOn }}

eng/common/core-templates/stages/renovate.yml

@@ -35,6 +35,21 @@ parameters:
   type: boolean
   default: false
 
+# Name of the arcade repository resource in the pipeline.
+# This allows repos which haven't been onboarded to Arcade to still use this 
+# template by checking out the repo as a resource with a custom name and pointing
+# this parameter to it.
+- name: arcadeRepoResource
+  type: string
+  default: 'self'
+
+- name: selfRepoName
+  type: string
+  default: ''
+- name: arcadeRepoName
+  type: string
+  default: ''
+
 # Pool configuration for the pipeline.
 - name: pool
   type: object
@@ -69,18 +84,28 @@ extends:
     pool: ${{ parameters.pool }}
     sdl:
       sourceAnalysisPool: ${{ parameters.sdlPool }}
+      # When repos that aren't onboarded to Arcade use this template, they set the
+      # arcadeRepoResource parameter to point to their Arcade repo resource. In that case,
+      # Aracde will be excluded from SDL analysis.
+      ${{ if ne(parameters.arcadeRepoResource, 'self') }}:
+        sourceRepositoriesToScan:
+          exclude:
+          - repository: ${{ parameters.arcadeRepoResource }}
     containers:
       RenovateContainer:
         image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-renovate-${{ parameters.renovateVersion }}-amd64
     stages:
     - stage: Renovate
       displayName: Run Renovate
       jobs:
-      - template: /eng/common/core-templates/job/renovate.yml@self
+      - template: /eng/common/core-templates/job/renovate.yml@${{ parameters.arcadeRepoResource }}
         parameters:
           renovateConfigPath: ${{ parameters.renovateConfigPath }}
           gitHubRepo: ${{ parameters.gitHubRepo }}
           baseBranches: ${{ parameters.baseBranches }}
           dryRun: ${{ parameters.dryRun }}
           forceRecreatePR: ${{ parameters.forceRecreatePR }}
           pool: ${{ parameters.pool }}
+          arcadeRepoResource: ${{ parameters.arcadeRepoResource }}
+          selfRepoName: ${{ parameters.selfRepoName }}
+          arcadeRepoName: ${{ parameters.arcadeRepoName }}

eng/common/core-templates/steps/install-microbuild.yml

@@ -73,7 +73,7 @@ steps:
     # YAML expansion, and Windows vs. Linux/Mac uses different service connections. However,
     # we can avoid including the MB install step if not enabled at all. This avoids a bunch of
     # extra pipeline authorizations, since most pipelines do not sign on non-Windows.
-    - template: /eng/common/core-templates/steps/install-microbuild-impl.yml@self
+    - template: /eng/common/core-templates/steps/install-microbuild-impl.yml
       parameters:
         enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }}
         microbuildTaskInputs:
@@ -95,7 +95,7 @@ steps:
         condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test'))
 
     - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}:
-      - template: /eng/common/core-templates/steps/install-microbuild-impl.yml@self
+      - template: /eng/common/core-templates/steps/install-microbuild-impl.yml
         parameters:
           enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }}
           microbuildTaskInputs:

eng/common/core-templates/steps/publish-logs.yml

@@ -31,7 +31,6 @@ steps:
       -runtimeSourceFeed https://ci.dot.net/internal 
       -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)'
       '$(publishing-dnceng-devdiv-code-r-build-re)'
-      '$(MaestroAccessToken)'
       '$(dn-bot-all-orgs-artifact-feeds-rw)'
       '$(akams-client-id)'
       '$(microsoft-symbol-server-pat)'