Skip to content

Matrix Testing Updates #166

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jsinglet merged 3 commits into from
Jan 23, 2023
Merged

Matrix Testing Updates #166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7e82241
skip summary report flag
jsinglet Jan 6, 2023
10c3d66
wrk
jsinglet Jan 17, 2023
8374019
Merge branch 'main' into jsinglet/matrix-testing
jsinglet Jan 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 17 additions & 23 deletions scripts/matrix_testing/CreateMatrixTestReport.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,12 @@ param(
[string]
$ReportDir = (Get-Location),

# Skip summary report -- used for Linux hosts that don't support
# the OLE database stuff.
[Parameter(Mandatory = $false)]
[switch]
$SkipSummaryReport,

# Tells the script to use the sytem tmp directory instead of the rule
# directory.
[Parameter(Mandatory = $false)]
Expand Down Expand Up @@ -245,6 +251,7 @@ else {
Write-Host "Loaded $($queriesToCheck.Count) Queries."
}


#
# Step 2: Verify All the Required CLI Tools are Installed
#
Expand Down Expand Up @@ -290,8 +297,7 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel
"RULE" = $CurrentRuleName;
"QUERY" = $CurrentQueryName;
"COMPILE_PASS" = $false;
"EXTRACTOR_PASS" = $false;
"EXTRACTOR_ERRORS" = "";
"COMPILE_ERROR_OUTPUT" = "";
"TEST_PASS" = $false ;
"TEST_DIFFERENCE" = "";
}
Expand All @@ -316,32 +322,19 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel
}
catch {
Write-Host -ForegroundColor ([ConsoleColor]4) "FAILED"
$row["COMPILE_ERROR_OUTPUT"] = $_

return $row # although it is unlikely to succeed with the next rule skipping to the next rule
# ensures all of the rules will be reported in the
# output.
}

$row["COMPILE_PASS"] = $true
Write-Host "Validating extractor results..." -NoNewline

try {
$diagnostics = Execute-QueryAndDecodeAsJson -DatabasePath $db -QueryPath $diagnostic_query
}catch {
Write-Host -ForegroundColor ([ConsoleColor]4) $_Exception.Message
return $row
}

if ( $diagnostics.'#select'.tuples.Length -eq 0 ) {
$row["EXTRACTOR_PASS"] = $true
Write-Host -ForegroundColor ([ConsoleColor]2) "OK"
} else {
Write-Host -ForegroundColor ([ConsoleColor]4) "FAILED"
$row["EXTRACTOR_ERRORS"] = $diagnostics | ConvertTo-Json -Depth 100
}


Write-Host "Checking expected output..."

# Dragons below 🐉🐉🐉
#
# Note this technique uses so-called "wizard" settings to make it possible
# to compare hand compiled databases using qltest. The relative paths and
# other options are required to be set as below (especially the detail about
Expand Down Expand Up @@ -381,7 +374,6 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel
Write-Host "Standard Out Buffered to: $stdOut"
Write-Host "Standard Error Buffered to: $stdErr"


$procDetails = Start-Process -FilePath "codeql" -PassThru -NoNewWindow -Wait -ArgumentList "test run $qlRefFile --dataset=`"$datasetRelPath`"" -RedirectStandardOutput $stdOut -RedirectStandardError $stdErr

if (-Not $procDetails.ExitCode -eq 0) {
Expand Down Expand Up @@ -420,6 +412,8 @@ foreach ($r in $REPORT) {
[PSCustomObject]$r | Export-CSV -Path $reportOutputFile -Append -NoTypeInformation
}

# write out a summary
Write-Host "Writing summary report to $summaryReportOutputFile"
Create-Summary-Report -DataFile $reportOutputFile -OutputFile $summaryReportOutputFile
if (-not $SkipSummaryReport){
# write out a summary
Write-Host "Writing summary report to $summaryReportOutputFile"
Create-Summary-Report -DataFile $reportOutputFile -OutputFile $summaryReportOutputFile
}
8 changes: 6 additions & 2 deletions scripts/matrix_testing/NewDatabaseForRule.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,14 @@ function New-Database-For-Rule {

Write-Host "codeql database create -l cpp -s $RuleTestDir --command='$BUILD_COMMAND' $DB_PATH"

$procDetails = Start-Process -FilePath "codeql" -PassThru -NoNewWindow -Wait -ArgumentList "database create -l cpp -s $RuleTestDir --command=`"$BUILD_COMMAND`" $DB_PATH"
$stdOut = Join-Path ([System.IO.Path]::GetTempPath()) ([System.Guid]::NewGuid())

$procDetails = Start-Process -FilePath "codeql" -RedirectStandardOutput $stdOut -PassThru -NoNewWindow -Wait -ArgumentList "database create -l cpp -s $RuleTestDir --command=`"$BUILD_COMMAND`" $DB_PATH"

Get-Content $stdOut | Out-String | Write-Host

if (-Not $procDetails.ExitCode -eq 0) {
throw "Database creation failed."
throw Get-Content $stdOut | Out-String
}

return $DB_PATH
Expand Down