test_runner: feat add coverage threshold for tests

Author: pulkit-30

lib/internal/test_runner/coverage.js

@@ -31,6 +31,11 @@ const kIgnoreRegex = /\/\* node:coverage ignore next (?<count>\d+ )?\*\//;
 const kLineEndingRegex = /\r?\n$/u;
 const kLineSplitRegex = /(?<=\r?\n)/u;
 const kStatusRegex = /\/\* node:coverage (?<status>enable|disable) \*\//;
+const defaultMinumumThreshold = {
+  lines: 0,
+  functions: 0,
+  branches: 0
+}
 
 class CoverageLine {
   #covered;
@@ -63,10 +68,11 @@ class CoverageLine {
 }
 
 class TestCoverage {
-  constructor(coverageDirectory, originalCoverageDirectory, workingDirectory) {
+  constructor(coverageDirectory, originalCoverageDirectory, workingDirectory, minimumThreshold = defaultMinumumThreshold) {
     this.coverageDirectory = coverageDirectory;
     this.originalCoverageDirectory = originalCoverageDirectory;
     this.workingDirectory = workingDirectory;
+    this.minimumThreshold = minimumThreshold;
   }
 
   summary() {
@@ -260,6 +266,11 @@ class TestCoverage {
     );
     coverageSummary.files.sort(sortCoverageFiles);
 
+    coverageSummary.threshold = {
+      lines: this.minimumThreshold.lines ? coverageSummary.totals.coveredLinePercent >= this.minimumThreshold.lines: true,
+      functions: this.minimumThreshold.functions ? coverageSummary.totals.coveredFunctionCount >= this.minimumThreshold.functions: true,
+      branches: this.minimumThreshold.branches ? coverageSummary.totals.coveredBranchCount >= this.minimumThreshold.branches: true,
+    }
     return coverageSummary;
   }
 
@@ -299,7 +310,7 @@ function sortCoverageFiles(a, b) {
   return StringPrototypeLocaleCompare(a.path, b.path);
 }
 
-function setupCoverage() {
+function setupCoverage(minimumThreshold) {
   let originalCoverageDirectory = process.env.NODE_V8_COVERAGE;
   const cwd = process.cwd();
 
@@ -323,7 +334,7 @@ function setupCoverage() {
   // child processes.
   process.env.NODE_V8_COVERAGE = coverageDirectory;
 
-  return new TestCoverage(coverageDirectory, originalCoverageDirectory, cwd);
+  return new TestCoverage(coverageDirectory, originalCoverageDirectory, cwd, minimumThreshold);
 }
 
 function mapRangeToLines(range, lines) {

lib/internal/test_runner/harness.js

@@ -1,9 +1,11 @@
 'use strict';
 const {
+  ArrayPrototypeFilter,
   ArrayPrototypeForEach,
   FunctionPrototypeBind,
   PromiseResolve,
   SafeMap,
+  StringPrototypeSplit,
 } = primordials;
 const { getCallerLocation } = internalBinding('util');
 const {
@@ -82,10 +84,11 @@ function configureCoverage(rootTest, globalOptions) {
     return null;
   }
 
+  const [_, value] = StringPrototypeSplit(ArrayPrototypeFilter(process.execArgv, (arg) => arg.startsWith("--experimental-test-coverage")), '=')
   const { setupCoverage } = require('internal/test_runner/coverage');
 
   try {
-    return setupCoverage();
+    return setupCoverage({ lines: value });
   } catch (err) {
     const msg = `Warning: Code coverage could not be enabled. ${err}`;
 

lib/internal/test_runner/utils.js

@@ -405,6 +405,16 @@ function getCoverageReport(pad, summary, symbol, color, table) {
             `${ArrayPrototypeJoin(ArrayPrototypeMap(kColumnsKeys, (columnKey, j) => getCell(NumberPrototypeToFixed(summary.totals[columnKey], 2), columnPadLengths[j], StringPrototypePadStart, false, summary.totals[columnKey])), kSeparator)} |\n`;
   if (table) report += addTableLine(prefix, tableWidth);
 
+  if(!summary.threshold.lines) {
+    report += `${red}${prefix}coverage threshold for lines not met\n${color}`;
+  }
+  if(!summary.threshold.branches) {
+    report += `${red}${prefix}coverage threshold for branches not met\n${color}`;
+  }
+  if(!summary.threshold.functions) {
+    report += `${red}${prefix}coverage threshold for functions not met\n${color}`;
+  }
+
   report += `${prefix}end of coverage report\n`;
   if (color) {
     report += white;

test/parallel/test-runner-coverage.js

@@ -21,7 +21,7 @@ function findCoverageFileForPid(pid) {
   });
 }
 
-function getTapCoverageFixtureReport() {
+function getTapCoverageFixtureReport(extraReport = []) {
   /* eslint-disable max-len */
   const report = [
     '# start of coverage report',
@@ -34,6 +34,7 @@ function getTapCoverageFixtureReport() {
     '# -------------------------------------------------------------------------------------------------------------------',
     '# all files                                |  78.35 |    43.75 |   60.00 |',
     '# -------------------------------------------------------------------------------------------------------------------',
+    ...extraReport,
     '# end of coverage report',
   ].join('\n');
   /* eslint-enable max-len */
@@ -45,7 +46,7 @@ function getTapCoverageFixtureReport() {
   return report;
 }
 
-function getSpecCoverageFixtureReport() {
+function getSpecCoverageFixtureReport(extraReport = []) {
   /* eslint-disable max-len */
   const report = [
     '\u2139 start of coverage report',
@@ -58,6 +59,7 @@ function getSpecCoverageFixtureReport() {
     '\u2139 -------------------------------------------------------------------------------------------------------------------',
     '\u2139 all files                                |  78.35 |    43.75 |   60.00 |',
     '\u2139 -------------------------------------------------------------------------------------------------------------------',
+    ...extraReport,
     '\u2139 end of coverage report',
   ].join('\n');
   /* eslint-enable max-len */
@@ -242,3 +244,16 @@ test('coverage reports on lines, functions, and branches', skipIfNoInspector, as
     });
   });
 });
+
+test('test tap coverage reporter with threshold value', skipIfNoInspector, async (t) => {
+    const fixture = fixtures.path('test-runner', 'coverage.js');
+    const args = ['--experimental-test-coverage=100', '--test-reporter', 'tap', fixture];
+    const options = { env: { ...process.env } };
+    const result = spawnSync(process.execPath, args, options);
+    const report = getTapCoverageFixtureReport(['# coverage threshold for lines not met']);
+    console.log(result.stdout.toString(),'\n')
+    console.log(report)
+    assert(result.stdout.toString().includes(report));
+    assert.strictEqual(result.stderr.toString(), '');
+    assert.strictEqual(result.status, 0);
+});