feat(scanner): integrate buit-in stats in the response of depWalker (#612)

* feat(scanner): integrate built-in stats in the response of depWalker

* Update workspaces/scanner/src/class/DateProvider.class.ts

Co-authored-by: Thomas.G <gentilhomme.thomas@gmail.com>

* Update workspaces/scanner/src/class/StatsCollector.class.ts

Co-authored-by: Thomas.G <gentilhomme.thomas@gmail.com>

* Update workspaces/scanner/src/class/StatsCollector.class.ts

Co-authored-by: Thomas.G <gentilhomme.thomas@gmail.com>

---------

Co-authored-by: Thomas.G <gentilhomme.thomas@gmail.com>

Author: clemgbld

.changeset/wet-bears-notice.md

@@ -0,0 +1,5 @@
+---
+"@nodesecure/scanner": minor
+---
+
+feat(scanner): integrate built-in stats in the response of depWalker

workspaces/scanner/src/class/DateProvider.class.ts

@@ -0,0 +1,17 @@
+export interface DateProvider {
+  now(): number;
+  oneYearAgo(): Date;
+}
+
+export class SystemDateProvider implements DateProvider {
+  now(): number {
+    return Date.now();
+  }
+
+  oneYearAgo(): Date {
+    const date = new Date();
+    date.setFullYear(date.getFullYear() - 1);
+
+    return date;
+  }
+}

workspaces/scanner/src/class/StatsCollector.class.ts

@@ -0,0 +1,50 @@
+// Import Internal Dependencies
+import { SystemDateProvider, type DateProvider } from "./DateProvider.class.ts";
+import type { ApiStats, Stats } from "../types.ts";
+
+export class StatsCollector {
+  #apiCalls: ApiStats[] = [];
+  #dateProvider: DateProvider;
+  #startedAt: number;
+
+  constructor(dateProvider: DateProvider = new SystemDateProvider()) {
+    this.#dateProvider = dateProvider;
+    this.#startedAt = this.#dateProvider.now();
+  }
+
+  track<T extends () => any>(name: string, fn: T): ReturnType<T> {
+    const startedAt = this.#dateProvider.now();
+    try {
+      const result = fn();
+      if (result instanceof Promise) {
+        return result.finally(() => this.#addApiStat(name, startedAt)
+        ) as ReturnType<T>;
+      }
+
+      this.#addApiStat(name, startedAt);
+
+      return result;
+    }
+    catch (err) {
+      this.#addApiStat(name, startedAt);
+      throw err;
+    }
+  }
+
+  #addApiStat(name: string, startedAt: number) {
+    this.#apiCalls.push({
+      name,
+      startedAt,
+      executionTime: this.#dateProvider.now() - startedAt
+    });
+  }
+
+  getStats(): Stats {
+    return {
+      startedAt: this.#startedAt,
+      executionTime: this.#dateProvider.now() - this.#startedAt,
+      apiCalls: this.#apiCalls,
+      apiCallsCount: this.#apiCalls.length
+    };
+  }
+}

workspaces/scanner/src/depWalker.ts

@@ -3,6 +3,8 @@ import path from "node:path";
 import { readFileSync } from "node:fs";
 
 // Import Third-party Dependencies
+import pacote from "pacote";
+import * as npmRegistrySDK from "@nodesecure/npm-registry-sdk";
 import { Mutex, MutexRelease } from "@openally/mutex";
 import {
   extractAndResolve,
@@ -26,7 +28,8 @@ import {
   getManifestLinks,
   NPM_TOKEN
 } from "./utils/index.ts";
-import { NpmRegistryProvider } from "./registry/NpmRegistryProvider.ts";
+import { NpmRegistryProvider, type NpmApiClient } from "./registry/NpmRegistryProvider.ts";
+import { StatsCollector } from "./class/StatsCollector.class.ts";
 import { RegistryTokenStore } from "./registry/RegistryTokenStore.ts";
 import { TempDirectory } from "./class/TempDirectory.class.ts";
 import { Logger, ScannerLoggerEvents } from "./class/logger.class.ts";
@@ -94,7 +97,6 @@ type InitialPayload =
   Partial<Payload> &
   {
     rootDependency: Payload["rootDependency"];
-    metadata: Payload["metadata"];
   };
 
 export async function depWalker(
@@ -113,7 +115,7 @@ export async function depWalker(
     npmRcConfig
   } = options;
 
-  const startedAt = Date.now();
+  const statsCollector = new StatsCollector();
   const isRemoteScanning = typeof location === "undefined";
   const tokenStore = new RegistryTokenStore(npmRcConfig, NPM_TOKEN.token);
 
@@ -130,18 +132,36 @@ export async function depWalker(
     },
     scannerVersion: packageVersion,
     vulnerabilityStrategy,
-    warnings: [],
-    metadata: {
-      startedAt,
-      executionTime: 0
-    }
+    warnings: []
   };
 
   const dependencies: Map<string, Dependency> = new Map();
   const highlightedPackages: Set<string> = new Set();
   const npmTreeWalker = new npm.TreeWalker({
-    registry
+    registry,
+    providers: {
+      pacote: {
+        manifest: (spec, opts) => statsCollector.track(`pacote.manifest ${spec}`, () => pacote.manifest(spec, opts)),
+        packument: (spec, opts) => statsCollector.track(`pacote.packument ${spec}`, () => pacote.packument(spec, opts))
+      }
+    }
   });
+  const npmApiClient: NpmApiClient = {
+    packument: (name, opts) => statsCollector.track(
+      `npmRegistrySDK.packument ${name}`,
+      () => npmRegistrySDK.packument(name, opts)
+    ),
+
+    packumentVersion: (name, version, opts) => statsCollector.track(
+      `npmRegistrySDK.packumentVersion ${name}@${version}`,
+      () => npmRegistrySDK.packumentVersion(name, version, opts)
+    ),
+
+    org: (namespace) => statsCollector.track(
+      `npmRegistrySDK.org ${namespace}`,
+      () => npmRegistrySDK.org(namespace)
+    )
+  };
   {
     logger
       .start(ScannerLoggerEvents.analysis.tree)
@@ -181,7 +201,8 @@ export async function depWalker(
         operationsQueue.push(
           new NpmRegistryProvider(name, version, {
             registry,
-            tokenStore
+            tokenStore,
+            npmApiClient
           }).enrichDependencyVersion(dep, dependencyConfusionWarnings, org)
         );
 
@@ -350,7 +371,7 @@ export async function depWalker(
       packages: [...highlightedPackages]
     };
     payload.dependencies = Object.fromEntries(dependencies);
-    payload.metadata.executionTime = Date.now() - startedAt;
+    payload.metadata = statsCollector.getStats();
 
     return payload as Payload;
   }

workspaces/scanner/src/registry/NpmRegistryProvider.ts

@@ -11,7 +11,8 @@ import * as i18n from "@nodesecure/i18n";
 import { isHTTPError } from "@openally/httpie";
 
 // Import Internal Dependencies
-import { PackumentExtractor, type DateProvider } from "./PackumentExtractor.ts";
+import { PackumentExtractor } from "./PackumentExtractor.ts";
+import type { DateProvider } from "../class/DateProvider.class.ts";
 import { fetchNpmAvatars } from "./fetchNpmAvatars.ts";
 import type {
   Dependency,

workspaces/scanner/src/registry/PackumentExtractor.ts

@@ -6,10 +6,7 @@ import { packageJSONIntegrityHash } from "@nodesecure/mama";
 import type {
   Dependency
 } from "../types.ts";
-
-export interface DateProvider {
-  oneYearAgo(): Date;
-}
+import { SystemDateProvider, type DateProvider } from "../class/DateProvider.class.ts";
 
 export interface PackumentExtractorOptions {
   dateProvider?: DateProvider;
@@ -103,12 +100,3 @@ export class PackumentExtractor {
     return result;
   }
 }
-
-class SystemDateProvider implements DateProvider {
-  oneYearAgo(): Date {
-    const date = new Date();
-    date.setFullYear(date.getFullYear() - 1);
-
-    return date;
-  }
-}

workspaces/scanner/src/types.ts

@@ -184,6 +184,40 @@ export type GlobalWarning = { message: string; } & (
   |
   DependencyConfusionWarning);
 
+export type ApiStats = {
+  /**
+    * UNIX Timestamp just before the api call start
+  */
+  startedAt: number;
+  /**
+    * Execution time in milliseconds
+  */
+  executionTime: number;
+
+  /**
+    * Name of the api call
+  */
+  name: string;
+};
+
+export type Stats = {
+  /**
+    * UNIX Timestamp when the scan started
+  */
+  startedAt: number;
+  /**
+    * Execution time in milliseconds
+  */
+  executionTime: number;
+
+  /**
+    * Number of external API calls
+  */
+  apiCallsCount: number;
+
+  apiCalls: ApiStats[];
+};
+
 export interface Payload {
   /** Payload unique id */
   id: string;
@@ -207,16 +241,7 @@ export interface Payload {
   /** Vulnerability strategy name (npm, snyk, node) */
   vulnerabilityStrategy: Vulnera.Kind;
 
-  metadata: {
-    /**
-     * UNIX Timestamp when the scan started
-     */
-    startedAt: number;
-    /**
-     * Execution time in milliseconds
-     */
-    executionTime: number;
-  };
+  metadata: Stats;
 }
 
 export type SemverRange = string | "*";

workspaces/scanner/test/StatsCollector.spec.ts

@@ -0,0 +1,100 @@
+// Import Node.js Dependencies
+import { describe, it } from "node:test";
+import assert from "node:assert";
+
+// Import Internal Dependencies
+import type { DateProvider } from "../src/class/DateProvider.class.ts";
+import { StatsCollector } from "../src/class/StatsCollector.class.ts";
+
+describe("StatsCollectors", () => {
+  it("should get the expected global start and execution time", () => {
+    const dateProvider = new FakeDateProvider();
+    dateProvider.setNow(1658512000000);
+    const statsCollector = new StatsCollector(dateProvider);
+    dateProvider.setNow(1658512001000);
+    const { startedAt, executionTime } = statsCollector.getStats();
+    assert.strictEqual(startedAt, 1658512000000);
+    assert.strictEqual(executionTime, 1000);
+  });
+
+  it("should still record the exexution time if the function being tracked throws", () => {
+    const dateProvider = new FakeDateProvider();
+    dateProvider.setNow(1658512000000);
+    const statsCollector = new StatsCollector(dateProvider);
+    assert.throws(() => {
+      statsCollector.track("api/test/1", () => {
+        dateProvider.setNow(1658512001000);
+        throw new Error("oh no!");
+      });
+    });
+
+    const { apiCalls, apiCallsCount } = statsCollector.getStats();
+    assert.strictEqual(apiCallsCount, 1);
+    assert.deepEqual(apiCalls, [
+      {
+        name: "api/test/1",
+        startedAt: 1658512000000,
+        executionTime: 1000
+      }
+
+    ]);
+  });
+
+  it("should be able to track the start and execution time of external api call", async() => {
+    let hasFnOneBeenCalled = false;
+    let hasFnTwoBeenCalled = false;
+    const dateProvider = new FakeDateProvider();
+    dateProvider.setNow(1658512000000);
+    const statsCollector = new StatsCollector(dateProvider);
+    dateProvider.setNow(1658512001001);
+    const promise = statsCollector.track("api/test/1", () => {
+      hasFnOneBeenCalled = true;
+
+      return Promise.resolve(1);
+    });
+
+    dateProvider.setNow(1658512002000);
+    const promiseResult = await promise;
+
+    dateProvider.setNow(1658512003000);
+    const fnResult = statsCollector.track("api/test/2", () => {
+      hasFnTwoBeenCalled = true;
+      dateProvider.setNow(1658512004000);
+
+      return null;
+    });
+    dateProvider.setNow(1658512005000);
+    const { apiCalls, apiCallsCount } = statsCollector.getStats();
+    assert.strictEqual(promiseResult, 1);
+    assert.strictEqual(fnResult, null);
+    assert.strictEqual(hasFnOneBeenCalled, true);
+    assert.strictEqual(hasFnTwoBeenCalled, true);
+    assert.strictEqual(apiCallsCount, 2);
+    assert.deepEqual(apiCalls, [
+      {
+        name: "api/test/1",
+        startedAt: 1658512001001,
+        executionTime: 999
+      },
+      {
+        name: "api/test/2",
+        startedAt: 1658512003000,
+        executionTime: 1000
+      }
+    ]);
+  });
+});
+
+class FakeDateProvider implements DateProvider {
+  #now: number;
+  now(): number {
+    return this.#now;
+  }
+  oneYearAgo(): Date {
+    return new Date(Date.now() - (365 * 24 * 60 * 60 * 1000));
+  }
+
+  setNow(now: number) {
+    this.#now = now;
+  }
+}