Add testing performance telemetry (#25980)

helps to measure:
#25978

Author: eleanorjboyd

src/client/telemetry/constants.ts

@@ -48,6 +48,7 @@ export enum EventName {
     UNITTEST_DISCOVERY_DONE = 'UNITTEST.DISCOVERY.DONE',
     UNITTEST_RUN_STOP = 'UNITTEST.RUN.STOP',
     UNITTEST_RUN = 'UNITTEST.RUN',
+    UNITTEST_RUN_DONE = 'UNITTEST.RUN.DONE',
     UNITTEST_RUN_ALL_FAILED = 'UNITTEST.RUN_ALL_FAILED',
     UNITTEST_DISABLED = 'UNITTEST.DISABLED',
 
@@ -101,6 +102,17 @@ export enum EventName {
     ENVIRONMENT_TERMINAL_GLOBAL_PIP = 'ENVIRONMENT.TERMINAL.GLOBAL_PIP',
 }
 
+export const UNITTEST_RUN_FAILURE_CATEGORIES = [
+    'pipe-cancelled',
+    'subprocess-crash',
+    'no-results',
+    'env-mismatch',
+    'cancelled',
+    'unknown',
+] as const;
+
+export type UnitTestRunFailureCategory = typeof UNITTEST_RUN_FAILURE_CATEGORIES[number];
+
 export enum PlatformErrors {
     FailedToParseVersion = 'FailedToParseVersion',
     FailedToDetermineOS = 'FailedToDetermineOS',

src/client/telemetry/index.ts

@@ -12,6 +12,7 @@ import { StopWatch } from '../common/utils/stopWatch';
 import { EnvironmentType, PythonEnvironment } from '../pythonEnvironments/info';
 import { TensorBoardPromptSelection } from '../tensorBoard/constants';
 import { EventName } from './constants';
+import type { UnitTestRunFailureCategory } from './constants';
 import type { TestTool } from './types';
 
 /**
@@ -2165,7 +2166,12 @@ export interface IEventNamePropertyMapping {
     /* __GDPR__
        "unittest.discovery.done" : {
           "tool" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
-          "failed" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }
+          "failed" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "mode" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "trigger" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "failureCategory" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd" },
+          "totalDurationMs" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true },
+          "testCount" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true }
        }
      */
     [EventName.UNITTEST_DISCOVERY_DONE]: {
@@ -2181,6 +2187,36 @@ export interface IEventNamePropertyMapping {
          * @type {boolean}
          */
         failed: boolean;
+        /**
+         * Testing architecture used for discovery:
+         * 'project' = per-project discovery through the Python Environments API;
+         * 'legacy' = workspace-level discovery through the existing WorkspaceTestAdapter.
+         */
+        mode?: 'project' | 'legacy';
+        /**
+         * Source that triggered the discovery.
+         */
+        trigger?: 'auto' | 'ui' | 'commandpalette' | 'watching' | 'interpreter';
+        /**
+         * Coarse failure category. Only populated when `failed` is true.
+         */
+        failureCategory?:
+            | 'subprocess-exit-non-zero'
+            | 'pipe-error'
+            | 'pytest-collect-error'
+            | 'plugin-exception'
+            | 'timeout'
+            | 'env-resolution'
+            | 'cancelled'
+            | 'unknown';
+        /**
+         * Wall-clock duration of the discovery cycle in milliseconds.
+         */
+        totalDurationMs?: number;
+        /**
+         * Number of test items discovered (leaf nodes).
+         */
+        testCount?: number;
     };
     /**
      * Telemetry event sent when cancelling discovering tests
@@ -2222,6 +2258,43 @@ export interface IEventNamePropertyMapping {
        "unittest.run.all_failed" : { "owner": "eleanorjboyd" }
      */
     [EventName.UNITTEST_RUN_ALL_FAILED]: never | undefined;
+    /**
+     * Telemetry event sent at the end of a test run, capturing duration and pipe health.
+     * Companion to UNITTEST_RUN (which is emitted at run start).
+     */
+    /* __GDPR__
+       "unittest.run.done" : {
+          "tool" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "debugging" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "mode" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "failed" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+                    "failureCategory" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd" },
+                    "durationMs" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true },
+                    "requestedCount" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true }
+       }
+     */
+    [EventName.UNITTEST_RUN_DONE]: {
+        tool: TestTool;
+        debugging: boolean;
+        mode: 'project' | 'legacy';
+        /**
+         * `true` if the run ended without reporting all requested results,
+         * or if the subprocess crashed / threw.
+         */
+        failed: boolean;
+        /**
+         * Coarse failure category when `failed` is true.
+         */
+        failureCategory?: UnitTestRunFailureCategory;
+        /**
+         * Wall-clock duration of the run in milliseconds.
+         */
+        durationMs?: number;
+        /**
+         * Number of test items the user asked to run.
+         */
+        requestedCount?: number;
+    };
     /**
      * Telemetry event sent when testing is disabled for a workspace.
      */

src/client/testing/main.ts

@@ -223,7 +223,7 @@ export class UnitTestManagementService implements IExtensionActivationService {
             interpreterService.onDidChangeInterpreter(async () => {
                 traceVerbose('Testing: Triggered refresh due to interpreter change.');
                 sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_TRIGGER, undefined, { trigger: 'interpreter' });
-                await this.testController?.refreshTestData(undefined, { forceRefresh: true });
+                await this.testController?.refreshTestData(undefined, { forceRefresh: true, trigger: 'interpreter' });
             }),
         );
     }

src/client/testing/testController/common/discoveryTelemetry.ts

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import { StopWatch } from '../../../common/utils/stopWatch';
+
+/** Source that requested a test discovery refresh. */
+export type DiscoveryTriggerKind = 'auto' | 'ui' | 'commandpalette' | 'watching' | 'interpreter';
+
+/** Testing architecture used for discovery. */
+export type DiscoveryMode = 'project' | 'legacy';
+
+export interface DiscoveryTelemetryCycle {
+    mode: DiscoveryMode;
+    trigger?: DiscoveryTriggerKind;
+    stopWatch: StopWatch;
+}
+
+export class DiscoveryTelemetryState {
+    private activeCycle?: DiscoveryTelemetryCycle;
+
+    constructor(public readonly defaultMode: DiscoveryMode) {}
+
+    public start(context: Omit<DiscoveryTelemetryCycle, 'stopWatch'>): void {
+        this.activeCycle = { ...context, stopWatch: new StopWatch() };
+    }
+
+    public complete(): DiscoveryTelemetryCycle | undefined {
+        const cycle = this.activeCycle;
+        this.activeCycle = undefined;
+        return cycle;
+    }
+}

src/client/testing/testController/common/projectTestExecution.ts

@@ -5,6 +5,8 @@ import { CancellationToken, FileCoverageDetail, TestItem, TestRun, TestRunProfil
 import { traceError, traceInfo, traceVerbose, traceWarn } from '../../../logging';
 import { sendTelemetryEvent } from '../../../telemetry';
 import { EventName } from '../../../telemetry/constants';
+import type { UnitTestRunFailureCategory } from '../../../telemetry/constants';
+import { StopWatch } from '../../../common/utils/stopWatch';
 import { IPythonExecutionFactory } from '../../../common/process/types';
 import { ITestDebugLauncher } from '../../common/types';
 import { ProjectAdapter } from './projectAdapter';
@@ -70,13 +72,28 @@ export async function executeTestsForProjects(
             debugging: isDebugMode,
         });
 
+        const stopWatch = new StopWatch();
+        let failed = false;
+        let failureCategory: UnitTestRunFailureCategory | undefined;
         try {
             await executeTestsForProject(project, items, runInstance, request, deps);
         } catch (error) {
+            failed = true;
+            failureCategory = token.isCancellationRequested ? 'cancelled' : 'unknown';
             // Don't log cancellation as an error
             if (!token.isCancellationRequested) {
                 traceError(`[test-by-project] Execution failed for project ${project.projectName}:`, error);
             }
+        } finally {
+            sendTelemetryEvent(EventName.UNITTEST_RUN_DONE, undefined, {
+                tool: project.testProvider,
+                debugging: isDebugMode,
+                mode: 'project',
+                failed,
+                failureCategory,
+                durationMs: stopWatch.elapsedTime,
+                requestedCount: items.length,
+            });
         }
     });
 

src/client/testing/testController/common/resultResolver.ts

@@ -11,6 +11,7 @@ import { TestItemIndex } from './testItemIndex';
 import { TestDiscoveryHandler } from './testDiscoveryHandler';
 import { TestExecutionHandler } from './testExecutionHandler';
 import { TestCoverageHandler } from './testCoverageHandler';
+import { DiscoveryTelemetryState } from './discoveryTelemetry';
 
 export class PythonResultResolver implements ITestResultResolver {
     testController: TestController;
@@ -26,6 +27,8 @@ export class PythonResultResolver implements ITestResultResolver {
 
     public detailedCoverageMap = new Map<string, FileCoverageDetail[]>();
 
+    public readonly discoveryTelemetry: DiscoveryTelemetryState;
+
     /**
      * Optional project ID for scoping test IDs.
      * When set, all test IDs are prefixed with `{projectId}@@vsc@@` for project-based testing.
@@ -52,6 +55,7 @@ export class PythonResultResolver implements ITestResultResolver {
         this.projectName = projectName;
         // Initialize a new TestItemIndex which will be used to track test items in this workspace/project
         this.testItemIndex = new TestItemIndex();
+        this.discoveryTelemetry = new DiscoveryTelemetryState(projectId ? 'project' : 'legacy');
     }
 
     // Expose for backward compatibility (WorkspaceTestAdapter accesses these)
@@ -76,7 +80,7 @@ export class PythonResultResolver implements ITestResultResolver {
     }
 
     public resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): void {
-        PythonResultResolver.discoveryHandler.processDiscovery(
+        const testCount = PythonResultResolver.discoveryHandler.processDiscovery(
             payload,
             this.testController,
             this.testItemIndex,
@@ -86,9 +90,17 @@ export class PythonResultResolver implements ITestResultResolver {
             this.projectId,
             this.projectName,
         );
+        const cycle = this.discoveryTelemetry.complete();
+        const mode = cycle?.mode ?? this.discoveryTelemetry.defaultMode;
+        const failed = payload?.status === 'error';
         sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, {
             tool: this.testProvider,
-            failed: false,
+            failed,
+            mode,
+            trigger: cycle?.trigger,
+            failureCategory: failed ? (token?.isCancellationRequested ? 'cancelled' : 'unknown') : undefined,
+            totalDurationMs: cycle?.stopWatch.elapsedTime,
+            testCount,
         });
     }
 

src/client/testing/testController/common/testDiscoveryHandler.ts

@@ -30,10 +30,10 @@ export class TestDiscoveryHandler {
         token?: CancellationToken,
         projectId?: string,
         projectName?: string,
-    ): void {
+    ): number {
         if (!payload) {
             // No test data is available
-            return;
+            return 0;
         }
 
         const workspacePath = workspaceUri.fsPath;
@@ -57,10 +57,14 @@ export class TestDiscoveryHandler {
             // Clear existing mappings before rebuilding test tree
             testItemIndex.clear();
 
+            if (rawTestData.tests === null) {
+                return 0;
+            }
+
             // If the test root for this folder exists: Workspace refresh, update its children.
             // Otherwise, it is a freshly discovered workspace, and we need to create a new test root and populate the test tree.
             // Note: populateTestTree will call testItemIndex.registerTestItem() for each discovered test
-            populateTestTree(
+            return populateTestTree(
                 testController,
                 rawTestData.tests,
                 undefined,
@@ -74,6 +78,8 @@ export class TestDiscoveryHandler {
                 projectName,
             );
         }
+
+        return 0;
     }
 
     /**

src/client/testing/testController/common/types.ts

@@ -17,6 +17,7 @@ import { ITestDebugLauncher } from '../../common/types';
 import { IPythonExecutionFactory } from '../../../common/process/types';
 import { PythonEnvironment } from '../../../pythonEnvironments/info';
 import { ProjectAdapter } from './projectAdapter';
+import { DiscoveryTriggerKind } from './discoveryTelemetry';
 
 export enum TestDataKinds {
     Workspace,
@@ -34,7 +35,10 @@ export interface TestData {
     kind: TestDataKinds;
 }
 
-export type TestRefreshOptions = { forceRefresh: boolean };
+export type TestRefreshOptions = {
+    forceRefresh: boolean;
+    trigger?: DiscoveryTriggerKind;
+};
 
 export const ITestController = Symbol('ITestController');
 export interface ITestController {

src/client/testing/testController/common/utils.ts

@@ -223,6 +223,10 @@ export function buildErrorNodeOptions(
     };
 }
 
+/**
+ * Populates the VS Code test tree from discovered test data.
+ * @returns The number of leaf test items added or updated while walking the tree.
+ */
 export function populateTestTree(
     testController: TestController,
     testTreeData: DiscoveredTestNode,
@@ -231,7 +235,10 @@ export function populateTestTree(
     token?: CancellationToken,
     projectId?: string,
     projectName?: string,
-): void {
+): number {
+    // Count leaf tests while walking the tree so telemetry does not need a second traversal.
+    let testCount = 0;
+
     // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller.
     if (!testRoot) {
         // Create project-scoped ID if projectId is provided
@@ -275,6 +282,7 @@ export function populateTestTree(
                 testItemMappings.runIdToTestItem.set(child.runID, testItem);
                 testItemMappings.runIdToVSid.set(child.runID, vsId);
                 testItemMappings.vsIdToRunId.set(vsId, child.runID);
+                testCount += 1;
             } else {
                 // Use project-scoped ID for non-test nodes and look up within the current root
                 const nodeId = projectId ? `${projectId}${PROJECT_ID_SEPARATOR}${child.id_}` : child.id_;
@@ -302,10 +310,20 @@ export function populateTestTree(
 
                     testRoot!.children.add(node);
                 }
-                populateTestTree(testController, child, node, testItemMappings, token, projectId, projectName);
+                testCount += populateTestTree(
+                    testController,
+                    child,
+                    node,
+                    testItemMappings,
+                    token,
+                    projectId,
+                    projectName,
+                );
             }
         }
     });
+
+    return testCount;
 }
 
 function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem {