[Monitor OpenTelemetry Exporter] Add Further Customer Statsbeat Codes (#35004)

### Packages impacted by this PR
@azure/monitor-opentelemetry-exporter

### Issues associated with this PR


### Describe the problem that is addressed by this PR
We should cover read-only access and client timeout issues in customer
statsbeat.

### Are there test cases added in this PR? _(If not, why?)_
Yes

### Checklists
- [x] Added impacted package name to the issue description
- [ ] Does this PR needs any fixes in the SDK Generator?** _(If so,
create an Issue in the
[Autorest/typescript](https://github.com/Azure/autorest.typescript)
repository and link it here)_
- [x] Added a changelog (if necessary)

Author: JacksonWeber

sdk/monitor/monitor-opentelemetry-exporter/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## 1.0.0-beta.33 ()
 
+### Features Added
+
+- Track CLIENT_READONLY and CLIENT_TIMEOUT customer statsbeat.
+
 ### Bugs Fixed
 
 - Fix auto-detection of RP environment for azure functions.

sdk/monitor/monitor-opentelemetry-exporter/src/export/statsbeat/customerStatsbeat.ts

@@ -537,4 +537,32 @@ export class CustomerStatsbeatMetrics extends StatsbeatMetrics {
     }
     return TelemetryType.UNKNOWN;
   }
+
+  /**
+   * Checks if the given error is a timeout-related error
+   * @param error - The error to check
+   * @returns true if the error is timeout-related, false otherwise
+   */
+  public isTimeoutError(error: { code?: string; message?: string }): boolean {
+    // Check for various timeout error codes that indicate client timeouts
+    const timeoutErrorCodes = [
+      "ETIMEDOUT", // Connection timed out
+      "ESOCKETTIMEDOUT", // Socket timeout
+      "ECONNRESET", // Connection reset (often due to timeout)
+      "ENOTFOUND", // DNS lookup failed/timeout
+    ];
+
+    if (error && error.code && timeoutErrorCodes.includes(error.code)) {
+      return true;
+    }
+
+    // Also check if the error message contains timeout-related keywords
+    if (error && error.message) {
+      const timeoutKeywords = ["timeout", "timed out", "connection reset"];
+      const errorMessage = error.message.toLowerCase();
+      return timeoutKeywords.some((keyword) => errorMessage.includes(keyword));
+    }
+
+    return false;
+  }
 }

sdk/monitor/monitor-opentelemetry-exporter/src/platform/nodejs/baseSender.ts

@@ -12,6 +12,7 @@ import { LongIntervalStatsbeatMetrics } from "../../export/statsbeat/longInterva
 import type { RestError } from "@azure/core-rest-pipeline";
 import {
   DropCode,
+  RetryCode,
   MAX_STATSBEAT_FAILURES,
   isStatsbeatShutdownStatus,
 } from "../../export/statsbeat/types.js";
@@ -259,23 +260,29 @@ export abstract class BaseSender {
         this.incrementStatsbeatFailure();
         return { code: ExportResultCode.SUCCESS };
       }
-      if (this.isRetriableRestError(restError)) {
-        if (restError.statusCode && !this.isStatsbeatSender) {
+
+      // For retriable REST errors
+      if (this.isRetriableRestError(restError) && !this.isStatsbeatSender) {
+        if (this.customerStatsbeatMetrics?.isTimeoutError(restError) && !this.isStatsbeatSender) {
+          this.customerStatsbeatMetrics?.countRetryItems(
+            envelopes,
+            RetryCode.CLIENT_TIMEOUT,
+            "timeout_exception",
+          );
+          diag.error("Request timed out. Error message:", restError.message);
+        } else if (restError.statusCode) {
           this.networkStatsbeatMetrics?.countRetry(restError.statusCode);
           this.customerStatsbeatMetrics?.countRetryItems(envelopes, restError.statusCode);
         }
-        if (!this.isStatsbeatSender) {
-          diag.error(
-            "Retrying due to transient client side error. Error message:",
-            restError.message,
-          );
-        }
+        diag.error(
+          "Retrying due to transient client side error. Error message:",
+          restError.message,
+        );
         return this.persist(envelopes);
       }
+      // For non-retriable REST errors or client exceptions
       if (!this.isStatsbeatSender) {
         this.networkStatsbeatMetrics?.countException(restError);
-      }
-      if (!this.isStatsbeatSender) {
         diag.error(
           "Envelopes could not be exported and are not retriable. Error message:",
           restError.message,

sdk/monitor/monitor-opentelemetry-exporter/src/platform/nodejs/persist/fileSystemPersist.ts

@@ -10,7 +10,7 @@ import { confirmDirExists, getShallowDirectorySize } from "./fileSystemHelpers.j
 import type { AzureMonitorExporterOptions } from "../../../config.js";
 import { readdir, readFile, stat, unlink, writeFile } from "node:fs/promises";
 import type { CustomerStatsbeatMetrics } from "../../../export/statsbeat/customerStatsbeat.js";
-import { DropCode, TelemetryType } from "../../../export/statsbeat/types.js";
+import { DropCode } from "../../../export/statsbeat/types.js";
 import type { TelemetryItem as Envelope } from "../../../generated/index.js";
 
 /**
@@ -152,7 +152,16 @@ export class FileSystemPersist implements PersistentStorage {
     try {
       await confirmDirExists(this._tempDirectory);
     } catch (error: any) {
-      diag.warn(`Error while checking/creating directory: `, error && error.message);
+      // Check if error is due to permission/readonly issues
+      if (error?.code === "EACCES" || error?.code === "EPERM") {
+        this._customerStatsbeatMetrics?.countDroppedItems(envelopes, DropCode.CLIENT_READONLY);
+        diag.warn(
+          `Permission denied while checking/creating directory: ${this._tempDirectory}`,
+          error?.message,
+        );
+      } else {
+        diag.warn(`Error while checking/creating directory: `, error && error.message);
+      }
       return false;
     }
 
@@ -163,7 +172,6 @@ export class FileSystemPersist implements PersistentStorage {
         this._customerStatsbeatMetrics?.countDroppedItems(
           envelopes,
           DropCode.CLIENT_PERSISTENCE_CAPACITY,
-          TelemetryType.UNKNOWN,
         );
         diag.warn(
           `Not saving data due to max size limit being met. Directory size in bytes is: ${size}`,
@@ -186,8 +194,8 @@ export class FileSystemPersist implements PersistentStorage {
       // If the envelopes cannot be written to disk, we send customer statsbeat and warn the user
       this._customerStatsbeatMetrics?.countDroppedItems(
         envelopes,
-        DropCode.CLIENT_STORAGE_DISABLED,
-        TelemetryType.UNKNOWN,
+        DropCode.CLIENT_EXCEPTION,
+        writeError?.message,
       );
       diag.warn(`Error writing file to persistent file storage`, writeError);
       return false;

sdk/monitor/monitor-opentelemetry-exporter/test/internal/baseSender.spec.ts

@@ -96,6 +96,7 @@ export const mockCustomerStatsbeatMetrics = {
   countSuccessfulItems: vi.fn(),
   countDroppedItems: vi.fn(),
   countRetryItems: vi.fn(),
+  isTimeoutError: vi.fn(),
   shutdown: vi.fn(),
 };
 
@@ -488,6 +489,9 @@ describe("BaseSender", () => {
       const nonRetriableError: any = new Error("Bad request");
       nonRetriableError.statusCode = 400;
 
+      // Mock isTimeoutError to return false so timeout logic isn't triggered
+      mockCustomerStatsbeatMetrics.isTimeoutError.mockReturnValue(false);
+
       sender.sendMock.mockRejectedValue(nonRetriableError);
 
       const result = await sender.exportEnvelopes([{ name: "test", time: new Date() }]);

sdk/monitor/monitor-opentelemetry-exporter/test/internal/fileSystemPersist.spec.ts

@@ -1,14 +1,16 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+import { describe, it, assert, expect, beforeEach, vi, afterEach } from "vitest";
+
 import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import { FileSystemPersist } from "../../src/platform/nodejs/persist/fileSystemPersist.js";
 import type { TelemetryItem as Envelope } from "../../src/generated/index.js";
 import { promisify } from "node:util";
 import { FileAccessControl } from "../../src/platform/nodejs/persist/fileAccessControl.js";
-import { describe, it, assert, expect, beforeEach } from "vitest";
+import { DropCode } from "../../src/export/statsbeat/types.js";
 
 const statAsync = promisify(fs.stat);
 const readdirAsync = promisify(fs.readdir);
@@ -199,4 +201,67 @@ describe("FileSystemPersist", () => {
       assert.deepStrictEqual(fileValue, null, "File is still present"); // File doesn't exist anymore
     });
   });
+
+  describe("#CLIENT_READONLY scenarios", () => {
+    let mockCustomerStatsbeat: any;
+    let originalOSFileProtection: boolean;
+
+    beforeEach(() => {
+      mockCustomerStatsbeat = {
+        countDroppedItems: vi.fn(),
+      };
+
+      // Store original value and enable file protection for tests
+      originalOSFileProtection = FileAccessControl.OS_PROVIDES_FILE_PROTECTION;
+      FileAccessControl.OS_PROVIDES_FILE_PROTECTION = true;
+    });
+
+    afterEach(() => {
+      vi.restoreAllMocks();
+      // Restore original value
+      FileAccessControl.OS_PROVIDES_FILE_PROTECTION = originalOSFileProtection;
+    });
+
+    it("should have CLIENT_READONLY logic implemented", () => {
+      const persister = new FileSystemPersist(instrumentationKey, {}, mockCustomerStatsbeat);
+      expect(persister).toBeDefined();
+      expect(DropCode.CLIENT_READONLY).toBeDefined();
+
+      // Verify that our mock is properly set up
+      expect(mockCustomerStatsbeat.countDroppedItems).toBeDefined();
+    });
+
+    it("should track CLIENT_READONLY when permission errors occur", async () => {
+      const envelope: Envelope = {
+        name: "test",
+        time: new Date(),
+      };
+
+      // Import the module dynamically to mock before usage
+      const helpersMod = await import("../../src/platform/nodejs/persist/fileSystemHelpers.js");
+
+      // Mock confirmDirExists to throw EACCES permission error
+      const error = new Error("EACCES: permission denied, mkdir") as NodeJS.ErrnoException;
+      error.code = "EACCES";
+      const mockConfirmDirExists = vi
+        .spyOn(helpersMod, "confirmDirExists")
+        .mockRejectedValue(error);
+
+      const persister = new FileSystemPersist(instrumentationKey, {}, mockCustomerStatsbeat);
+
+      const result = await persister.push([envelope]);
+
+      // Should return false due to permission error
+      expect(result).toBe(false);
+
+      // Should have called countDroppedItems with CLIENT_READONLY
+      expect(mockCustomerStatsbeat.countDroppedItems).toHaveBeenCalledWith(
+        [envelope],
+        DropCode.CLIENT_READONLY,
+      );
+
+      // Restore the spy
+      mockConfirmDirExists.mockRestore();
+    });
+  });
 });