Add env var to enable/disable custom dimensions truncation limit

sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md

@@ -7,6 +7,8 @@
 ### Breaking Changes
 
 ### Bugs Fixed
+- Add environment variable to disable/enable custom properties truncation
+  ([#45479](https://github.com/Azure/azure-sdk-for-python/pull/45479))
 - Fix io counters import issue in performance counters
   ([#45286](https://github.com/Azure/azure-sdk-for-python/pull/45286))
 - Remove custom properties truncation

sdk/monitor/azure-monitor-opentelemetry-exporter/README.md

@@ -139,6 +139,10 @@ All configuration options can be passed through the constructors of exporters th
 * `storage_directory`: Storage directory in which to store retry files. Defaults to `<tempfile.gettempdir()>/Microsoft/AzureMonitor/opentelemetry-python-<your-instrumentation-key>`.
 * `credential`: Token credential, such as ManagedIdentityCredential or ClientSecretCredential, used for [Azure Active Directory (AAD) authentication][aad_for_ai_docs]. Defaults to None. See [samples][exporter_samples] for examples. The credential will be automatically created from the `APPLICATIONINSIGHTS_AUTHENTICATION_STRING` environment variable if not explicitly passed in. See [documentation][aad_env_var_docs] for more.
 
+## Environment Variables
+
+* Set `AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT` to `True` to remove the 64kb truncation limit on custom dimensions. Defaults to `False`.
+
 ## Examples
 
 ### Logging (experimental)

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py

@@ -353,4 +353,7 @@ class _RP_Names(Enum):
 # Resource attribute applicationId
 _APPLICATION_ID_RESOURCE_KEY = "microsoft.applicationId"
 
+# Custom dimensions limit truncation toggle
+AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT = "AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT"
+
 # cSpell:disable

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py

@@ -33,6 +33,7 @@
     _KUBERNETES_SERVICE_HOST,
     _PYTHON_APPLICATIONINSIGHTS_ENABLE_TELEMETRY,
     _WEBSITE_SITE_NAME,
+    AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT,
 )
 from azure.monitor.opentelemetry.exporter._constants import (
     _TYPE_MAP,
@@ -353,20 +354,27 @@ def _is_any_synthetic_source(properties: Optional[Any]) -> bool:
 
 # pylint: disable=W0622
 def _filter_custom_properties(properties: Attributes, filter=None) -> Dict[str, str]:
-    filtered_properties: Dict[str, str] = {}
+    disable_custom_dimensions_limit = (
+        environ.get(AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT, "").strip().lower() == "true"
+    )
+    processed_properties: Dict[str, str] = {}
     if not properties:
-        return filtered_properties
+        return processed_properties
     for key, val in properties.items():
         # Apply filter function
         if filter is not None:
             if not filter(key, val):
                 continue
-        # Apply filtering rules
+        # Apply truncation/filtering rules
         # Max key length is 150
         if not key or len(key) > 150 or val is None:
             continue
-        filtered_properties[key] = str(val)
-    return filtered_properties
+        if disable_custom_dimensions_limit:
+            processed_properties[key] = str(val)
+        else:
+            max_length = 64 * 1024
+            processed_properties[key] = str(val)[:max_length]
+    return processed_properties
 
 
 def _get_auth_policy(credential, default_auth_policy, aad_audience=None):

sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_utils.py

@@ -36,8 +36,9 @@ def setUp(self):
         self._valid_instrumentation_key = "1234abcd-5678-4efa-8abc-1234567890ab"
 
     def test_filter_custom_properties_drops_invalid_entries(self):
+        oversized_value = "v" * 9000
         properties = {
-            "valid_key": "valid_value",
+            "valid_key": oversized_value,
             "k" * 151: "should_be_dropped",
             "": "missing_key",
             "short": "ok",
@@ -48,22 +49,44 @@ def test_filter_custom_properties_drops_invalid_entries(self):
 
         self.assertEqual(len(filtered), 2)
         self.assertIn("valid_key", filtered)
-        self.assertEqual(filtered["valid_key"], "valid_value")
+        self.assertEqual(len(filtered["valid_key"]), 9000)
         self.assertEqual(filtered["short"], "ok")
         self.assertNotIn("k" * 151, filtered)
 
-    def test_filter_custom_properties_preserves_large_values(self):
-        # Ensure values larger than 64KiB are not truncated
+    def test_filter_custom_properties_preserves_large_values_after_disable_limit(self):
+        # Ensure values larger than 64KiB are not truncated when the env variable is set to true
+        enable_values = ["true", "True", "TRUE", "TrUE", "  true  "]
         large_value = "x" * (64 * 1024 + 1000)
-        properties = {
-            "large_key": large_value,
-        }
-
-        filtered = _utils._filter_custom_properties(properties)
-
-        self.assertIn("large_key", filtered)
-        self.assertEqual(filtered["large_key"], large_value)
-        self.assertEqual(len(filtered["large_key"]), 64 * 1024 + 1000)
+        properties = {"large_key": large_value}
+
+        for env_value in enable_values:
+            with self.subTest(env_value=env_value):
+                with patch.dict(
+                    "azure.monitor.opentelemetry.exporter._utils.environ",
+                    {"AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT": env_value},
+                ):
+                    filtered = _utils._filter_custom_properties(properties)
+                    self.assertIn("large_key", filtered)
+                    self.assertEqual(filtered["large_key"], large_value)
+                    self.assertEqual(len(filtered["large_key"]), 64 * 1024 + 1000)
+
+    def test_filter_custom_properties_preserves_large_values_after_enable_limit(self):
+        # Ensure values larger than 64KiB are not truncated when the env variable is set to false/empty/invalid
+        disable_values = ["", "False", "truthy", "89", "fALSE", " "]
+        large_value = "x" * (64 * 1024 + 1000)
+        max_length = 64 * 1024
+        properties = {"large_key": large_value}
+
+        for env_value in disable_values:
+            with self.subTest(env_value=env_value):
+                with patch.dict(
+                    "azure.monitor.opentelemetry.exporter._utils.environ",
+                    {"AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT": env_value},
+                ):
+                    filtered = _utils._filter_custom_properties(properties)
+                    self.assertIn("large_key", filtered)
+                    self.assertEqual(filtered["large_key"], "x" * max_length)
+                    self.assertEqual(len(filtered["large_key"]), max_length)
 
     def test_nanoseconds_to_duration(self):
         ns_to_duration = _utils.ns_to_duration