diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md index f5d0a58a5d24..9265839abe9e 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md @@ -7,6 +7,8 @@ ### Breaking Changes ### Bugs Fixed +- Add environment variable to disable/enable custom properties truncation + ([#45479](https://github.com/Azure/azure-sdk-for-python/pull/45479)) - Fix io counters import issue in performance counters ([#45286](https://github.com/Azure/azure-sdk-for-python/pull/45286)) - Remove custom properties truncation diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/README.md b/sdk/monitor/azure-monitor-opentelemetry-exporter/README.md index 720a1e5bb027..b4bb06afad30 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/README.md +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/README.md @@ -139,6 +139,10 @@ All configuration options can be passed through the constructors of exporters th * `storage_directory`: Storage directory in which to store retry files. Defaults to `/Microsoft/AzureMonitor/opentelemetry-python-`. * `credential`: Token credential, such as ManagedIdentityCredential or ClientSecretCredential, used for [Azure Active Directory (AAD) authentication][aad_for_ai_docs]. Defaults to None. See [samples][exporter_samples] for examples. The credential will be automatically created from the `APPLICATIONINSIGHTS_AUTHENTICATION_STRING` environment variable if not explicitly passed in. See [documentation][aad_env_var_docs] for more. +## Environment Variables + +* Set `AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT` to `True` to remove the 64kb truncation limit on custom dimensions. Defaults to `False`. + ## Examples ### Logging (experimental) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py index 11ec9db31f53..0dcf6c57c200 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py @@ -353,4 +353,7 @@ class _RP_Names(Enum): # Resource attribute applicationId _APPLICATION_ID_RESOURCE_KEY = "microsoft.applicationId" +# Custom dimensions limit truncation toggle +AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT = "AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT" + # cSpell:disable diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py index dd354ae80773..4f6512b04d30 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_utils.py @@ -33,6 +33,7 @@ _KUBERNETES_SERVICE_HOST, _PYTHON_APPLICATIONINSIGHTS_ENABLE_TELEMETRY, _WEBSITE_SITE_NAME, + AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT, ) from azure.monitor.opentelemetry.exporter._constants import ( _TYPE_MAP, @@ -353,20 +354,27 @@ def _is_any_synthetic_source(properties: Optional[Any]) -> bool: # pylint: disable=W0622 def _filter_custom_properties(properties: Attributes, filter=None) -> Dict[str, str]: - filtered_properties: Dict[str, str] = {} + disable_custom_dimensions_limit = ( + environ.get(AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT, "").strip().lower() == "true" + ) + processed_properties: Dict[str, str] = {} if not properties: - return filtered_properties + return processed_properties for key, val in properties.items(): # Apply filter function if filter is not None: if not filter(key, val): continue - # Apply filtering rules + # Apply truncation/filtering rules # Max key length is 150 if not key or len(key) > 150 or val is None: continue - filtered_properties[key] = str(val) - return filtered_properties + if disable_custom_dimensions_limit: + processed_properties[key] = str(val) + else: + max_length = 64 * 1024 + processed_properties[key] = str(val)[:max_length] + return processed_properties def _get_auth_policy(credential, default_auth_policy, aad_audience=None): diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_utils.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_utils.py index c92f8dc55705..23c15fddb614 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_utils.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_utils.py @@ -36,8 +36,9 @@ def setUp(self): self._valid_instrumentation_key = "1234abcd-5678-4efa-8abc-1234567890ab" def test_filter_custom_properties_drops_invalid_entries(self): + oversized_value = "v" * 9000 properties = { - "valid_key": "valid_value", + "valid_key": oversized_value, "k" * 151: "should_be_dropped", "": "missing_key", "short": "ok", @@ -48,22 +49,44 @@ def test_filter_custom_properties_drops_invalid_entries(self): self.assertEqual(len(filtered), 2) self.assertIn("valid_key", filtered) - self.assertEqual(filtered["valid_key"], "valid_value") + self.assertEqual(len(filtered["valid_key"]), 9000) self.assertEqual(filtered["short"], "ok") self.assertNotIn("k" * 151, filtered) - def test_filter_custom_properties_preserves_large_values(self): - # Ensure values larger than 64KiB are not truncated + def test_filter_custom_properties_preserves_large_values_after_disable_limit(self): + # Ensure values larger than 64KiB are not truncated when the env variable is set to true + enable_values = ["true", "True", "TRUE", "TrUE", " true "] large_value = "x" * (64 * 1024 + 1000) - properties = { - "large_key": large_value, - } - - filtered = _utils._filter_custom_properties(properties) - - self.assertIn("large_key", filtered) - self.assertEqual(filtered["large_key"], large_value) - self.assertEqual(len(filtered["large_key"]), 64 * 1024 + 1000) + properties = {"large_key": large_value} + + for env_value in enable_values: + with self.subTest(env_value=env_value): + with patch.dict( + "azure.monitor.opentelemetry.exporter._utils.environ", + {"AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT": env_value}, + ): + filtered = _utils._filter_custom_properties(properties) + self.assertIn("large_key", filtered) + self.assertEqual(filtered["large_key"], large_value) + self.assertEqual(len(filtered["large_key"]), 64 * 1024 + 1000) + + def test_filter_custom_properties_preserves_large_values_after_enable_limit(self): + # Ensure values larger than 64KiB are not truncated when the env variable is set to false/empty/invalid + disable_values = ["", "False", "truthy", "89", "fALSE", " "] + large_value = "x" * (64 * 1024 + 1000) + max_length = 64 * 1024 + properties = {"large_key": large_value} + + for env_value in disable_values: + with self.subTest(env_value=env_value): + with patch.dict( + "azure.monitor.opentelemetry.exporter._utils.environ", + {"AZURE_MONITOR_DISABLE_CUSTOM_DIMENSIONS_LIMIT": env_value}, + ): + filtered = _utils._filter_custom_properties(properties) + self.assertIn("large_key", filtered) + self.assertEqual(filtered["large_key"], "x" * max_length) + self.assertEqual(len(filtered["large_key"]), max_length) def test_nanoseconds_to_duration(self): ns_to_duration = _utils.ns_to_duration