feat: Automatically populate uuid4 fields (#1985)

Co-authored-by: Victor Chudnovsky <vchudnov@google.com>

Author: parthea

packages/gapic-generator/.github/workflows/tests.yaml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  SHOWCASE_VERSION: 0.31.0
+  SHOWCASE_VERSION: 0.32.0
   PROTOC_VERSION: 3.20.2
 
 jobs:

packages/gapic-generator/gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/client.py.j2

@@ -6,6 +6,9 @@ from collections import OrderedDict
 import os
 import re
 from typing import Callable, Dict, Mapping, MutableMapping, MutableSequence, Optional, {% if service.any_server_streaming %}Iterable, {% endif %}{% if service.any_client_streaming %}Iterator, {% endif %}Sequence, Tuple, Type, Union, cast
+{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|list %}
+import uuid
+{% endif %}
 {% if service.any_deprecated %}
 import warnings
 {% endif %}
@@ -473,6 +476,27 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
         )
         {% endif %}
 
+{#
+    Automatically populate UUID4 fields according to
+    https://google.aip.dev/client-libraries/4235 when the
+    field satisfies either of:
+    - The field supports explicit presence and has not been set by the user.
+    - The field doesn't support explicit presence, and its value is the empty
+      string (i.e. the default value).
+#}
+{% with method_settings = api.all_method_settings.get(method.meta.address.proto) %}
+{% if method_settings is not none %}
+{% for auto_populated_field in method_settings.auto_populated_fields %}
+    {% if method.input.fields[auto_populated_field].proto3_optional %}
+        if '{{ auto_populated_field }}' not in request:
+    {% else %}
+        if not request.{{ auto_populated_field }}:
+    {% endif %}
+            request.{{ auto_populated_field }} = str(uuid.uuid4())
+{% endfor %}
+{% endif %}{# if method_settings is not none #}
+{% endwith %}{# method_settings #}
+
         # Send the request.
         {%+ if not method.void %}response = {% endif %}rpc(
             {% if not method.client_streaming %}

packages/gapic-generator/gapic/ads-templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2

@@ -3,6 +3,9 @@
 {% block content %}
 
 import os
+{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|list %}
+import re
+{% endif %}
 # try/except added for compatibility with python < 3.8
 try:
     from unittest import mock
@@ -67,6 +70,7 @@ from google.iam.v1 import policy_pb2  # type: ignore
 {% endif %}
 {% endfilter %}
 
+{% with uuid4_re = "[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}" %}
 
 def client_cert_source_callback():
     return b"cert bytes", b"key bytes"
@@ -513,6 +517,7 @@ def test_{{ service.client_name|snake_case }}_create_channel_credentials_file(cl
   dict,
 ])
 def test_{{ method_name }}(request_type, transport: str = 'grpc'):
+    {% with auto_populated_field_sample_value = "explicit value for autopopulate-able field" %}
     client = {{ service.client_name }}(
         credentials=ga_credentials.AnonymousCredentials(),
         transport=transport,
@@ -521,6 +526,18 @@ def test_{{ method_name }}(request_type, transport: str = 'grpc'):
     # Everything is optional in proto3 as far as the runtime is concerned,
     # and we are mocking out the actual API, so just send an empty request.
     request = request_type()
+
+    {# Set UUID4 fields so that they are not automatically populated. #}
+    {% with method_settings = api.all_method_settings.get(method.meta.address.proto) %}
+    {% if method_settings is not none %}
+    {% for auto_populated_field in method_settings.auto_populated_fields %}
+    if isinstance(request, dict):
+        request['{{ auto_populated_field }}'] = "{{ auto_populated_field_sample_value }}"
+    else:
+        request.{{ auto_populated_field }} = "{{ auto_populated_field_sample_value }}"
+    {% endfor %}
+    {% endif %}{# if method_settings is not none #}
+    {% endwith %}{# method_settings #}
     {% if method.client_streaming %}
     requests = [request]
     {% endif %}
@@ -568,7 +585,15 @@ def test_{{ method_name }}(request_type, transport: str = 'grpc'):
         {% if method.client_streaming %}
         assert next(args[0]) == request
         {% else %}
-        assert args[0] == {{ method.input.ident }}()
+        request = {{ method.input.ident }}()
+        {% with method_settings = api.all_method_settings.get(method.meta.address.proto) %}
+        {% if method_settings is not none %}
+        {% for auto_populated_field in method_settings.auto_populated_fields %}
+        request.{{ auto_populated_field }} = "{{ auto_populated_field_sample_value }}"
+        {% endfor %}
+        {% endif %}{# if method_settings is not none #}
+        {% endwith %}{# method_settings #}
+        assert args[0] == request
         {% endif %}
 
     # Establish that the response is the type that we expect.
@@ -608,6 +633,7 @@ def test_{{ method_name }}(request_type, transport: str = 'grpc'):
     {% endif %}{# end oneof/optional #}
     {% endfor %}
     {% endif %}
+    {% endwith %}{# auto_populated_field_sample_value #}
 
 
 {% if not method.client_streaming %}
@@ -629,8 +655,59 @@ def test_{{ method_name }}_empty_call():
         {% if method.client_streaming %}
         assert next(args[0]) == request
         {% else %}
+{% with method_settings = api.all_method_settings.get(method.meta.address.proto) %}
+{% if method_settings is not none %}
+{% for auto_populated_field in method_settings.auto_populated_fields %}
+        # Ensure that the uuid4 field is set according to AIP 4235
+        assert re.match(r"{{ uuid4_re }}", args[0].{{ auto_populated_field }})
+        # clear UUID field so that the check below succeeds
+        args[0].{{ auto_populated_field }} = None
+{% endfor %}
+{% endif %}{# if method_settings is not none #}
+{% endwith %}{# method_settings #}
         assert args[0] == {{ method.input.ident }}()
         {% endif %}
+
+
+def test_{{ method_name }}_non_empty_request_with_auto_populated_field():
+    # This test is a coverage failsafe to make sure that UUID4 fields are
+    # automatically populated, according to AIP-4235, with non-empty requests.
+    client = {{ service.client_name }}(
+        credentials=ga_credentials.AnonymousCredentials(),
+        transport='grpc',
+    )
+
+    # Populate all string fields in the request which are not UUID4
+    # since we want to check that UUID4 are populated automatically
+    # if they meet the requirements of AIP 4235.
+    request = {{ method.input.ident }}(
+    {% for field in method.input.fields.values() if field.ident|string() == "str" and not field.uuid4 %}
+        {{ field.name }}={{ field.mock_value }},
+    {% endfor %}
+    )
+
+    # Mock the actual call within the gRPC stub, and fake the request.
+    with mock.patch.object(
+            type(client.transport.{{ method.transport_safe_name|snake_case }}),
+            '__call__') as call:
+        client.{{ method_name }}(request=request)
+        call.assert_called()
+        _, args, _ = call.mock_calls[0]
+{% with method_settings = api.all_method_settings.get(method.meta.address.proto) %}
+{% if method_settings is not none %}
+{% for auto_populated_field in method_settings.auto_populated_fields %}
+        # Ensure that the uuid4 field is set according to AIP 4235
+        assert re.match(r"{{ uuid4_re }}", args[0].{{ auto_populated_field }})
+        # clear UUID field so that the check below succeeds
+        args[0].{{ auto_populated_field }} = None
+{% endfor %}
+{% endif %}{# if method_settings is not none #}
+{% endwith %}{# method_settings #}
+        assert args[0] == {{ method.input.ident }}(
+        {% for field in method.input.fields.values() if field.ident|string() == "str" and not field.uuid4 %}
+            {{ field.name }}={{ field.mock_value }},
+        {% endfor %}
+        )
 {% endif %}
 
 
@@ -2364,4 +2441,5 @@ def test_client_ctx():
                 pass
             close.assert_called()
 
+{% endwith %}{# uuid4_re #}
 {% endblock %}

packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_client_macros.j2

@@ -183,6 +183,8 @@
         )
         {% endif %} {# method.explicit_routing #}
 
+{{ auto_populate_uuid4_fields(api, method) }}
+
         # Validate the universe domain.
         self._validate_universe_domain()
 
@@ -265,3 +267,27 @@
 
 {% macro define_extended_operation_subclass(extended_operation) %}
 {% endmacro %}
+
+{% macro auto_populate_uuid4_fields(api, method) %}
+{#
+    Automatically populate UUID4 fields according to
+    https://google.aip.dev/client-libraries/4235 when the
+    field satisfies either of:
+    - The field supports explicit presence and has not been set by the user.
+    - The field doesn't support explicit presence, and its value is the empty
+      string (i.e. the default value).
+    When using this macro, ensure the calling template generates a line `import uuid`
+#}
+{% with method_settings = api.all_method_settings.get(method.meta.address.proto) %}
+{% if method_settings is not none %}
+{% for auto_populated_field in method_settings.auto_populated_fields %}
+    {% if method.input.fields[auto_populated_field].proto3_optional %}
+        if '{{ auto_populated_field }}' not in request:
+    {% else %}
+        if not request.{{ auto_populated_field }}:
+    {% endif %}
+            request.{{ auto_populated_field }} = str(uuid.uuid4())
+{% endfor %}
+{% endif %}{# if method_settings is not none #}
+{% endwith %}{# method_settings #}
+{% endmacro %}

packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2

@@ -1,11 +1,15 @@
 {% extends "_base.py.j2" %}
 
 {% block content %}
+{% import "%namespace/%name_%version/%sub/services/%service/_client_macros.j2" as macros %}
 
 from collections import OrderedDict
 import functools
 import re
 from typing import Dict, Mapping, MutableMapping, MutableSequence, Optional, {% if service.any_server_streaming %}AsyncIterable, Awaitable, {% endif %}{% if service.any_client_streaming %}AsyncIterator, {% endif %}Sequence, Tuple, Type, Union
+{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|list %}
+import uuid
+{% endif %}
 {% if service.any_deprecated %}
 import warnings
 {% endif %}
@@ -386,6 +390,8 @@ class {{ service.async_client_name }}:
         )
         {% endif %}
 
+{{ macros.auto_populate_uuid4_fields(api, method) }}
+
         # Validate the universe domain.
         self._client._validate_universe_domain()
 

packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2

@@ -10,6 +10,9 @@ import functools
 import os
 import re
 from typing import Dict, Mapping, MutableMapping, MutableSequence, Optional, {% if service.any_server_streaming %}Iterable, {% endif %}{% if service.any_client_streaming %}Iterator, {% endif %}Sequence, Tuple, Type, Union, cast
+{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|list %}
+import uuid
+{% endif %}
 import warnings
 
 {% set package_path = api.naming.module_namespace|join('.') + "." + api.naming.versioned_module_name %}

packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2

@@ -4,6 +4,9 @@
 {% import "tests/unit/gapic/%name_%version/%sub/test_macros.j2" as test_macros %}
 
 import os
+{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|list %}
+import re
+{% endif %}
 # try/except added for compatibility with python < 3.8
 try:
     from unittest import mock
@@ -849,10 +852,10 @@ def test_{{ service.client_name|snake_case }}_create_channel_credentials_file(cl
 
 {% for method in service.methods.values() if 'grpc' in opts.transport %}{# method_name #}
 {% if method.extended_lro %}
-{{ test_macros.grpc_required_tests(method, service, full_extended_lro=True) }}
+{{ test_macros.grpc_required_tests(method, service, api, full_extended_lro=True) }}
 
 {% endif %}
-{{ test_macros.grpc_required_tests(method, service) }}
+{{ test_macros.grpc_required_tests(method, service, api) }}
 {% endfor %} {# method in methods for grpc #}
 
 {% for method in service.methods.values() if 'rest' in opts.transport %}