Merge pull request #1952 from tseaver/logging-fix_gax_system_tests

Fix GAX logging system tests

Author: tseaver

gcloud/logging/_gax.py

@@ -154,7 +154,8 @@ def list_sinks(self, project, page_size=0, page_token=None):
                   with another call (pass that value as ``page_token``).
         """
         options = _build_paging_options(page_token)
-        page_iter = self._gax_api.list_sinks(project, page_size, options)
+        path = 'projects/%s' % (project,)
+        page_iter = self._gax_api.list_sinks(path, page_size, options)
         sinks = [_log_sink_pb_to_mapping(log_sink_pb)
                  for log_sink_pb in page_iter.next()]
         token = page_iter.page_token or None
@@ -289,7 +290,8 @@ def list_metrics(self, project, page_size=0, page_token=None):
                   with another call (pass that value as ``page_token``).
         """
         options = _build_paging_options(page_token)
-        page_iter = self._gax_api.list_log_metrics(project, page_size, options)
+        path = 'projects/%s' % (project,)
+        page_iter = self._gax_api.list_log_metrics(path, page_size, options)
         metrics = [_log_metric_pb_to_mapping(log_metric_pb)
                    for log_metric_pb in page_iter.next()]
         token = page_iter.page_token or None
@@ -416,6 +418,41 @@ def _pb_timestamp_to_rfc3339(timestamp_pb):
     return _datetime_to_rfc3339(timestamp)
 
 
+def _value_pb_to_value(value_pb):
+    """Helper for :func:`_log_entry_pb_to_mapping`."""
+    kind = value_pb.WhichOneof('kind')
+
+    if kind is None:
+        result = None
+
+    elif kind == 'string_value':
+        result = value_pb.string_value
+
+    elif kind == 'bool_value':
+        result = value_pb.bool_value
+
+    elif kind == 'number_value':
+        result = value_pb.number_value
+
+    elif kind == 'list_value':
+        result = [_value_pb_to_value(element)
+                  for element in value_pb.list_value.values]
+
+    elif kind == 'struct_value':
+        result = _struct_pb_to_mapping(value_pb.struct_value)
+
+    else:
+        raise ValueError('Value protobuf had unknown kind: %s' % (kind,))
+
+    return result
+
+
+def _struct_pb_to_mapping(struct_pb):
+    """Helper for :func:`_log_entry_pb_to_mapping`."""
+    return dict([(key, _value_pb_to_value(struct_pb.fields[key]))
+                 for key in struct_pb.fields])
+
+
 def _log_entry_pb_to_mapping(entry_pb):
     """Helper for :meth:`list_entries`, et aliae
 
@@ -426,27 +463,32 @@ def _log_entry_pb_to_mapping(entry_pb):
     mapping = {
         'logName': entry_pb.log_name,
         'resource': _mon_resource_pb_to_mapping(entry_pb.resource),
-        'severity': entry_pb.severity,
+        'severity': LogSeverity.Name(entry_pb.severity),
         'insertId': entry_pb.insert_id,
         'timestamp': _pb_timestamp_to_rfc3339(entry_pb.timestamp),
         'labels': entry_pb.labels,
-        'textPayload': entry_pb.text_payload,
-        'jsonPayload': entry_pb.json_payload,
-        'protoPayload': entry_pb.proto_payload,
     }
+    if entry_pb.HasField('text_payload'):
+        mapping['textPayload'] = entry_pb.text_payload
+
+    if entry_pb.HasField('json_payload'):
+        mapping['jsonPayload'] = _struct_pb_to_mapping(entry_pb.json_payload)
+
+    if entry_pb.HasField('proto_payload'):
+        mapping['protoPayload'] = entry_pb.proto_payload
 
     if entry_pb.http_request:
         request = entry_pb.http_request
         mapping['httpRequest'] = {
-            'request_method': request.request_method,
-            'request_url': request.request_url,
+            'requestMethod': request.request_method,
+            'requestUrl': request.request_url,
             'status': request.status,
             'referer': request.referer,
-            'user_agent': request.user_agent,
-            'cache_hit': request.cache_hit,
-            'request_size': request.request_size,
-            'response_size': request.response_size,
-            'remote_ip': request.remote_ip,
+            'userAgent': request.user_agent,
+            'cacheHit': request.cache_hit,
+            'requestSize': request.request_size,
+            'responseSize': request.response_size,
+            'remoteIp': request.remote_ip,
         }
 
     if entry_pb.operation:

gcloud/logging/test__gax.py

@@ -78,13 +78,16 @@ def test_list_entries_no_paging(self):
         self.assertTrue(options.page_token is INITIAL_PAGE)
 
     def test_list_entries_with_paging(self):
+        from google.protobuf.struct_pb2 import Value
         from gcloud._testing import _GAXPageIterator
         SIZE = 23
         TOKEN = 'TOKEN'
         NEW_TOKEN = 'NEW_TOKEN'
         PAYLOAD = {'message': 'MESSAGE', 'weather': 'sunny'}
+        struct_pb = _StructPB(dict([(key, Value(string_value=value))
+                                    for key, value in PAYLOAD.items()]))
         response = _GAXPageIterator(
-            [_LogEntryPB(self.LOG_NAME, json_payload=PAYLOAD)], NEW_TOKEN)
+            [_LogEntryPB(self.LOG_NAME, json_payload=struct_pb)], NEW_TOKEN)
         gax_api = _GAXLoggingAPI(_list_log_entries_response=response)
         api = self._makeOne(gax_api)
 
@@ -109,6 +112,7 @@ def test_list_entries_with_paging(self):
 
     def test_list_entries_with_extra_properties(self):
         from datetime import datetime
+        from google.logging.type.log_severity_pb2 import WARNING
         from gcloud._testing import _GAXPageIterator
         from gcloud._helpers import UTC
         from gcloud._helpers import _datetime_to_rfc3339
@@ -126,7 +130,7 @@ def test_list_entries_with_extra_properties(self):
         request = _HTTPRequestPB()
         operation = _LogEntryOperationPB()
         EXTRAS = {
-            'severity': SEVERITY,
+            'severity': WARNING,
             'labels': LABELS,
             'insert_id': IID,
             'http_request': request,
@@ -154,15 +158,15 @@ def test_list_entries_with_extra_properties(self):
         self.assertEqual(entry['insertId'], IID)
         self.assertEqual(entry['timestamp'], _datetime_to_rfc3339(NOW))
         EXPECTED_REQUEST = {
-            'request_method': request.request_method,
-            'request_url': request.request_url,
+            'requestMethod': request.request_method,
+            'requestUrl': request.request_url,
             'status': request.status,
-            'request_size': request.request_size,
-            'response_size': request.response_size,
+            'requestSize': request.request_size,
+            'responseSize': request.response_size,
             'referer': request.referer,
-            'user_agent': request.user_agent,
-            'remote_ip': request.remote_ip,
-            'cache_hit': request.cache_hit,
+            'userAgent': request.user_agent,
+            'remoteIp': request.remote_ip,
+            'cacheHit': request.cache_hit,
         }
         self.assertEqual(entry['httpRequest'], EXPECTED_REQUEST)
         EXPECTED_OPERATION = {
@@ -439,7 +443,7 @@ def test_list_sinks_no_paging(self):
         self.assertEqual(token, TOKEN)
 
         project, page_size, options = gax_api._list_sinks_called_with
-        self.assertEqual(project, self.PROJECT)
+        self.assertEqual(project, self.PROJECT_PATH)
         self.assertEqual(page_size, 0)
         self.assertEqual(options.page_token, INITIAL_PAGE)
 
@@ -465,7 +469,7 @@ def test_list_sinks_w_paging(self):
         self.assertEqual(token, None)
 
         project, page_size, options = gax_api._list_sinks_called_with
-        self.assertEqual(project, self.PROJECT)
+        self.assertEqual(project, self.PROJECT_PATH)
         self.assertEqual(page_size, PAGE_SIZE)
         self.assertEqual(options.page_token, TOKEN)
 
@@ -643,7 +647,7 @@ def test_list_metrics_no_paging(self):
         self.assertEqual(token, TOKEN)
 
         project, page_size, options = gax_api._list_log_metrics_called_with
-        self.assertEqual(project, self.PROJECT)
+        self.assertEqual(project, self.PROJECT_PATH)
         self.assertEqual(page_size, 0)
         self.assertEqual(options.page_token, INITIAL_PAGE)
 
@@ -669,7 +673,7 @@ def test_list_metrics_w_paging(self):
         self.assertEqual(token, None)
 
         project, page_size, options = gax_api._list_log_metrics_called_with
-        self.assertEqual(project, self.PROJECT)
+        self.assertEqual(project, self.PROJECT_PATH)
         self.assertEqual(page_size, PAGE_SIZE)
         self.assertEqual(options.page_token, TOKEN)
 
@@ -811,6 +815,75 @@ def test_metric_delete_hit(self):
         self.assertEqual(options, None)
 
 
+@unittest2.skipUnless(_HAVE_GAX, 'No gax-python')
+class Test_value_pb_to_value(_Base, unittest2.TestCase):
+
+    def _callFUT(self, value_pb):
+        from gcloud.logging._gax import _value_pb_to_value
+        return _value_pb_to_value(value_pb)
+
+    def test_w_null_values(self):
+        from google.protobuf.struct_pb2 import Value
+        value_pb = Value()
+        self.assertEqual(self._callFUT(value_pb), None)
+        value_pb = Value(null_value=None)
+        self.assertEqual(self._callFUT(value_pb), None)
+
+    def test_w_string_value(self):
+        from google.protobuf.struct_pb2 import Value
+        STRING = 'STRING'
+        value_pb = Value(string_value=STRING)
+        self.assertEqual(self._callFUT(value_pb), STRING)
+
+    def test_w_bool_values(self):
+        from google.protobuf.struct_pb2 import Value
+        true_value_pb = Value(bool_value=True)
+        self.assertTrue(self._callFUT(true_value_pb) is True)
+        false_value_pb = Value(bool_value=False)
+        self.assertTrue(self._callFUT(false_value_pb) is False)
+
+    def test_w_number_values(self):
+        from google.protobuf.struct_pb2 import Value
+        ANSWER = 42
+        PI = 3.1415926
+        int_value_pb = Value(number_value=ANSWER)
+        self.assertEqual(self._callFUT(int_value_pb), ANSWER)
+        float_value_pb = Value(number_value=PI)
+        self.assertEqual(self._callFUT(float_value_pb), PI)
+
+    def test_w_list_value(self):
+        from google.protobuf.struct_pb2 import Value
+        STRING = 'STRING'
+        PI = 3.1415926
+        value_pb = Value()
+        value_pb.list_value.values.add(string_value=STRING)
+        value_pb.list_value.values.add(bool_value=True)
+        value_pb.list_value.values.add(number_value=PI)
+        self.assertEqual(self._callFUT(value_pb), [STRING, True, PI])
+
+    def test_w_struct_value(self):
+        from google.protobuf.struct_pb2 import Value
+        STRING = 'STRING'
+        PI = 3.1415926
+        value_pb = Value()
+        value_pb.struct_value.fields['string'].string_value = STRING
+        value_pb.struct_value.fields['bool'].bool_value = True
+        value_pb.struct_value.fields['number'].number_value = PI
+        self.assertEqual(self._callFUT(value_pb),
+                         {'string': STRING, 'bool': True, 'number': PI})
+
+    def test_w_unknown_kind(self):
+
+        class _Value(object):
+
+            def WhichOneof(self, name):
+                assert name == 'kind'
+                return 'UNKNOWN'
+
+        with self.assertRaises(ValueError):
+            self._callFUT(_Value())
+
+
 class _GAXBaseAPI(object):
 
     _random_gax_error = False
@@ -974,9 +1047,15 @@ def __init__(self, type_='global', **labels):
         self.labels = labels
 
 
+class _StructPB(object):
+
+    def __init__(self, fields):
+        self.fields = fields
+
+
 class _LogEntryPB(object):
 
-    severity = 'DEFAULT'
+    severity = 0
     http_request = operation = insert_id = None
     text_payload = json_payload = proto_payload = None
 
@@ -987,6 +1066,9 @@ def __init__(self, log_name, **kw):
         self.labels = kw.pop('labels', {})
         self.__dict__.update(kw)
 
+    def HasField(self, field_name):
+        return getattr(self, field_name, None) is not None
+
     @staticmethod
     def _make_timestamp():
         from datetime import datetime