Merge pull request #1128 from tseaver/1125-fix_bq_timestamp_fetch_conversion

Fix conversion of backend-generated timestamps

Author: tseaver

gcloud/bigquery/table.py

@@ -411,7 +411,7 @@ def create(self, client=None):
         """API call:  create the dataset via a PUT request
 
         See:
-        https://cloud.google.com/bigquery/reference/rest/v2/tables/insert
+        https://cloud.google.com/bigquery/docs/reference/v2/tables/insert
 
         :type client: :class:`gcloud.bigquery.client.Client` or ``NoneType``
         :param client: the client to use.  If not passed, falls back to the
@@ -552,7 +552,7 @@ def delete(self, client=None):
         """API call:  delete the table via a DELETE request
 
         See:
-        https://cloud.google.com/bigquery/reference/rest/v2/tables/delete
+        https://cloud.google.com/bigquery/docs/reference/v2/tables/delete
 
         :type client: :class:`gcloud.bigquery.client.Client` or ``NoneType``
         :param client: the client to use.  If not passed, falls back to the
@@ -565,7 +565,7 @@ def fetch_data(self, max_results=None, page_token=None, client=None):
         """API call:  fetch the table data via a GET request
 
         See:
-        https://cloud.google.com/bigquery/reference/rest/v2/tabledata/list
+        https://cloud.google.com/bigquery/docs/reference/v2/tabledata/list
 
         .. note::
 
@@ -631,7 +631,7 @@ def insert_data(self,
         """API call:  insert table data via a POST request
 
         See:
-        https://cloud.google.com/bigquery/reference/rest/v2/tabledata/insertAll
+        https://cloud.google.com/bigquery/docs/reference/v2/tabledata/insertAll
 
         :type rows: list of tuples
         :param rows: row data to be inserted
@@ -761,8 +761,8 @@ def _bool_from_json(value, field):
 
 def _datetime_from_json(value, field):
     if _not_null(value, field):
-        # Field value will be in milliseconds.
-        return _datetime_from_microseconds(1000.0 * float(value))
+        # value will be a float in seconds, to microsecond precision, in UTC.
+        return _datetime_from_microseconds(1e6 * float(value))
 
 
 def _record_from_json(value, field):

gcloud/bigquery/test_table.py

@@ -824,7 +824,6 @@ def test_fetch_data_w_bound_client(self):
         import datetime
         from gcloud._helpers import UTC
         from gcloud.bigquery.table import SchemaField
-        from gcloud._helpers import _millis_from_datetime
 
         PATH = 'projects/%s/datasets/%s/tables/%s/data' % (
             self.PROJECT, self.DS_NAME, self.TABLE_NAME)
@@ -835,24 +834,29 @@ def test_fetch_data_w_bound_client(self):
         WHEN_2 = WHEN + datetime.timedelta(seconds=2)
         ROWS = 1234
         TOKEN = 'TOKEN'
+
+        def _bigquery_timestamp_float_repr(ts_float):
+            # Preserve microsecond precision for E+09 timestamps
+            return '%0.15E' % (ts_float,)
+
         DATA = {
             'totalRows': ROWS,
             'pageToken': TOKEN,
             'rows': [
                 {'f': [
                     {'v': 'Phred Phlyntstone'},
                     {'v': '32'},
-                    {'v': _millis_from_datetime(WHEN)},
+                    {'v': _bigquery_timestamp_float_repr(WHEN_TS)},
                 ]},
                 {'f': [
                     {'v': 'Bharney Rhubble'},
                     {'v': '33'},
-                    {'v': _millis_from_datetime(WHEN_1)},
+                    {'v': _bigquery_timestamp_float_repr(WHEN_TS + 1)},
                 ]},
                 {'f': [
                     {'v': 'Wylma Phlyntstone'},
                     {'v': '29'},
-                    {'v': _millis_from_datetime(WHEN_2)},
+                    {'v': _bigquery_timestamp_float_repr(WHEN_TS + 2)},
                 ]},
                 {'f': [
                     {'v': 'Bhettye Rhubble'},
@@ -861,6 +865,7 @@ def test_fetch_data_w_bound_client(self):
                 ]},
             ]
         }
+
         conn = _Connection(DATA)
         client = _Client(project=self.PROJECT, connection=conn)
         dataset = _Dataset(client)