Move google.cloud.iterator to google.api.core.page_iterator (googleapis#3770)

* Move google.cloud.iterator to google.api.core.page_iterator
* Re-write tests to pytest style.
* Make GAXIterator private- it will soon be removed.
* Pass api_request into HTTPIterator to avoid accessing private members
* BigQuery: use google.api.core.page_iterator
* DNS: use google.api.core.page_iterator
* Logging: use google.api.core.page_iterator
* PubSub: use google.api.core.page_iterator
* Resource manager: use google.api.core.page_iterator
* Runtimeconfig: use google.api.core.page_iterator
* logging: use google.api.core._GAXIterator
* Storage: use google.api.core.page_iterator
* Pubsub: use google.api.core._GAXIterator
* Trace: use google.api.core._GAXIterator
* Spanner: use google.api.core._GAXIterator

Author: Jon Wayne Parrott

bigquery/google/cloud/bigquery/_helpers.py

@@ -684,7 +684,7 @@ def _item_to_row(iterator, resource):
         added to the iterator after being created, which
         should be done by the caller.
 
-    :type iterator: :class:`~google.cloud.iterator.Iterator`
+    :type iterator: :class:`~google.api.core.page_iterator.Iterator`
     :param iterator: The iterator that is currently in use.
 
     :type resource: dict
@@ -700,7 +700,7 @@ def _item_to_row(iterator, resource):
 def _rows_page_start(iterator, page, response):
     """Grab total rows when :class:`~google.cloud.iterator.Page` starts.
 
-    :type iterator: :class:`~google.cloud.iterator.Iterator`
+    :type iterator: :class:`~google.api.core.page_iterator.Iterator`
     :param iterator: The iterator that is currently in use.
 
     :type page: :class:`~google.cloud.iterator.Page`

bigquery/google/cloud/bigquery/client.py

@@ -14,7 +14,7 @@
 
 """Client for interacting with the Google BigQuery API."""
 
-
+from google.api.core import page_iterator
 from google.cloud.client import ClientWithProject
 from google.cloud.bigquery._http import Connection
 from google.cloud.bigquery.dataset import Dataset
@@ -23,7 +23,6 @@
 from google.cloud.bigquery.job import LoadTableFromStorageJob
 from google.cloud.bigquery.job import QueryJob
 from google.cloud.bigquery.query import QueryResults
-from google.cloud.iterator import HTTPIterator
 
 
 class Project(object):
@@ -98,13 +97,17 @@ def list_projects(self, max_results=None, page_token=None):
                            not passed, the API will return the first page of
                            projects.
 
-        :rtype: :class:`~google.cloud.iterator.Iterator`
+        :rtype: :class:`~google.api.core.page_iterator.Iterator`
         :returns: Iterator of :class:`~google.cloud.bigquery.client.Project`
                   accessible to the current client.
         """
-        return HTTPIterator(
-            client=self, path='/projects', item_to_value=_item_to_project,
-            items_key='projects', page_token=page_token,
+        return page_iterator.HTTPIterator(
+            client=self,
+            api_request=self._connection.api_request,
+            path='/projects',
+            item_to_value=_item_to_project,
+            items_key='projects',
+            page_token=page_token,
             max_results=max_results)
 
     def list_datasets(self, include_all=False, max_results=None,
@@ -126,18 +129,23 @@ def list_datasets(self, include_all=False, max_results=None,
                            not passed, the API will return the first page of
                            datasets.
 
-        :rtype: :class:`~google.cloud.iterator.Iterator`
+        :rtype: :class:`~google.api.core.page_iterator.Iterator`
         :returns: Iterator of :class:`~google.cloud.bigquery.dataset.Dataset`.
                   accessible to the current client.
         """
         extra_params = {}
         if include_all:
             extra_params['all'] = True
         path = '/projects/%s/datasets' % (self.project,)
-        return HTTPIterator(
-            client=self, path=path, item_to_value=_item_to_dataset,
-            items_key='datasets', page_token=page_token,
-            max_results=max_results, extra_params=extra_params)
+        return page_iterator.HTTPIterator(
+            client=self,
+            api_request=self._connection.api_request,
+            path=path,
+            item_to_value=_item_to_dataset,
+            items_key='datasets',
+            page_token=page_token,
+            max_results=max_results,
+            extra_params=extra_params)
 
     def dataset(self, dataset_name, project=None):
         """Construct a dataset bound to this client.
@@ -207,7 +215,7 @@ def list_jobs(self, max_results=None, page_token=None, all_users=None,
                              * ``"pending"``
                              * ``"running"``
 
-        :rtype: :class:`~google.cloud.iterator.Iterator`
+        :rtype: :class:`~google.api.core.page_iterator.Iterator`
         :returns: Iterable of job instances.
         """
         extra_params = {'projection': 'full'}
@@ -219,10 +227,15 @@ def list_jobs(self, max_results=None, page_token=None, all_users=None,
             extra_params['stateFilter'] = state_filter
 
         path = '/projects/%s/jobs' % (self.project,)
-        return HTTPIterator(
-            client=self, path=path, item_to_value=_item_to_job,
-            items_key='jobs', page_token=page_token,
-            max_results=max_results, extra_params=extra_params)
+        return page_iterator.HTTPIterator(
+            client=self,
+            api_request=self._connection.api_request,
+            path=path,
+            item_to_value=_item_to_job,
+            items_key='jobs',
+            page_token=page_token,
+            max_results=max_results,
+            extra_params=extra_params)
 
     def load_table_from_storage(self, job_name, destination, *source_uris):
         """Construct a job for loading data into a table from CloudStorage.
@@ -349,7 +362,7 @@ def run_sync_query(self, query, udf_resources=(), query_parameters=()):
 def _item_to_project(iterator, resource):
     """Convert a JSON project to the native object.
 
-    :type iterator: :class:`~google.cloud.iterator.Iterator`
+    :type iterator: :class:`~google.api.core.page_iterator.Iterator`
     :param iterator: The iterator that is currently in use.
 
     :type resource: dict
@@ -365,7 +378,7 @@ def _item_to_project(iterator, resource):
 def _item_to_dataset(iterator, resource):
     """Convert a JSON dataset to the native object.
 
-    :type iterator: :class:`~google.cloud.iterator.Iterator`
+    :type iterator: :class:`~google.api.core.page_iterator.Iterator`
     :param iterator: The iterator that is currently in use.
 
     :type resource: dict
@@ -380,7 +393,7 @@ def _item_to_dataset(iterator, resource):
 def _item_to_job(iterator, resource):
     """Convert a JSON job to the native object.
 
-    :type iterator: :class:`~google.cloud.iterator.Iterator`
+    :type iterator: :class:`~google.api.core.page_iterator.Iterator`
     :param iterator: The iterator that is currently in use.
 
     :type resource: dict

bigquery/google/cloud/bigquery/dataset.py

@@ -15,10 +15,10 @@
 """Define API Datasets."""
 import six
 
+from google.api.core import page_iterator
 from google.cloud._helpers import _datetime_from_microseconds
 from google.cloud.exceptions import NotFound
 from google.cloud.bigquery.table import Table
-from google.cloud.iterator import HTTPIterator
 
 
 class AccessGrant(object):
@@ -561,14 +561,19 @@ def list_tables(self, max_results=None, page_token=None):
                            datasets. If not passed, the API will return the
                            first page of datasets.
 
-        :rtype: :class:`~google.cloud.iterator.Iterator`
+        :rtype: :class:`~google.api.core.page_iterator.Iterator`
         :returns: Iterator of :class:`~google.cloud.bigquery.table.Table`
                   contained within the current dataset.
         """
         path = '/projects/%s/datasets/%s/tables' % (self.project, self.name)
-        result = HTTPIterator(client=self._client, path=path,
-                              item_to_value=_item_to_table, items_key='tables',
-                              page_token=page_token, max_results=max_results)
+        result = page_iterator.HTTPIterator(
+            client=self._client,
+            api_request=self._client._connection.api_request,
+            path=path,
+            item_to_value=_item_to_table,
+            items_key='tables',
+            page_token=page_token,
+            max_results=max_results)
         result.dataset = self
         return result
 
@@ -590,7 +595,7 @@ def table(self, name, schema=()):
 def _item_to_table(iterator, resource):
     """Convert a JSON table to the native object.
 
-    :type iterator: :class:`~google.cloud.iterator.Iterator`
+    :type iterator: :class:`~google.api.core.page_iterator.Iterator`
     :param iterator: The iterator that is currently in use.
 
     :type resource: dict

bigquery/google/cloud/bigquery/query.py

@@ -16,7 +16,7 @@
 
 import six
 
-from google.cloud.iterator import HTTPIterator
+from google.api.core import page_iterator
 from google.cloud.bigquery._helpers import _TypedProperty
 from google.cloud.bigquery._helpers import _rows_from_json
 from google.cloud.bigquery.dataset import Dataset
@@ -414,7 +414,7 @@ def fetch_data(self, max_results=None, page_token=None, start_index=None,
         :param client: the client to use.  If not passed, falls back to the
                        ``client`` stored on the current dataset.
 
-        :rtype: :class:`~google.cloud.iterator.Iterator`
+        :rtype: :class:`~google.api.core.page_iterator.Iterator`
         :returns: Iterator of row data :class:`tuple`s. During each page, the
                   iterator will have the ``total_rows`` attribute set,
                   which counts the total number of rows **in the result
@@ -435,13 +435,16 @@ def fetch_data(self, max_results=None, page_token=None, start_index=None,
             params['timeoutMs'] = timeout_ms
 
         path = '/projects/%s/queries/%s' % (self.project, self.name)
-        iterator = HTTPIterator(client=client, path=path,
-                                item_to_value=_item_to_row,
-                                items_key='rows',
-                                page_token=page_token,
-                                max_results=max_results,
-                                page_start=_rows_page_start_query,
-                                extra_params=params)
+        iterator = page_iterator.HTTPIterator(
+            client=client,
+            api_request=client._connection.api_request,
+            path=path,
+            item_to_value=_item_to_row,
+            items_key='rows',
+            page_token=page_token,
+            max_results=max_results,
+            page_start=_rows_page_start_query,
+            extra_params=params)
         iterator.query_result = self
         # Over-ride the key used to retrieve the next page token.
         iterator._NEXT_TOKEN = 'pageToken'
@@ -457,7 +460,7 @@ def _rows_page_start_query(iterator, page, response):
         added to the iterator after being created, which
         should be done by the caller.
 
-    :type iterator: :class:`~google.cloud.iterator.Iterator`
+    :type iterator: :class:`~google.api.core.page_iterator.Iterator`
     :param iterator: The iterator that is currently in use.
 
     :type page: :class:`~google.cloud.iterator.Page`

bigquery/google/cloud/bigquery/table.py

@@ -23,10 +23,10 @@
 from google.resumable_media.requests import MultipartUpload
 from google.resumable_media.requests import ResumableUpload
 
+from google.api.core import page_iterator
 from google.cloud import exceptions
 from google.cloud._helpers import _datetime_from_microseconds
 from google.cloud._helpers import _millis_from_datetime
-from google.cloud.iterator import HTTPIterator
 from google.cloud.bigquery.schema import SchemaField
 from google.cloud.bigquery._helpers import _item_to_row
 from google.cloud.bigquery._helpers import _rows_page_start
@@ -712,7 +712,7 @@ def fetch_data(self, max_results=None, page_token=None, client=None):
         :param client: (Optional) The client to use.  If not passed, falls
                        back to the ``client`` stored on the current dataset.
 
-        :rtype: :class:`~google.cloud.iterator.Iterator`
+        :rtype: :class:`~google.api.core.page_iterator.Iterator`
         :returns: Iterator of row data :class:`tuple`s. During each page, the
                   iterator will have the ``total_rows`` attribute set,
                   which counts the total number of rows **in the table**
@@ -724,10 +724,15 @@ def fetch_data(self, max_results=None, page_token=None, client=None):
 
         client = self._require_client(client)
         path = '%s/data' % (self.path,)
-        iterator = HTTPIterator(client=client, path=path,
-                                item_to_value=_item_to_row, items_key='rows',
-                                page_token=page_token, max_results=max_results,
-                                page_start=_rows_page_start)
+        iterator = page_iterator.HTTPIterator(
+            client=client,
+            api_request=client._connection.api_request,
+            path=path,
+            item_to_value=_item_to_row,
+            items_key='rows',
+            page_token=page_token,
+            max_results=max_results,
+            page_start=_rows_page_start)
         iterator.schema = self._schema
         # Over-ride the key used to retrieve the next page token.
         iterator._NEXT_TOKEN = 'pageToken'