Playing with retries.

system_tests/bigquery.py

@@ -20,7 +20,9 @@
 from gcloud import _helpers
 from gcloud.environment_vars import TESTS_PROJECT
 from gcloud import bigquery
+from gcloud.exceptions import Forbidden
 
+from retry import retry
 from system_test_utils import unique_resource_id
 
 
@@ -90,7 +92,12 @@ def test_update_dataset(self):
         after = [grant for grant in dataset.access_grants
                  if grant.entity_id != 'projectWriters']
         dataset.access_grants = after
-        dataset.update()
+
+        @retry(Forbidden, tries=3, delay=30)
+        def update_dataset():
+            dataset.update()
+
+        update_dataset()
         self.assertEqual(len(dataset.access_grants), len(after))
         for found, expected in zip(dataset.access_grants, after):
             self.assertEqual(found.role, expected.role)
@@ -188,7 +195,11 @@ def test_patch_table(self):
     def test_update_table(self):
         dataset = Config.CLIENT.dataset(DATASET_NAME)
         self.assertFalse(dataset.exists())
-        dataset.create()
+
+        @retry(Forbidden, tries=3, delay=20)
+        def create_dataset():
+            dataset.create()
+        create_dataset()
         self.to_delete.append(dataset)
         TABLE_NAME = 'test_table'
         full_name = bigquery.SchemaField('full_name', 'STRING',

system_tests/retry.py

@@ -0,0 +1,49 @@
+import time
+from functools import wraps
+
+
+def retry(exception, tries=4, delay=3, backoff=2, logger=None):
+    """Retry calling the decorated function using an exponential backoff.
+
+    :type exception: Exception or tuple
+    :param exception: the exception to check. may be a tuple of
+        exceptions to check
+
+    :type tries: int
+    :param tries: number of times to try (not retry) before giving up
+
+    :type delay: int
+    :param delay: initial delay between retries in seconds
+
+    :type backoff: int
+    :param backoff: backoff multiplier e.g. value of 2 will double the delay
+        each retry
+
+    :type logger: logging.Logger instance
+    :param logger: logger to use. If None, print
+
+    :rtype: func
+    :returns: Retry wrapper function.
+    """
+    def retry_decorator(f):
+
+        @wraps(f)
+        def f_retry(*args, **kwargs):
+            mtries, mdelay = tries, delay
+            while mtries > 1:
+                try:
+                    return f(*args, **kwargs)
+                except exception as e:
+                    msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
+                    if logger:
+                        logger.warning(msg)
+                    else:
+                        print(msg)
+                    time.sleep(mdelay)
+                    mtries -= 1
+                    mdelay *= backoff
+            return f(*args, **kwargs)
+
+        return f_retry
+
+    return retry_decorator