release: retry transient upstream errors when fetching release inputs

scripts/release-upstream-input-test.py

@@ -274,6 +274,32 @@ def test_redirect_host_and_response_bounds(self):
                 upstream.response(source)
             build.return_value.open.return_value.close.assert_called_once()
 
+    def test_transient_upstream_errors_are_retried_and_permanent_ones_are_not(self):
+        source = {"url": "https://upstream.example.test/input", "max_bytes": 10}
+        busy = lambda: urllib.error.HTTPError(source["url"], 504, "Gateway Time-out", {}, io.BytesIO())
+        reply = io.BytesIO(b"inert")
+        reply.status = 200
+        reply.headers = {"Content-Length": "5"}
+        with reply, patch.object(upstream.urllib.request, "build_opener") as build, \
+                patch.object(upstream.time, "sleep") as sleep:
+            build.return_value.open.side_effect = [busy(), busy(), reply]
+            self.assertIs(upstream.response(source), reply)
+            self.assertEqual(build.return_value.open.call_count, 3)
+            self.assertEqual([call.args[0] for call in sleep.call_args_list], [2, 4])
+        with patch.object(upstream.urllib.request, "build_opener") as build, \
+                patch.object(upstream.time, "sleep"):
+            build.return_value.open.side_effect = [busy() for _ in range(4)]
+            with self.assertRaises(urllib.error.HTTPError):
+                upstream.response(source)
+            self.assertEqual(build.return_value.open.call_count, 4)
+        with patch.object(upstream.urllib.request, "build_opener") as build, \
+                patch.object(upstream.time, "sleep") as sleep:
+            build.return_value.open.side_effect = urllib.error.HTTPError(source["url"], 404, "Not Found", {}, io.BytesIO())
+            with self.assertRaises(urllib.error.HTTPError):
+                upstream.response(source)
+            self.assertEqual(build.return_value.open.call_count, 1)
+            sleep.assert_not_called()
+
     def test_kubo_recipes_accept_official_asset_redirect_and_refuse_unlisted_host(self):
         inventory = json.loads(Path(__file__).with_name("release-upstream-recipes.json").read_text())
         recipes = [recipe for recipe in inventory["recipes"] if recipe["component"] == "kubo"]

scripts/release-upstream-input.py

@@ -19,6 +19,7 @@
 import sys
 import tarfile
 import tempfile
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -135,6 +136,26 @@ def redirect_request(self, *args, **kwargs):
         return None
 
 
+TRANSIENT_HTTP = frozenset({429, 500, 502, 503, 504})
+
+
+def open_with_retry(opener, request, attempts=4):
+    """A transient server error or timeout gets three more tries (2, 4, 8 s apart):
+    a release build fetches dozens of upstream inputs, and one 504 failed a 60-minute job."""
+    for attempt in range(attempts):
+        try:
+            return opener.open(request, timeout=30)
+        except urllib.error.HTTPError as error:
+            if error.code not in TRANSIENT_HTTP or attempt == attempts - 1:
+                raise
+            error.close()
+        except (urllib.error.URLError, TimeoutError, ConnectionError) as error:
+            if isinstance(error, urllib.error.HTTPError) or attempt == attempts - 1:
+                raise
+        time.sleep(2 ** (attempt + 1))
+    raise AssertionError("unreachable")
+
+
 def response(source):
     """Only HTTPS redirects to recipe-owned hosts; each response has a bound."""
     url = source["url"]
@@ -145,7 +166,7 @@ def response(source):
             raise ValueError("upstream redirect host is outside the recorded allowlist")
         request = urllib.request.Request(url, headers={"User-Agent": "ElastOS-release-input", "Accept-Encoding": "identity"})
         try:
-            reply = opener.open(request, timeout=30)
+            reply = open_with_retry(opener, request)
         except urllib.error.HTTPError as error:
             if error.code not in (301, 302, 303, 307, 308) or hop == 5:
                 error.close()