diff --git a/scripts/release-upstream-input-test.py b/scripts/release-upstream-input-test.py index 39bcf0f29..beffd01a4 100644 --- a/scripts/release-upstream-input-test.py +++ b/scripts/release-upstream-input-test.py @@ -274,6 +274,32 @@ def test_redirect_host_and_response_bounds(self): upstream.response(source) build.return_value.open.return_value.close.assert_called_once() + def test_transient_upstream_errors_are_retried_and_permanent_ones_are_not(self): + source = {"url": "https://upstream.example.test/input", "max_bytes": 10} + busy = lambda: urllib.error.HTTPError(source["url"], 504, "Gateway Time-out", {}, io.BytesIO()) + reply = io.BytesIO(b"inert") + reply.status = 200 + reply.headers = {"Content-Length": "5"} + with reply, patch.object(upstream.urllib.request, "build_opener") as build, \ + patch.object(upstream.time, "sleep") as sleep: + build.return_value.open.side_effect = [busy(), busy(), reply] + self.assertIs(upstream.response(source), reply) + self.assertEqual(build.return_value.open.call_count, 3) + self.assertEqual([call.args[0] for call in sleep.call_args_list], [2, 4]) + with patch.object(upstream.urllib.request, "build_opener") as build, \ + patch.object(upstream.time, "sleep"): + build.return_value.open.side_effect = [busy() for _ in range(4)] + with self.assertRaises(urllib.error.HTTPError): + upstream.response(source) + self.assertEqual(build.return_value.open.call_count, 4) + with patch.object(upstream.urllib.request, "build_opener") as build, \ + patch.object(upstream.time, "sleep") as sleep: + build.return_value.open.side_effect = urllib.error.HTTPError(source["url"], 404, "Not Found", {}, io.BytesIO()) + with self.assertRaises(urllib.error.HTTPError): + upstream.response(source) + self.assertEqual(build.return_value.open.call_count, 1) + sleep.assert_not_called() + def test_kubo_recipes_accept_official_asset_redirect_and_refuse_unlisted_host(self): inventory = json.loads(Path(__file__).with_name("release-upstream-recipes.json").read_text()) recipes = [recipe for recipe in inventory["recipes"] if recipe["component"] == "kubo"] diff --git a/scripts/release-upstream-input.py b/scripts/release-upstream-input.py index 560b437a2..a36ec7b40 100644 --- a/scripts/release-upstream-input.py +++ b/scripts/release-upstream-input.py @@ -19,6 +19,7 @@ import sys import tarfile import tempfile +import time import urllib.error import urllib.parse import urllib.request @@ -135,6 +136,26 @@ def redirect_request(self, *args, **kwargs): return None +TRANSIENT_HTTP = frozenset({429, 500, 502, 503, 504}) + + +def open_with_retry(opener, request, attempts=4): + """A transient server error or timeout gets three more tries (2, 4, 8 s apart): + a release build fetches dozens of upstream inputs, and one 504 failed a 60-minute job.""" + for attempt in range(attempts): + try: + return opener.open(request, timeout=30) + except urllib.error.HTTPError as error: + if error.code not in TRANSIENT_HTTP or attempt == attempts - 1: + raise + error.close() + except (urllib.error.URLError, TimeoutError, ConnectionError) as error: + if isinstance(error, urllib.error.HTTPError) or attempt == attempts - 1: + raise + time.sleep(2 ** (attempt + 1)) + raise AssertionError("unreachable") + + def response(source): """Only HTTPS redirects to recipe-owned hosts; each response has a bound.""" url = source["url"] @@ -145,7 +166,7 @@ def response(source): raise ValueError("upstream redirect host is outside the recorded allowlist") request = urllib.request.Request(url, headers={"User-Agent": "ElastOS-release-input", "Accept-Encoding": "identity"}) try: - reply = opener.open(request, timeout=30) + reply = open_with_retry(opener, request) except urllib.error.HTTPError as error: if error.code not in (301, 302, 303, 307, 308) or hop == 5: error.close()