refactor(core): consolidate fetch retry helper (#16155)

Co-authored-by: mbenhamd <16720467+mbenhamd@users.noreply.github.com>

Author: mbenhamd

.changeset/four-foxes-act.md

@@ -0,0 +1,13 @@
+---
+'@mastra/core': patch
+---
+
+Added configurable response-based retry handling to `fetchWithRetry`.
+
+Developers can now pass `shouldRetryResponse` to control which non-OK HTTP responses should be retried while network failures continue to retry automatically.
+
+```ts
+await fetchWithRetry(url, requestOptions, 3, {
+  shouldRetryResponse: response => response.status === 429 || response.status >= 500,
+});
+```

packages/core/src/agent/message-list/prompt/download-assets.test.ts

@@ -0,0 +1,60 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+import { downloadFromUrl } from './download-assets';
+
+describe('downloadFromUrl', () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.unstubAllGlobals();
+  });
+
+  function mockRetryDelays() {
+    const delays: number[] = [];
+
+    vi.spyOn(globalThis, 'setTimeout').mockImplementation(((fn: () => void, delay?: number) => {
+      if (delay && delay > 100) {
+        delays.push(delay);
+      }
+      if (typeof fn === 'function') fn();
+      return 0 as unknown as ReturnType<typeof setTimeout>;
+    }) as typeof setTimeout);
+
+    return delays;
+  }
+
+  it('should not retry client error responses', async () => {
+    const delays = mockRetryDelays();
+    const mockFetch = vi.fn().mockResolvedValue(new Response('not found', { status: 404, statusText: 'Not Found' }));
+    vi.stubGlobal('fetch', mockFetch);
+
+    await expect(
+      downloadFromUrl({ url: new URL('https://example.com/missing.png'), downloadRetries: 3 }),
+    ).rejects.toThrow('Failed to download asset');
+
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    expect(delays).toEqual([]);
+  });
+
+  it('should retry server error responses', async () => {
+    const delays = mockRetryDelays();
+    const response = new Response('image-data', {
+      status: 200,
+      headers: { 'content-type': 'image/png' },
+    });
+    const mockFetch = vi
+      .fn()
+      .mockResolvedValueOnce(new Response('server error', { status: 500, statusText: 'Server Error' }))
+      .mockResolvedValueOnce(response);
+    vi.stubGlobal('fetch', mockFetch);
+
+    await expect(
+      downloadFromUrl({ url: new URL('https://example.com/image.png'), downloadRetries: 3 }),
+    ).resolves.toEqual({
+      data: new Uint8Array(await new Response('image-data').arrayBuffer()),
+      mediaType: 'image/png',
+    });
+
+    expect(mockFetch).toHaveBeenCalledTimes(2);
+    expect(delays).toEqual([2000]);
+  });
+});

packages/core/src/agent/message-list/prompt/download-assets.ts

@@ -13,6 +13,9 @@ export const downloadFromUrl = async ({ url, downloadRetries }: { url: URL; down
         method: 'GET',
       },
       downloadRetries,
+      {
+        shouldRetryResponse: response => response.status >= 500,
+      },
     );
 
     if (!response.ok) {

packages/core/src/utils.test.ts

@@ -383,7 +383,7 @@ describe('fetchWithRetry', () => {
     vi.unstubAllGlobals();
   });
 
-  it('should use exponential backoff delays capped at 10 seconds', async () => {
+  function mockRetryDelays() {
     const delays: number[] = [];
 
     vi.spyOn(globalThis, 'setTimeout').mockImplementation(((fn: () => void, delay?: number) => {
@@ -395,6 +395,104 @@ describe('fetchWithRetry', () => {
       return 0 as unknown as ReturnType<typeof setTimeout>;
     }) as typeof setTimeout);
 
+    return delays;
+  }
+
+  it('should return a successful response without retrying', async () => {
+    const response = new Response('ok', { status: 200 });
+    const mockFetch = vi.fn().mockResolvedValue(response);
+    vi.stubGlobal('fetch', mockFetch);
+
+    await expect(fetchWithRetry('https://example.com', { method: 'POST' }, 3)).resolves.toBe(response);
+
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    expect(mockFetch).toHaveBeenCalledWith('https://example.com', { method: 'POST' });
+  });
+
+  it('should retry a failed response and return a later success', async () => {
+    const delays = mockRetryDelays();
+    const response = new Response('ok', { status: 200 });
+    const mockFetch = vi
+      .fn()
+      .mockResolvedValueOnce(new Response('error', { status: 500, statusText: 'Server Error' }))
+      .mockResolvedValueOnce(response);
+    vi.stubGlobal('fetch', mockFetch);
+
+    await expect(fetchWithRetry('https://example.com', {}, 3)).resolves.toBe(response);
+
+    expect(mockFetch).toHaveBeenCalledTimes(2);
+    expect(delays).toEqual([2000]);
+  });
+
+  it('should retry network failures until retries are exhausted', async () => {
+    const delays = mockRetryDelays();
+    const mockFetch = vi.fn().mockRejectedValue(new Error('Network error'));
+    vi.stubGlobal('fetch', mockFetch);
+
+    await expect(fetchWithRetry('https://example.com', {}, 3)).rejects.toThrow('Network error');
+
+    expect(mockFetch).toHaveBeenCalledTimes(3);
+    expect(delays).toEqual([2000, 4000]);
+  });
+
+  it.each([404, 408, 429])('should preserve public retry behavior for %s responses by default', async status => {
+    const delays = mockRetryDelays();
+    const mockFetch = vi.fn().mockResolvedValue(new Response('error', { status }));
+    vi.stubGlobal('fetch', mockFetch);
+
+    await expect(fetchWithRetry('https://example.com/missing', {}, 2)).rejects.toThrow(`status: ${status}`);
+
+    expect(mockFetch).toHaveBeenCalledTimes(2);
+    expect(delays).toEqual([2000]);
+  });
+
+  it('should not retry a response when shouldRetryResponse returns false', async () => {
+    const delays = mockRetryDelays();
+    const mockFetch = vi.fn().mockResolvedValue(new Response('not found', { status: 404, statusText: 'Not Found' }));
+    vi.stubGlobal('fetch', mockFetch);
+
+    await expect(
+      fetchWithRetry('https://example.com/missing', {}, 3, {
+        shouldRetryResponse: response => response.status >= 500,
+      }),
+    ).rejects.toThrow('status: 404 Not Found');
+
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    expect(delays).toEqual([]);
+  });
+
+  it('should retry network errors even when the error message contains a 4xx status', async () => {
+    const delays = mockRetryDelays();
+    const response = new Response('ok', { status: 200 });
+    const mockFetch = vi.fn().mockRejectedValueOnce(new Error('upstream status: 404')).mockResolvedValueOnce(response);
+    vi.stubGlobal('fetch', mockFetch);
+
+    await expect(
+      fetchWithRetry('https://example.com/transient', {}, 3, {
+        shouldRetryResponse: response => response.status >= 500,
+      }),
+    ).resolves.toBe(response);
+
+    expect(mockFetch).toHaveBeenCalledTimes(2);
+    expect(delays).toEqual([2000]);
+  });
+
+  it('should throw the last response error after exhausting retries', async () => {
+    const delays = mockRetryDelays();
+    const mockFetch = vi
+      .fn()
+      .mockResolvedValueOnce(new Response('first', { status: 500, statusText: 'First Error' }))
+      .mockResolvedValueOnce(new Response('second', { status: 503, statusText: 'Second Error' }));
+    vi.stubGlobal('fetch', mockFetch);
+
+    await expect(fetchWithRetry('https://example.com/flaky', {}, 2)).rejects.toThrow('status: 503 Second Error');
+
+    expect(mockFetch).toHaveBeenCalledTimes(2);
+    expect(delays).toEqual([2000]);
+  });
+
+  it('should use exponential backoff delays capped at 10 seconds', async () => {
+    const delays = mockRetryDelays();
     const mockFetch = vi.fn().mockRejectedValue(new Error('Network error'));
     vi.stubGlobal('fetch', mockFetch);
 

packages/core/src/utils.ts

@@ -23,6 +23,8 @@ import type { Workspace } from './workspace/workspace';
 
 // Re-export Zod utilities for external use (isZodType is defined locally below)
 export { getZodTypeName, getZodDef, isZodArray, isZodObject } from './utils/zod-utils';
+export { fetchWithRetry } from './utils/fetchWithRetry';
+export type { FetchWithRetryOptions } from './utils/fetchWithRetry';
 
 export const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
 
@@ -691,47 +693,6 @@ export function parseFieldKey(key: string): FieldKey {
   return key as FieldKey;
 }
 
-/**
- * Performs a fetch request with automatic retries using exponential backoff
- * @param url The URL to fetch from
- * @param options Standard fetch options
- * @param maxRetries Maximum number of retry attempts
- * @param validateResponse Optional function to validate the response beyond HTTP status
- * @returns The fetch Response if successful
- */
-export async function fetchWithRetry(
-  url: string,
-  options: RequestInit = {},
-  maxRetries: number = 3,
-): Promise<Response> {
-  let retryCount = 0;
-  let lastError: Error | null = null;
-
-  while (retryCount < maxRetries) {
-    try {
-      const response = await fetch(url, options);
-
-      if (!response.ok) {
-        throw new Error(`Request failed with status: ${response.status} ${response.statusText}`);
-      }
-
-      return response;
-    } catch (error) {
-      lastError = error instanceof Error ? error : new Error(String(error));
-      retryCount++;
-
-      if (retryCount >= maxRetries) {
-        break;
-      }
-
-      const delay = Math.min(1000 * Math.pow(2, retryCount), 10000);
-      await new Promise(resolve => setTimeout(resolve, delay));
-    }
-  }
-
-  throw lastError || new Error('Request failed after multiple retry attempts');
-}
-
 /**
  * Removes specific keys from an object.
  * @param obj - The original object

packages/core/src/utils/fetchWithRetry.ts

@@ -1,59 +1,53 @@
+export interface FetchWithRetryOptions {
+  shouldRetryResponse?: (response: Response) => boolean;
+}
+
+const defaultShouldRetryResponse = () => true;
+
 /**
- * Fetches a URL with retry logic
- * @param url The URL to fetch
- * @param options Fetch options
- * @param maxRetries Maximum number of retry attempts
- * @returns The fetch Response if successful
+ * Performs a fetch request with automatic retries using exponential backoff.
+ * Network failures are always retried. Non-OK responses are retried unless
+ * `shouldRetryResponse` returns false.
  */
 export async function fetchWithRetry(
   url: string,
   options: RequestInit = {},
   maxRetries: number = 3,
+  retryOptions: FetchWithRetryOptions = {},
 ): Promise<Response> {
   let retryCount = 0;
   let lastError: Error | null = null;
+  const shouldRetryResponse = retryOptions.shouldRetryResponse ?? defaultShouldRetryResponse;
 
   while (retryCount < maxRetries) {
+    let response: Response | undefined;
+
     try {
-      const response = await fetch(url, options);
+      response = await fetch(url, options);
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+    }
 
+    if (response) {
       if (!response.ok) {
-        // Only retry on server errors (5xx) or network failures
-        // Don't retry on client errors (4xx)
-        if (response.status >= 400 && response.status < 500) {
-          throw new Error(`Request failed with status: ${response.status} ${response.statusText}`);
-        }
-
         lastError = new Error(`Request failed with status: ${response.status} ${response.statusText}`);
-        retryCount++;
 
-        if (retryCount >= maxRetries) {
+        if (!shouldRetryResponse(response)) {
           throw lastError;
         }
-
-        const delay = Math.min(1000 * Math.pow(2, retryCount), 10000);
-        await new Promise(resolve => setTimeout(resolve, delay));
-        continue;
+      } else {
+        return response;
       }
+    }
 
-      return response;
-    } catch (error) {
-      lastError = error instanceof Error ? error : new Error(String(error));
-
-      // If it's a client error (4xx), don't retry
-      if (lastError.message.includes('status: 4')) {
-        throw lastError;
-      }
-
-      retryCount++;
-
-      if (retryCount >= maxRetries) {
-        break;
-      }
+    retryCount++;
 
-      const delay = Math.min(1000 * Math.pow(2, retryCount), 10000);
-      await new Promise(resolve => setTimeout(resolve, delay));
+    if (retryCount >= maxRetries) {
+      break;
     }
+
+    const delay = Math.min(1000 * Math.pow(2, retryCount), 10000);
+    await new Promise(resolve => setTimeout(resolve, delay));
   }
 
   throw lastError || new Error('Request failed after multiple retry attempts');