Spaces:
Running
Running
File size: 2,440 Bytes
d135f12 e16bbaf d135f12 e16bbaf d135f12 3a30924 d135f12 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | 'use strict';
/**
* Robust fetch helper with retries, exponential backoff, and timeout.
*/
async function fetchRobust(url, options = {}) {
const { retries = 5, backoff = 1000, timeout = 120000, ...fetchOptions } = options;
let lastError;
for (let i = 0; i < retries; i++) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeout);
try {
const res = await fetch(url, {
...fetchOptions,
signal: controller.signal,
headers: {
'User-Agent': 'providers-benchmark-fetcher',
...fetchOptions.headers,
},
});
clearTimeout(timer);
if (res.ok) return res;
// Retry on transient status codes: 429 (Rate Limit), 408 (Timeout), and 5xx (Server Errors)
if (res.status === 429 || res.status === 408 || (res.status >= 500 && res.status < 600)) {
lastError = new Error(`HTTP ${res.status} from ${url}`);
if (i < retries - 1) {
const delay = backoff * Math.pow(2, i) + Math.random() * 1000;
process.stdout.write(`\n ⚠ ${lastError.message}. Retrying in ${Math.round(delay)}ms... (${i + 1}/${retries})\n`);
await new Promise((r) => setTimeout(r, delay));
continue;
}
} else {
// Don't retry on other 4xx errors (e.g. 404, 401, 403)
throw new Error(`HTTP ${res.status} from ${url}`);
}
} catch (err) {
clearTimeout(timer);
lastError = err;
const isTimeout = err.name === 'AbortError';
const isAuthOrNotFound = err.message.includes('401') || err.message.includes('404');
if (i < retries - 1 && !isAuthOrNotFound) {
const delay = backoff * Math.pow(2, i) + Math.random() * 1000;
const msg = isTimeout ? `Timeout after ${timeout}ms` : err.message;
process.stdout.write(`\n ⚠ Fetch error from ${url}: ${msg}. Retrying in ${Math.round(delay)}ms... (${i + 1}/${retries})\n`);
await new Promise((r) => setTimeout(r, delay));
continue;
}
}
}
throw lastError;
}
async function getJson(url, options = {}) {
const res = await fetchRobust(url, { ...options, headers: { Accept: 'application/json', ...options.headers } });
return res.json();
}
async function getText(url, options = {}) {
const res = await fetchRobust(url, options);
return res.text();
}
module.exports = {
fetchRobust,
getJson,
getText,
};
|