From 64170bf9a2a616ecec3d2a547789fc8a724533d0 Mon Sep 17 00:00:00 2001 From: faber Date: Fri, 10 Jul 2026 15:50:43 -0400 Subject: [PATCH] fix: add per-model timeout (25s) and 429 retry with backoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses #50 #51 #52 — queryModel() now has a 25s per-model timeout so a single hanging model can't stall the entire race, and retries up to 2x on 429 with exponential backoff (respects retry-after header). Same retry logic added to the standard chat streaming path. Co-Authored-By: Claude Opus 4.6 --- api/lib/ultraplinian.ts | 55 ++++++++++++++++++++++++++++++++--------- api/routes/chat.ts | 40 +++++++++++++++++++----------- 2 files changed, 70 insertions(+), 25 deletions(-) diff --git a/api/lib/ultraplinian.ts b/api/lib/ultraplinian.ts index e0ccebf..628defa 100644 --- a/api/lib/ultraplinian.ts +++ b/api/lib/ultraplinian.ts @@ -442,6 +442,9 @@ const PROVIDER_ENDPOINTS: Record = { venice: 'https://api.venice.ai/api/v1/chat/completions', } +const PER_MODEL_TIMEOUT_MS = 25_000 +const MAX_RETRIES_ON_429 = 2 + export async function queryModel( model: string, messages: Message[], @@ -483,19 +486,49 @@ export async function queryModel( headers['X-Title'] = 'GODMOD3.AI-ultraplinian-api' } - const response = await fetch(PROVIDER_ENDPOINTS[provider], { - method: 'POST', - headers, - body: JSON.stringify(body), - signal, - }) + // Per-model timeout: abort if a single model hangs + const timeoutController = new AbortController() + const timeout = setTimeout(() => timeoutController.abort(), PER_MODEL_TIMEOUT_MS) + const combinedSignal = signal + ? AbortSignal.any([signal, timeoutController.signal]) + : timeoutController.signal - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - throw new Error(errorData.error?.message || `HTTP ${response.status}`) + let response: Response | undefined + let lastError: string | undefined + + for (let attempt = 0; attempt <= MAX_RETRIES_ON_429; attempt++) { + try { + response = await fetch(PROVIDER_ENDPOINTS[provider], { + method: 'POST', + headers, + body: JSON.stringify(body), + signal: combinedSignal, + }) + + if (response.status === 429 && attempt < MAX_RETRIES_ON_429) { + const retryAfter = response.headers.get('retry-after') + const waitMs = retryAfter ? Math.min(parseInt(retryAfter, 10) * 1000, 5000) : (attempt + 1) * 1500 + await new Promise(r => setTimeout(r, waitMs)) + continue + } + + break + } catch (fetchErr: any) { + if (fetchErr.name === 'AbortError') { + throw new Error('Request timed out') + } + throw fetchErr + } } - const data = await response.json() + clearTimeout(timeout) + + if (!response!.ok) { + const errorData = await response!.json().catch(() => ({})) + throw new Error(errorData.error?.message || `HTTP ${response!.status}`) + } + + const data = await response!.json() const content = data.choices?.[0]?.message?.content || '' if (!content) throw new Error('Empty response') @@ -505,7 +538,7 @@ export async function queryModel( content, duration_ms: Date.now() - startTime, success: true, - score: 0, // scored later + score: 0, } } catch (err: any) { return { diff --git a/api/routes/chat.ts b/api/routes/chat.ts index 8395bc1..ca4d037 100644 --- a/api/routes/chat.ts +++ b/api/routes/chat.ts @@ -656,20 +656,32 @@ chatRoutes.post('/completions', async (req, res) => { if (pipeline.finalParams.presence_penalty !== undefined) streamBody.presence_penalty = pipeline.finalParams.presence_penalty if (pipeline.finalParams.repetition_penalty !== undefined) streamBody.repetition_penalty = pipeline.finalParams.repetition_penalty - const upstreamRes = await fetch('https://openrouter.ai/api/v1/chat/completions', { - method: 'POST', - headers: { - 'Authorization': `Bearer ${openrouter_api_key}`, - 'Content-Type': 'application/json', - 'HTTP-Referer': 'https://godmod3.ai', - 'X-Title': 'GODMOD3.AI', - }, - body: JSON.stringify(streamBody), - }) + // Retry on 429 with exponential backoff (addresses rate-limit failures) + let upstreamRes: Response | undefined + for (let attempt = 0; attempt <= 2; attempt++) { + upstreamRes = await fetch('https://openrouter.ai/api/v1/chat/completions', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${openrouter_api_key}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': 'https://godmod3.ai', + 'X-Title': 'GODMOD3.AI', + }, + body: JSON.stringify(streamBody), + }) - if (!upstreamRes.ok) { - const errData = await upstreamRes.json().catch(() => ({})) - const errMsg = (errData as any).error?.message || `Upstream error: ${upstreamRes.status}` + if (upstreamRes.status === 429 && attempt < 2) { + const retryAfter = upstreamRes.headers.get('retry-after') + const waitMs = retryAfter ? Math.min(parseInt(retryAfter, 10) * 1000, 5000) : (attempt + 1) * 1500 + await new Promise(r => setTimeout(r, waitMs)) + continue + } + break + } + + if (!upstreamRes!.ok) { + const errData = await upstreamRes!.json().catch(() => ({})) + const errMsg = (errData as any).error?.message || `Upstream error: ${upstreamRes!.status}` const chunk = { id: completionId, object: 'chat.completion.chunk', @@ -688,7 +700,7 @@ chatRoutes.post('/completions', async (req, res) => { return } - const reader = upstreamRes.body?.getReader() + const reader = upstreamRes!.body?.getReader() if (!reader) { res.write('data: [DONE]\n\n') res.end()