fix: add per-model timeout (25s) and 429 retry with backoff

Addresses #50 #51 #52 — queryModel() now has a 25s per-model timeout
so a single hanging model can't stall the entire race, and retries
up to 2x on 429 with exponential backoff (respects retry-after header).
Same retry logic added to the standard chat streaming path.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
faber
2026-07-10 15:50:43 -04:00
parent 9420624c03
commit 64170bf9a2
2 changed files with 70 additions and 25 deletions

View File

@@ -442,6 +442,9 @@ const PROVIDER_ENDPOINTS: Record<Provider, string> = {
venice: 'https://api.venice.ai/api/v1/chat/completions',
}
const PER_MODEL_TIMEOUT_MS = 25_000
const MAX_RETRIES_ON_429 = 2
export async function queryModel(
model: string,
messages: Message[],
@@ -483,19 +486,49 @@ export async function queryModel(
headers['X-Title'] = 'GODMOD3.AI-ultraplinian-api'
}
const response = await fetch(PROVIDER_ENDPOINTS[provider], {
method: 'POST',
headers,
body: JSON.stringify(body),
signal,
})
// Per-model timeout: abort if a single model hangs
const timeoutController = new AbortController()
const timeout = setTimeout(() => timeoutController.abort(), PER_MODEL_TIMEOUT_MS)
const combinedSignal = signal
? AbortSignal.any([signal, timeoutController.signal])
: timeoutController.signal
if (!response.ok) {
const errorData = await response.json().catch(() => ({}))
throw new Error(errorData.error?.message || `HTTP ${response.status}`)
let response: Response | undefined
let lastError: string | undefined
for (let attempt = 0; attempt <= MAX_RETRIES_ON_429; attempt++) {
try {
response = await fetch(PROVIDER_ENDPOINTS[provider], {
method: 'POST',
headers,
body: JSON.stringify(body),
signal: combinedSignal,
})
if (response.status === 429 && attempt < MAX_RETRIES_ON_429) {
const retryAfter = response.headers.get('retry-after')
const waitMs = retryAfter ? Math.min(parseInt(retryAfter, 10) * 1000, 5000) : (attempt + 1) * 1500
await new Promise(r => setTimeout(r, waitMs))
continue
}
break
} catch (fetchErr: any) {
if (fetchErr.name === 'AbortError') {
throw new Error('Request timed out')
}
throw fetchErr
}
}
const data = await response.json()
clearTimeout(timeout)
if (!response!.ok) {
const errorData = await response!.json().catch(() => ({}))
throw new Error(errorData.error?.message || `HTTP ${response!.status}`)
}
const data = await response!.json()
const content = data.choices?.[0]?.message?.content || ''
if (!content) throw new Error('Empty response')
@@ -505,7 +538,7 @@ export async function queryModel(
content,
duration_ms: Date.now() - startTime,
success: true,
score: 0, // scored later
score: 0,
}
} catch (err: any) {
return {

View File

@@ -656,20 +656,32 @@ chatRoutes.post('/completions', async (req, res) => {
if (pipeline.finalParams.presence_penalty !== undefined) streamBody.presence_penalty = pipeline.finalParams.presence_penalty
if (pipeline.finalParams.repetition_penalty !== undefined) streamBody.repetition_penalty = pipeline.finalParams.repetition_penalty
const upstreamRes = await fetch('https://openrouter.ai/api/v1/chat/completions', {
method: 'POST',
headers: {
'Authorization': `Bearer ${openrouter_api_key}`,
'Content-Type': 'application/json',
'HTTP-Referer': 'https://godmod3.ai',
'X-Title': 'GODMOD3.AI',
},
body: JSON.stringify(streamBody),
})
// Retry on 429 with exponential backoff (addresses rate-limit failures)
let upstreamRes: Response | undefined
for (let attempt = 0; attempt <= 2; attempt++) {
upstreamRes = await fetch('https://openrouter.ai/api/v1/chat/completions', {
method: 'POST',
headers: {
'Authorization': `Bearer ${openrouter_api_key}`,
'Content-Type': 'application/json',
'HTTP-Referer': 'https://godmod3.ai',
'X-Title': 'GODMOD3.AI',
},
body: JSON.stringify(streamBody),
})
if (!upstreamRes.ok) {
const errData = await upstreamRes.json().catch(() => ({}))
const errMsg = (errData as any).error?.message || `Upstream error: ${upstreamRes.status}`
if (upstreamRes.status === 429 && attempt < 2) {
const retryAfter = upstreamRes.headers.get('retry-after')
const waitMs = retryAfter ? Math.min(parseInt(retryAfter, 10) * 1000, 5000) : (attempt + 1) * 1500
await new Promise(r => setTimeout(r, waitMs))
continue
}
break
}
if (!upstreamRes!.ok) {
const errData = await upstreamRes!.json().catch(() => ({}))
const errMsg = (errData as any).error?.message || `Upstream error: ${upstreamRes!.status}`
const chunk = {
id: completionId,
object: 'chat.completion.chunk',
@@ -688,7 +700,7 @@ chatRoutes.post('/completions', async (req, res) => {
return
}
const reader = upstreamRes.body?.getReader()
const reader = upstreamRes!.body?.getReader()
if (!reader) {
res.write('data: [DONE]\n\n')
res.end()