mirror of
https://github.com/elder-plinius/G0DM0D3.git
synced 2026-11-16 07:02:25 +00:00
fix: add per-model timeout (25s) and 429 retry with backoff
Addresses #50 #51 #52 — queryModel() now has a 25s per-model timeout so a single hanging model can't stall the entire race, and retries up to 2x on 429 with exponential backoff (respects retry-after header). Same retry logic added to the standard chat streaming path. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -442,6 +442,9 @@ const PROVIDER_ENDPOINTS: Record<Provider, string> = {
|
||||
venice: 'https://api.venice.ai/api/v1/chat/completions',
|
||||
}
|
||||
|
||||
const PER_MODEL_TIMEOUT_MS = 25_000
|
||||
const MAX_RETRIES_ON_429 = 2
|
||||
|
||||
export async function queryModel(
|
||||
model: string,
|
||||
messages: Message[],
|
||||
@@ -483,19 +486,49 @@ export async function queryModel(
|
||||
headers['X-Title'] = 'GODMOD3.AI-ultraplinian-api'
|
||||
}
|
||||
|
||||
const response = await fetch(PROVIDER_ENDPOINTS[provider], {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
signal,
|
||||
})
|
||||
// Per-model timeout: abort if a single model hangs
|
||||
const timeoutController = new AbortController()
|
||||
const timeout = setTimeout(() => timeoutController.abort(), PER_MODEL_TIMEOUT_MS)
|
||||
const combinedSignal = signal
|
||||
? AbortSignal.any([signal, timeoutController.signal])
|
||||
: timeoutController.signal
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
throw new Error(errorData.error?.message || `HTTP ${response.status}`)
|
||||
let response: Response | undefined
|
||||
let lastError: string | undefined
|
||||
|
||||
for (let attempt = 0; attempt <= MAX_RETRIES_ON_429; attempt++) {
|
||||
try {
|
||||
response = await fetch(PROVIDER_ENDPOINTS[provider], {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
signal: combinedSignal,
|
||||
})
|
||||
|
||||
if (response.status === 429 && attempt < MAX_RETRIES_ON_429) {
|
||||
const retryAfter = response.headers.get('retry-after')
|
||||
const waitMs = retryAfter ? Math.min(parseInt(retryAfter, 10) * 1000, 5000) : (attempt + 1) * 1500
|
||||
await new Promise(r => setTimeout(r, waitMs))
|
||||
continue
|
||||
}
|
||||
|
||||
break
|
||||
} catch (fetchErr: any) {
|
||||
if (fetchErr.name === 'AbortError') {
|
||||
throw new Error('Request timed out')
|
||||
}
|
||||
throw fetchErr
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
clearTimeout(timeout)
|
||||
|
||||
if (!response!.ok) {
|
||||
const errorData = await response!.json().catch(() => ({}))
|
||||
throw new Error(errorData.error?.message || `HTTP ${response!.status}`)
|
||||
}
|
||||
|
||||
const data = await response!.json()
|
||||
const content = data.choices?.[0]?.message?.content || ''
|
||||
|
||||
if (!content) throw new Error('Empty response')
|
||||
@@ -505,7 +538,7 @@ export async function queryModel(
|
||||
content,
|
||||
duration_ms: Date.now() - startTime,
|
||||
success: true,
|
||||
score: 0, // scored later
|
||||
score: 0,
|
||||
}
|
||||
} catch (err: any) {
|
||||
return {
|
||||
|
||||
@@ -656,20 +656,32 @@ chatRoutes.post('/completions', async (req, res) => {
|
||||
if (pipeline.finalParams.presence_penalty !== undefined) streamBody.presence_penalty = pipeline.finalParams.presence_penalty
|
||||
if (pipeline.finalParams.repetition_penalty !== undefined) streamBody.repetition_penalty = pipeline.finalParams.repetition_penalty
|
||||
|
||||
const upstreamRes = await fetch('https://openrouter.ai/api/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${openrouter_api_key}`,
|
||||
'Content-Type': 'application/json',
|
||||
'HTTP-Referer': 'https://godmod3.ai',
|
||||
'X-Title': 'GODMOD3.AI',
|
||||
},
|
||||
body: JSON.stringify(streamBody),
|
||||
})
|
||||
// Retry on 429 with exponential backoff (addresses rate-limit failures)
|
||||
let upstreamRes: Response | undefined
|
||||
for (let attempt = 0; attempt <= 2; attempt++) {
|
||||
upstreamRes = await fetch('https://openrouter.ai/api/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${openrouter_api_key}`,
|
||||
'Content-Type': 'application/json',
|
||||
'HTTP-Referer': 'https://godmod3.ai',
|
||||
'X-Title': 'GODMOD3.AI',
|
||||
},
|
||||
body: JSON.stringify(streamBody),
|
||||
})
|
||||
|
||||
if (!upstreamRes.ok) {
|
||||
const errData = await upstreamRes.json().catch(() => ({}))
|
||||
const errMsg = (errData as any).error?.message || `Upstream error: ${upstreamRes.status}`
|
||||
if (upstreamRes.status === 429 && attempt < 2) {
|
||||
const retryAfter = upstreamRes.headers.get('retry-after')
|
||||
const waitMs = retryAfter ? Math.min(parseInt(retryAfter, 10) * 1000, 5000) : (attempt + 1) * 1500
|
||||
await new Promise(r => setTimeout(r, waitMs))
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if (!upstreamRes!.ok) {
|
||||
const errData = await upstreamRes!.json().catch(() => ({}))
|
||||
const errMsg = (errData as any).error?.message || `Upstream error: ${upstreamRes!.status}`
|
||||
const chunk = {
|
||||
id: completionId,
|
||||
object: 'chat.completion.chunk',
|
||||
@@ -688,7 +700,7 @@ chatRoutes.post('/completions', async (req, res) => {
|
||||
return
|
||||
}
|
||||
|
||||
const reader = upstreamRes.body?.getReader()
|
||||
const reader = upstreamRes!.body?.getReader()
|
||||
if (!reader) {
|
||||
res.write('data: [DONE]\n\n')
|
||||
res.end()
|
||||
|
||||
Reference in New Issue
Block a user