mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-18 07:58:27 +00:00
fix(combo): retain skipped connection cooldown retry hints (#16718)
This commit is contained in:
1
changelog.d/fixes/16718-combo-connection-cooldown.md
Normal file
1
changelog.d/fixes/16718-combo-connection-cooldown.md
Normal file
@@ -0,0 +1 @@
|
||||
- Preserve a retryable response with `Retry-After` when a combo peer is temporarily unavailable at the credential gate and another target returns a payment failure (#16718).
|
||||
@@ -33,6 +33,8 @@ export type AttemptLoopState = {
|
||||
lastError: string | null;
|
||||
lastStatus: number | null;
|
||||
earliestRetryAfter: ComboRetryAfter | null;
|
||||
/** Deadline from a target skipped by the credential availability gate. */
|
||||
skippedConnectionCooldownUntil?: number;
|
||||
comboExpired: boolean;
|
||||
exhaustedProviders: Set<string>;
|
||||
exhaustedConnections: Set<string>;
|
||||
|
||||
@@ -64,6 +64,7 @@ import { executeTargetAttempt } from "./executeTargetAttempt.ts";
|
||||
import { buildComboDiag } from "./executeTargetClassify.ts";
|
||||
import type { AttemptLoopDeps, AttemptLoopState, ExecuteTargetResult } from "./attemptLoopTypes.ts";
|
||||
import { getStrategyTraits } from "./strategyRegistry.ts";
|
||||
import { resolveSkippedConnectionCooldownTerminal } from "./skippedConnectionCooldown.ts";
|
||||
|
||||
/** A second in-flight copy of a body larger than this sits in the TLS send buffer. */
|
||||
const HEDGE_MAX_BODY_BYTES = 256 * 1024;
|
||||
@@ -221,6 +222,7 @@ export async function dispatchWithCooldownRetry(opts: {
|
||||
state.transientRateLimitedProviders = new Set<string>();
|
||||
state.skippedForCircuitOpen = false;
|
||||
state.earliestCircuitOpenRetryMs = 0;
|
||||
state.skippedConnectionCooldownUntil = undefined;
|
||||
if (setTry > 0) {
|
||||
deps.log.info(
|
||||
"COMBO",
|
||||
@@ -701,7 +703,12 @@ export async function dispatchWithCooldownRetry(opts: {
|
||||
// classes (e.g. a quality failure + a sibling's 401) normalizes to a
|
||||
// 5xx-class status reflecting an infra/provider problem, not a client
|
||||
// error. See comboErrorAggregation.ts::resolveComboTerminalStatus.
|
||||
const status = resolveComboTerminalStatus(state.comboErrors, state.lastStatus);
|
||||
const terminal = resolveSkippedConnectionCooldownTerminal(
|
||||
resolveComboTerminalStatus(state.comboErrors, state.lastStatus),
|
||||
state
|
||||
);
|
||||
const status = terminal.status;
|
||||
state.earliestRetryAfter = terminal.retryAfter ?? state.earliestRetryAfter;
|
||||
// #10314: build the terminal message from the structured per-target
|
||||
// outcomes (each distinct class+reason listed separately) instead of
|
||||
// mashing a single state.lastError with raw `[model (status)]` markers. Connection
|
||||
@@ -777,7 +784,7 @@ export async function dispatchWithCooldownRetry(opts: {
|
||||
deps.log.warn("COMBO", `All models failed | ${msg} (${retryHuman})`);
|
||||
return withQuotaExhaustionClassification(
|
||||
unavailableResponse(status, msg, state.earliestRetryAfter, retryHuman),
|
||||
state.observedFailure ? state.allObservedFailuresQuota : null
|
||||
terminal.quotaExhausted
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ import { resolveQuotaExhaustionCutoffForTarget } from "./quotaExhaustionCutoff.t
|
||||
import type { ProtectedPriorityStopCause } from "./protectedPriorityStopStatus.ts";
|
||||
import type { AttemptLoopDeps, AttemptLoopState, GateDecision } from "./attemptLoopTypes.ts";
|
||||
import { modelAvailabilitySkipReason, type ResolvedComboTarget } from "./types.ts";
|
||||
import { recordSkippedConnectionCooldown } from "./skippedConnectionCooldown.ts";
|
||||
import type { PreDispatchExclusion } from "./pinRecovery.ts";
|
||||
import { getStrategyTraits } from "./strategyRegistry.ts";
|
||||
|
||||
@@ -350,6 +351,7 @@ export async function evaluateExecuteTargetGates(opts: {
|
||||
const available = await deps.isModelAvailable(modelStr, targetForAttempt);
|
||||
const skipReason = modelAvailabilitySkipReason(available);
|
||||
if (skipReason) {
|
||||
recordSkippedConnectionCooldown(state, available);
|
||||
deps.log.debug?.(
|
||||
"COMBO",
|
||||
skipReason === "model_not_in_catalog"
|
||||
|
||||
41
open-sse/services/combo/skippedConnectionCooldown.ts
Normal file
41
open-sse/services/combo/skippedConnectionCooldown.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
import type { AttemptLoopState } from "./attemptLoopTypes.ts";
|
||||
import type { ModelAvailabilityResult } from "./types.ts";
|
||||
|
||||
type CooldownState = Pick<AttemptLoopState, "skippedConnectionCooldownUntil">;
|
||||
|
||||
/** Retain only a finite deadline from the actual credential availability gate. */
|
||||
export function recordSkippedConnectionCooldown(
|
||||
state: CooldownState,
|
||||
availability: ModelAvailabilityResult,
|
||||
now = Date.now()
|
||||
): void {
|
||||
if (typeof availability !== "object" || availability?.reason !== "connection_cooldown") return;
|
||||
const delay = availability.retryAfterMs;
|
||||
const until = now + delay;
|
||||
if (!Number.isFinite(delay) || delay <= 0 || !Number.isFinite(until)) return;
|
||||
const previous = state.skippedConnectionCooldownUntil ?? 0;
|
||||
state.skippedConnectionCooldownUntil = previous > now ? Math.min(previous, until) : until;
|
||||
}
|
||||
|
||||
/** A peer's temporary cooldown must not become another account's payment failure. */
|
||||
export function resolveSkippedConnectionCooldownTerminal(
|
||||
status: number,
|
||||
state: CooldownState & Pick<AttemptLoopState, "observedFailure" | "allObservedFailuresQuota">,
|
||||
now = Date.now()
|
||||
): { status: number; retryAfter: string | null; quotaExhausted: boolean | null } {
|
||||
const until = state.skippedConnectionCooldownUntil ?? 0;
|
||||
if (status === 402 && Number.isFinite(until) && until > 0 && until <= 8.64e15) {
|
||||
// The peer may have recovered while another target was failing. It still
|
||||
// warrants an immediate client retry, not a permanent payment response.
|
||||
return {
|
||||
status: 429,
|
||||
retryAfter: new Date(Math.max(until, now + 1)).toISOString(),
|
||||
quotaExhausted: false,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status,
|
||||
retryAfter: null,
|
||||
quotaExhausted: state.observedFailure ? state.allObservedFailuresQuota : null,
|
||||
};
|
||||
}
|
||||
74
tests/unit/combo-connection-cooldown-policy-16718.test.ts
Normal file
74
tests/unit/combo-connection-cooldown-policy-16718.test.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
recordSkippedConnectionCooldown,
|
||||
resolveSkippedConnectionCooldownTerminal,
|
||||
} from "../../open-sse/services/combo/skippedConnectionCooldown.ts";
|
||||
|
||||
const NOW = Date.parse("2030-01-01T00:00:00Z");
|
||||
const createState = () => ({
|
||||
skippedConnectionCooldownUntil: undefined as number | undefined,
|
||||
observedFailure: true,
|
||||
allObservedFailuresQuota: true,
|
||||
});
|
||||
|
||||
test("selects the earliest eligible deadline without marking the whole combo exhausted", () => {
|
||||
const state = createState();
|
||||
for (const retryAfterMs of [20_000, 5000, 10_000]) {
|
||||
recordSkippedConnectionCooldown(
|
||||
state,
|
||||
{ available: false, reason: "connection_cooldown", retryAfterMs },
|
||||
NOW
|
||||
);
|
||||
}
|
||||
assert.deepEqual(resolveSkippedConnectionCooldownTerminal(402, state, NOW), {
|
||||
status: 429,
|
||||
retryAfter: new Date(NOW + 5000).toISOString(),
|
||||
quotaExhausted: false,
|
||||
});
|
||||
});
|
||||
|
||||
test("unknown, missing, and malformed cooldowns do not fabricate retry hints", () => {
|
||||
const state = createState();
|
||||
for (const retryAfterMs of [0, -1, NaN, Infinity]) {
|
||||
recordSkippedConnectionCooldown(
|
||||
state,
|
||||
{ available: false, reason: "connection_cooldown", retryAfterMs },
|
||||
NOW
|
||||
);
|
||||
}
|
||||
recordSkippedConnectionCooldown(state, false, NOW);
|
||||
recordSkippedConnectionCooldown(state, "model_not_in_catalog", NOW);
|
||||
assert.deepEqual(resolveSkippedConnectionCooldownTerminal(402, state, NOW), {
|
||||
status: 402,
|
||||
retryAfter: null,
|
||||
quotaExhausted: true,
|
||||
});
|
||||
});
|
||||
|
||||
test("a peer that recovered during dispatch still allows an immediate retry", () => {
|
||||
const state = createState();
|
||||
recordSkippedConnectionCooldown(
|
||||
state,
|
||||
{ available: false, reason: "connection_cooldown", retryAfterMs: 5000 },
|
||||
NOW
|
||||
);
|
||||
const result = resolveSkippedConnectionCooldownTerminal(402, state, NOW + 6000);
|
||||
assert.equal(result.status, 429);
|
||||
assert.equal(result.retryAfter, new Date(NOW + 6001).toISOString());
|
||||
});
|
||||
|
||||
test("request failures and other terminal classes retain their contract", () => {
|
||||
const state = createState();
|
||||
state.skippedConnectionCooldownUntil = NOW + 5000;
|
||||
for (const status of [400, 401, 403, 404, 422, 500, 503]) {
|
||||
assert.equal(resolveSkippedConnectionCooldownTerminal(status, state, NOW).status, status);
|
||||
assert.equal(resolveSkippedConnectionCooldownTerminal(status, state, NOW).retryAfter, null);
|
||||
}
|
||||
});
|
||||
|
||||
test("out-of-range dates cannot throw while building the terminal response", () => {
|
||||
const state = createState();
|
||||
state.skippedConnectionCooldownUntil = Number.MAX_VALUE;
|
||||
assert.equal(resolveSkippedConnectionCooldownTerminal(402, state, NOW).status, 402);
|
||||
});
|
||||
89
tests/unit/combo-connection-cooldown-terminal-16718.test.ts
Normal file
89
tests/unit/combo-connection-cooldown-terminal-16718.test.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { ModelAvailabilityResult } from "../../open-sse/services/combo/types.ts";
|
||||
|
||||
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-16718-"));
|
||||
process.env.DATA_DIR = dataDir;
|
||||
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
|
||||
const { resetDbInstance } = await import("../../src/lib/db/core.ts");
|
||||
const { clearAllModelLockouts } = await import("../../open-sse/services/accountFallback.ts");
|
||||
const { resetAllComboMetrics } = await import("../../open-sse/services/comboMetrics.ts");
|
||||
const log = { info() {}, warn() {}, error() {}, debug() {} };
|
||||
|
||||
test.beforeEach(() => {
|
||||
clearAllModelLockouts();
|
||||
resetAllComboMetrics();
|
||||
});
|
||||
test.after(() => {
|
||||
clearAllModelLockouts();
|
||||
resetDbInstance();
|
||||
fs.rmSync(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function run(models: string[], availability: ModelAvailabilityResult, failureStatus = 402) {
|
||||
const dispatched: string[] = [];
|
||||
const response = await handleComboChat({
|
||||
body: {},
|
||||
combo: {
|
||||
name: "connection-cooldown-16718",
|
||||
strategy: "priority",
|
||||
models,
|
||||
config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0 },
|
||||
},
|
||||
isModelAvailable: async (model) => (model.startsWith("anthropic/") ? availability : true),
|
||||
handleSingleModel: async (_body, model) => {
|
||||
dispatched.push(model);
|
||||
return model.startsWith("anthropic/")
|
||||
? Response.json({ choices: [{ message: { content: "recovered" } }] })
|
||||
: Response.json({ error: { message: "Payment required" } }, { status: failureStatus });
|
||||
},
|
||||
log,
|
||||
settings: {
|
||||
resilienceSettings: {
|
||||
comboCooldownWait: { enabled: false, maxWaitMs: 90_000, maxAttempts: 1, budgetMs: 90_000 },
|
||||
},
|
||||
},
|
||||
allCombos: null,
|
||||
});
|
||||
return { response, dispatched };
|
||||
}
|
||||
|
||||
for (const models of [
|
||||
["openai/gpt-4o", "anthropic/claude-sonnet-4"],
|
||||
["anthropic/claude-sonnet-4", "openai/gpt-4o"],
|
||||
]) {
|
||||
test(`a temporary peer makes 402 retryable in order ${models.join(", ")}`, async () => {
|
||||
const { response, dispatched } = await run(models, {
|
||||
available: false,
|
||||
reason: "connection_cooldown",
|
||||
retryAfterMs: 30_000,
|
||||
});
|
||||
assert.equal(response.status, 429);
|
||||
const retryAfter = Number(response.headers.get("retry-after"));
|
||||
assert.ok(retryAfter > 0 && retryAfter <= 30, `expected a bounded Retry-After: ${retryAfter}`);
|
||||
assert.deepEqual(dispatched, ["openai/gpt-4o"]);
|
||||
});
|
||||
}
|
||||
|
||||
test("an unavailable peer without a cooldown does not change payment failure", async () => {
|
||||
const { response } = await run(["openai/gpt-4o", "anthropic/claude-sonnet-4"], false);
|
||||
assert.equal(response.status, 402);
|
||||
assert.equal(response.headers.get("retry-after"), null);
|
||||
});
|
||||
|
||||
test("a ready peer still serves the request", async () => {
|
||||
const { response } = await run(["openai/gpt-4o", "anthropic/claude-sonnet-4"], true);
|
||||
assert.equal(response.status, 200);
|
||||
});
|
||||
|
||||
test("a request error is not converted into a payment cooldown retry", async () => {
|
||||
const { response } = await run(
|
||||
["openai/gpt-4o", "anthropic/claude-sonnet-4"],
|
||||
{ available: false, reason: "connection_cooldown", retryAfterMs: 30_000 },
|
||||
400
|
||||
);
|
||||
assert.equal(response.status, 400);
|
||||
});
|
||||
Reference in New Issue
Block a user