fix(combo): retain skipped connection cooldown retry hints (#16718)

This commit is contained in:
diegosouzapw
2026-10-15 15:06:14 -03:00
parent 6a1437e6e2
commit f6cd5487d2
7 changed files with 218 additions and 2 deletions

View File

@@ -0,0 +1 @@
- Preserve a retryable response with `Retry-After` when a combo peer is temporarily unavailable at the credential gate and another target returns a payment failure (#16718).

View File

@@ -33,6 +33,8 @@ export type AttemptLoopState = {
lastError: string | null;
lastStatus: number | null;
earliestRetryAfter: ComboRetryAfter | null;
/** Deadline from a target skipped by the credential availability gate. */
skippedConnectionCooldownUntil?: number;
comboExpired: boolean;
exhaustedProviders: Set<string>;
exhaustedConnections: Set<string>;

View File

@@ -64,6 +64,7 @@ import { executeTargetAttempt } from "./executeTargetAttempt.ts";
import { buildComboDiag } from "./executeTargetClassify.ts";
import type { AttemptLoopDeps, AttemptLoopState, ExecuteTargetResult } from "./attemptLoopTypes.ts";
import { getStrategyTraits } from "./strategyRegistry.ts";
import { resolveSkippedConnectionCooldownTerminal } from "./skippedConnectionCooldown.ts";
/** A second in-flight copy of a body larger than this sits in the TLS send buffer. */
const HEDGE_MAX_BODY_BYTES = 256 * 1024;
@@ -221,6 +222,7 @@ export async function dispatchWithCooldownRetry(opts: {
state.transientRateLimitedProviders = new Set<string>();
state.skippedForCircuitOpen = false;
state.earliestCircuitOpenRetryMs = 0;
state.skippedConnectionCooldownUntil = undefined;
if (setTry > 0) {
deps.log.info(
"COMBO",
@@ -701,7 +703,12 @@ export async function dispatchWithCooldownRetry(opts: {
// classes (e.g. a quality failure + a sibling's 401) normalizes to a
// 5xx-class status reflecting an infra/provider problem, not a client
// error. See comboErrorAggregation.ts::resolveComboTerminalStatus.
const status = resolveComboTerminalStatus(state.comboErrors, state.lastStatus);
const terminal = resolveSkippedConnectionCooldownTerminal(
resolveComboTerminalStatus(state.comboErrors, state.lastStatus),
state
);
const status = terminal.status;
state.earliestRetryAfter = terminal.retryAfter ?? state.earliestRetryAfter;
// #10314: build the terminal message from the structured per-target
// outcomes (each distinct class+reason listed separately) instead of
// mashing a single state.lastError with raw `[model (status)]` markers. Connection
@@ -777,7 +784,7 @@ export async function dispatchWithCooldownRetry(opts: {
deps.log.warn("COMBO", `All models failed | ${msg} (${retryHuman})`);
return withQuotaExhaustionClassification(
unavailableResponse(status, msg, state.earliestRetryAfter, retryHuman),
state.observedFailure ? state.allObservedFailuresQuota : null
terminal.quotaExhausted
);
}

View File

@@ -35,6 +35,7 @@ import { resolveQuotaExhaustionCutoffForTarget } from "./quotaExhaustionCutoff.t
import type { ProtectedPriorityStopCause } from "./protectedPriorityStopStatus.ts";
import type { AttemptLoopDeps, AttemptLoopState, GateDecision } from "./attemptLoopTypes.ts";
import { modelAvailabilitySkipReason, type ResolvedComboTarget } from "./types.ts";
import { recordSkippedConnectionCooldown } from "./skippedConnectionCooldown.ts";
import type { PreDispatchExclusion } from "./pinRecovery.ts";
import { getStrategyTraits } from "./strategyRegistry.ts";
@@ -350,6 +351,7 @@ export async function evaluateExecuteTargetGates(opts: {
const available = await deps.isModelAvailable(modelStr, targetForAttempt);
const skipReason = modelAvailabilitySkipReason(available);
if (skipReason) {
recordSkippedConnectionCooldown(state, available);
deps.log.debug?.(
"COMBO",
skipReason === "model_not_in_catalog"

View File

@@ -0,0 +1,41 @@
import type { AttemptLoopState } from "./attemptLoopTypes.ts";
import type { ModelAvailabilityResult } from "./types.ts";
type CooldownState = Pick<AttemptLoopState, "skippedConnectionCooldownUntil">;
/** Retain only a finite deadline from the actual credential availability gate. */
export function recordSkippedConnectionCooldown(
state: CooldownState,
availability: ModelAvailabilityResult,
now = Date.now()
): void {
if (typeof availability !== "object" || availability?.reason !== "connection_cooldown") return;
const delay = availability.retryAfterMs;
const until = now + delay;
if (!Number.isFinite(delay) || delay <= 0 || !Number.isFinite(until)) return;
const previous = state.skippedConnectionCooldownUntil ?? 0;
state.skippedConnectionCooldownUntil = previous > now ? Math.min(previous, until) : until;
}
/** A peer's temporary cooldown must not become another account's payment failure. */
export function resolveSkippedConnectionCooldownTerminal(
status: number,
state: CooldownState & Pick<AttemptLoopState, "observedFailure" | "allObservedFailuresQuota">,
now = Date.now()
): { status: number; retryAfter: string | null; quotaExhausted: boolean | null } {
const until = state.skippedConnectionCooldownUntil ?? 0;
if (status === 402 && Number.isFinite(until) && until > 0 && until <= 8.64e15) {
// The peer may have recovered while another target was failing. It still
// warrants an immediate client retry, not a permanent payment response.
return {
status: 429,
retryAfter: new Date(Math.max(until, now + 1)).toISOString(),
quotaExhausted: false,
};
}
return {
status,
retryAfter: null,
quotaExhausted: state.observedFailure ? state.allObservedFailuresQuota : null,
};
}

View File

@@ -0,0 +1,74 @@
import test from "node:test";
import assert from "node:assert/strict";
import {
recordSkippedConnectionCooldown,
resolveSkippedConnectionCooldownTerminal,
} from "../../open-sse/services/combo/skippedConnectionCooldown.ts";
const NOW = Date.parse("2030-01-01T00:00:00Z");
const createState = () => ({
skippedConnectionCooldownUntil: undefined as number | undefined,
observedFailure: true,
allObservedFailuresQuota: true,
});
test("selects the earliest eligible deadline without marking the whole combo exhausted", () => {
const state = createState();
for (const retryAfterMs of [20_000, 5000, 10_000]) {
recordSkippedConnectionCooldown(
state,
{ available: false, reason: "connection_cooldown", retryAfterMs },
NOW
);
}
assert.deepEqual(resolveSkippedConnectionCooldownTerminal(402, state, NOW), {
status: 429,
retryAfter: new Date(NOW + 5000).toISOString(),
quotaExhausted: false,
});
});
test("unknown, missing, and malformed cooldowns do not fabricate retry hints", () => {
const state = createState();
for (const retryAfterMs of [0, -1, NaN, Infinity]) {
recordSkippedConnectionCooldown(
state,
{ available: false, reason: "connection_cooldown", retryAfterMs },
NOW
);
}
recordSkippedConnectionCooldown(state, false, NOW);
recordSkippedConnectionCooldown(state, "model_not_in_catalog", NOW);
assert.deepEqual(resolveSkippedConnectionCooldownTerminal(402, state, NOW), {
status: 402,
retryAfter: null,
quotaExhausted: true,
});
});
test("a peer that recovered during dispatch still allows an immediate retry", () => {
const state = createState();
recordSkippedConnectionCooldown(
state,
{ available: false, reason: "connection_cooldown", retryAfterMs: 5000 },
NOW
);
const result = resolveSkippedConnectionCooldownTerminal(402, state, NOW + 6000);
assert.equal(result.status, 429);
assert.equal(result.retryAfter, new Date(NOW + 6001).toISOString());
});
test("request failures and other terminal classes retain their contract", () => {
const state = createState();
state.skippedConnectionCooldownUntil = NOW + 5000;
for (const status of [400, 401, 403, 404, 422, 500, 503]) {
assert.equal(resolveSkippedConnectionCooldownTerminal(status, state, NOW).status, status);
assert.equal(resolveSkippedConnectionCooldownTerminal(status, state, NOW).retryAfter, null);
}
});
test("out-of-range dates cannot throw while building the terminal response", () => {
const state = createState();
state.skippedConnectionCooldownUntil = Number.MAX_VALUE;
assert.equal(resolveSkippedConnectionCooldownTerminal(402, state, NOW).status, 402);
});

View File

@@ -0,0 +1,89 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import type { ModelAvailabilityResult } from "../../open-sse/services/combo/types.ts";
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-16718-"));
process.env.DATA_DIR = dataDir;
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
const { resetDbInstance } = await import("../../src/lib/db/core.ts");
const { clearAllModelLockouts } = await import("../../open-sse/services/accountFallback.ts");
const { resetAllComboMetrics } = await import("../../open-sse/services/comboMetrics.ts");
const log = { info() {}, warn() {}, error() {}, debug() {} };
test.beforeEach(() => {
clearAllModelLockouts();
resetAllComboMetrics();
});
test.after(() => {
clearAllModelLockouts();
resetDbInstance();
fs.rmSync(dataDir, { recursive: true, force: true });
});
async function run(models: string[], availability: ModelAvailabilityResult, failureStatus = 402) {
const dispatched: string[] = [];
const response = await handleComboChat({
body: {},
combo: {
name: "connection-cooldown-16718",
strategy: "priority",
models,
config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0 },
},
isModelAvailable: async (model) => (model.startsWith("anthropic/") ? availability : true),
handleSingleModel: async (_body, model) => {
dispatched.push(model);
return model.startsWith("anthropic/")
? Response.json({ choices: [{ message: { content: "recovered" } }] })
: Response.json({ error: { message: "Payment required" } }, { status: failureStatus });
},
log,
settings: {
resilienceSettings: {
comboCooldownWait: { enabled: false, maxWaitMs: 90_000, maxAttempts: 1, budgetMs: 90_000 },
},
},
allCombos: null,
});
return { response, dispatched };
}
for (const models of [
["openai/gpt-4o", "anthropic/claude-sonnet-4"],
["anthropic/claude-sonnet-4", "openai/gpt-4o"],
]) {
test(`a temporary peer makes 402 retryable in order ${models.join(", ")}`, async () => {
const { response, dispatched } = await run(models, {
available: false,
reason: "connection_cooldown",
retryAfterMs: 30_000,
});
assert.equal(response.status, 429);
const retryAfter = Number(response.headers.get("retry-after"));
assert.ok(retryAfter > 0 && retryAfter <= 30, `expected a bounded Retry-After: ${retryAfter}`);
assert.deepEqual(dispatched, ["openai/gpt-4o"]);
});
}
test("an unavailable peer without a cooldown does not change payment failure", async () => {
const { response } = await run(["openai/gpt-4o", "anthropic/claude-sonnet-4"], false);
assert.equal(response.status, 402);
assert.equal(response.headers.get("retry-after"), null);
});
test("a ready peer still serves the request", async () => {
const { response } = await run(["openai/gpt-4o", "anthropic/claude-sonnet-4"], true);
assert.equal(response.status, 200);
});
test("a request error is not converted into a payment cooldown retry", async () => {
const { response } = await run(
["openai/gpt-4o", "anthropic/claude-sonnet-4"],
{ available: false, reason: "connection_cooldown", retryAfterMs: 30_000 },
400
);
assert.equal(response.status, 400);
});