Merge #19592 (head fc4b408e) into contributor train e1b

This commit is contained in:
diegosouzapw
2026-11-21 03:46:20 -03:00
7 changed files with 89 additions and 47 deletions

View File

@@ -2613,6 +2613,13 @@ APP_LOG_TO_FILE=true
# Used by: src/lib/healthzLag.ts (event-loop stall sampler). Disabled by default;
# logs sizes and inflight count only, never bodies or heap snapshots.
# OMNIROUTE_EVENT_LOOP_STALL_THRESHOLD_MS=0 stops that sampler, and this diagnostic with it.
# Optional response header for troubleshooting Codex OAuth plan eligibility.
# OFF by default: when enabled, API callers can see the recognized subscription tier
# of the selected Codex connection in X-OmniRoute-Selected-Connection-Plan.
# Do not enable on public/multi-tenant gateways if this account metadata is private.
# Arbitrary/unrecognized plan values and non-Codex requests never get this header.
# OMNIROUTE_CODEX_PLAN_RESPONSE_HEADER=0
# OMNIROUTE_HEAP_DIAGNOSTICS_ENABLED=0
# Heap-use threshold in MiB for the opt-in diagnostic (default: 900).
# A high-heap log is rate-limited to once per 60 seconds.

View File

@@ -0,0 +1 @@
- Add an opt-in `X-OmniRoute-Selected-Connection-Plan` response header for Codex account-routing troubleshooting (`OMNIROUTE_CODEX_PLAN_RESPONSE_HEADER=1`), off by default with an explicit plan allowlist.

View File

@@ -929,6 +929,7 @@ REQUEST_TIMEOUT_MS (global override)
| `OMNIROUTE_AGENT_GOAL_READINESS_MAX_TIMEOUT_MS` | `600000` | Maximum first-event readiness window for detected `/goal` agent runs or requests forced with `x-omniroute-agent-goal`. |
| `OMNIROUTE_AGENT_GOAL_STREAM_RECOVERY` | `true` | Enable early stream recovery automatically for detected `/goal` agent runs. Set `false`/`0`/`off` to disable the goal-specific opt-in. This can only ADD recovery on top of the operator default — it never overrides an explicit `STREAM_RECOVERY_ENABLED`/DB settings opt-out. |
| `OMNIROUTE_CODEX_DROP_NONSTANDARD_EVENTS` | `true` | Strip non-standard `codex.*` SSE events (e.g. `codex.rate_limits`) that break the OpenAI SDK's `responses.stream()` with a 502. Default ON (#11014). Set `0`/`false`/`no`/`off` to forward them. |
| `OMNIROUTE_CODEX_PLAN_RESPONSE_HEADER` | `0` (off) | Opt-in Codex OAuth troubleshooting response header. Set to `1` to emit `X-OmniRoute-Selected-Connection-Plan` with the recognized plan tier of the selected Codex connection. Not sent for non-Codex requests or unrecognized plans. Avoid enabling on public or multi-tenant gateways where subscription metadata is private. |
| `OMNIROUTE_CODEX_APPSERVER_WS` | _(unset)_ | Opt-in Codex app-server transport. WebSocket endpoint (`ws://`/`wss://`) of a local `codex app-server` sidecar. When set together with a token, Codex requests are routed over JSON-RPC to the sidecar instead of the HTTP Responses API. Also settable per-connection via `providerSpecificData.codexAppServerUrl`. Used by `open-sse/executors/codex/appServerConfig.ts`. |
| `OMNIROUTE_CODEX_APPSERVER_WS_TOKEN` | _(unset)_ | Inline capability/bearer token presented to the app-server. Per-connection override: `providerSpecificData.codexAppServerToken`. |
| `OMNIROUTE_CODEX_APPSERVER_WS_TOKEN_FILE` | _(unset)_ | Path to a file holding the app-server capability token (from `codex app-server --ws-token-file`). Used when `OMNIROUTE_CODEX_APPSERVER_WS_TOKEN` is unset. Per-connection override: `providerSpecificData.codexAppServerTokenFile`. |

View File

@@ -11,6 +11,7 @@ export const OMNIROUTE_RESPONSE_HEADERS = {
fallbackAttempts: "X-OmniRoute-Fallback-Attempts",
accountFallbacks: "X-OmniRoute-Account-Fallbacks",
accountFallbackReasons: "X-OmniRoute-Account-Fallback-Reasons",
selectedConnectionPlan: "X-OmniRoute-Selected-Connection-Plan",
sameAccountRetries: "X-OmniRoute-Same-Account-Retries",
routingWaitMs: "X-OmniRoute-Routing-Wait-Ms",
latencyMs: "X-OmniRoute-Latency-Ms",

View File

@@ -38,6 +38,7 @@ import {
resolveAvalancheSettings,
} from "@omniroute/open-sse/services/accountFallback/antiAvalanche.ts";
import { estimateTokens } from "@omniroute/open-sse/services/contextManager.ts";
import { getCodexPlanType } from "@omniroute/open-sse/services/codexPlanEligibility.ts";
import { getCombo, getComboForModel, getModelInfo } from "../services/model";
import { stripContextWindowSuffix } from "@omniroute/open-sse/services/model.ts";
import { resolveBareModelToConnectionDefault } from "@omniroute/open-sse/services/model.ts";
@@ -112,8 +113,6 @@ import { comboCheckProvider, ghComboGate } from "./chat/githubLiveCatalogFilter.
import { markEmergencyFallback } from "./emergencyFallbackHeader.ts";
import { resolveEmergencyFallbackDecision } from "./emergencyFallbackHeader.ts";
import { resolveInternalUpstreamStartTimeoutMs } from "../services/internalUpstreamStartTimeout";
import { resolveResilienceSettings } from "@/lib/resilience/settings";
import { resolvePreferredConnectionUpstreamStartTimeoutMs } from "../services/preferredConnectionFailover";
import { evaluateComboTargetPreflight } from "./chat/comboTargetKeyPolicy.ts";
import * as resolvedPolicy from "./chat/resolvedModelPolicy.ts";
import { recordGateRejection, recordQuotaParkedSkip } from "./quotaParkedSkipUsage";
@@ -1822,13 +1821,9 @@ async function handleSingleModelChat(
});
const userAgent = request?.headers?.get("user-agent") || "";
const cachedRuntimeSettings =
runtimeOptions.cachedSettings ?? (await getCachedSettings().catch(() => ({})));
const baseRetrySettings = resolveCooldownAwareRetrySettings(cachedRuntimeSettings);
// The read cache invalidates on settings writes (up to its 5s TTL).
// No restart is needed for newly dispatched chat requests.
const preferredFailoverSettings =
resolveResilienceSettings(cachedRuntimeSettings).preferredAccountFailover;
const baseRetrySettings = resolveCooldownAwareRetrySettings(
runtimeOptions.cachedSettings ?? (await getCachedSettings().catch(() => ({})))
);
const retrySettings = disableCooldownAwareRetry(
baseRetrySettings,
provider === "claude-web" ||
@@ -1868,14 +1863,12 @@ async function handleSingleModelChat(
// STREAM_EARLY_EOF_SIBLING_FAILOVER_ENABLED: at most ONE sibling hop per request. Keeps the
// original early-EOF 502 so an exhausted sibling pool surfaces it verbatim (combo detection).
let earlyEofOriginal: Response | null = null;
// Preferred connection start timeout is a single-hop routing hint, never a cooldown.
let preferredTimeoutOriginal: Response | null = null;
let preferredFailoverUsed = false;
const sameAccountTransportRetries = new Map<string, number>();
// An obsolete Codex selection/response is not a fault of the current account. Reselect once
// without cooling or excluding its connection ID; never replay a successful response.
let staleCodexReselected = false;
let accountFallbacks = 0;
let selectedConnectionPlan: string | null = null;
let sameAccountRetries = 0;
let routingWaitMs = 0;
const accountFallbackReasons: string[] = [];
@@ -1887,6 +1880,7 @@ async function handleSingleModelChat(
};
const routingTelemetry = () => ({
accountFallbacks,
selectedConnectionPlan,
accountFallbackReasons,
sameAccountRetries,
routingWaitMs,
@@ -2027,7 +2021,6 @@ async function handleSingleModelChat(
"allExpired" in credentials ||
!credentials.connectionId
) {
if (preferredTimeoutOriginal) return preferredTimeoutOriginal;
if (earlyEofOriginal) return withRoutingTelemetry(earlyEofOriginal, null);
if (!credentials?.allRateLimited && agy.earliestRetryHintAtMs !== null)
return agyLease.buildAntigravityPoolBusyResponse(agy.earliestRetryHintAtMs);
@@ -2126,6 +2119,8 @@ async function handleSingleModelChat(
excludedConnectionIds.size > 0
? Array.from(excludedConnectionIds)[excludedConnectionIds.size - 1]
: null;
// No account was selected for this response; don't report a previous attempt's plan.
selectedConnectionPlan = null;
return withRoutingTelemetry(noCredsRes, lastFailedConnectionId);
}
@@ -2148,6 +2143,9 @@ async function handleSingleModelChat(
}
pendingStreamRetryConnectionId = null;
}
// Refresh on every selected credential so fallback never reports a stale plan.
selectedConnectionPlan =
provider === "codex" ? getCodexPlanType(credentials.providerSpecificData) : null;
const accountId = credentials.connectionId.slice(0, 8);
const releaseOAuthSession = credentials.releaseOAuthSession ?? (() => {});
// Undefined whenever the lease flag is off, which makes every release/hold below a no-op.
@@ -2310,21 +2308,6 @@ async function handleSingleModelChat(
reselectPoolMember?: () => Promise<unknown>;
} = { proxy: null };
const proxyStartTime = Date.now();
// A preferred API-key account can be given one shorter time-to-headers
// deadline when a vetted sibling exists. Pins/leases/combo/Muse are excluded.
const preferredDeadline = resolvePreferredConnectionUpstreamStartTimeoutMs(
provider,
credentials,
requestUpstreamStartTimeoutMs,
effectiveModel,
preferredFailoverUsed ||
hasForcedConnection ||
runtimeOptions.managedLease ||
museOwner ||
isCombo
? { ...preferredFailoverSettings, enabled: false }
: preferredFailoverSettings
);
// 4. Execute chat via core after breaker gate checks (with optional TLS tracking)
if (telemetry) telemetry.startPhase("connect");
let execution: Awaited<ReturnType<typeof dispatchChatWithAffinityEviction>>;
@@ -2372,7 +2355,7 @@ async function handleSingleModelChat(
videoBridgeLog: runtimeOptions.videoBridgeLog,
previousResponseResumed: runtimeOptions.previousResponseResumed,
fallbackAttempts: runtimeOptions.fallbackAttempts,
requestUpstreamStartTimeoutMs: preferredDeadline.timeoutMs,
requestUpstreamStartTimeoutMs,
beforeUpstreamAttempt:
museOwner && muse.museBeforeUpstreamAttempt(museOwner, credentials),
forcedConnectionId: hasForcedConnection ? forcedConnectionId : null, // #14116
@@ -2491,23 +2474,6 @@ async function handleSingleModelChat(
// A shorter *preferred* deadline may perform one in-request sibling hop,
// without recording a connection failure or evicting session affinity.
if (result.errorCode === REQUEST_SCOPED_UPSTREAM_TIMEOUT_CODE) {
if (
preferredDeadline.mayRotateOnRequestTimeout &&
!preferredFailoverUsed &&
requestSignal?.aborted !== true
) {
preferredFailoverUsed = true;
preferredTimeoutOriginal = withSelectedConnectionHeader(
result.response,
credentials.connectionId
);
excludedConnectionIds.add(credentials.connectionId);
log.info(
"AUTH",
"Preferred account response-start timed out; trying one eligible sibling"
);
continue;
}
return withSelectedConnectionHeader(result.response, credentials.connectionId);
}
// Never let generic account fallback or affinity eviction move a Muse caller.

View File

@@ -10,6 +10,7 @@ import { inheritResponsePolicies } from "./responsePolicies.ts";
export type AccountRoutingTelemetry = {
accountFallbacks?: number;
accountFallbackReasons?: readonly string[];
selectedConnectionPlan?: string | null;
sameAccountRetries?: number;
routingWaitMs?: number;
};
@@ -37,6 +38,30 @@ function fallbackReasons(values: readonly string[] | undefined): string[] {
.filter((reason) => SAFE_FALLBACK_REASONS.has(reason) || /^http_[1-5][0-9]{2}$/.test(reason));
}
// Only explicitly enabled server operators can expose recognized subscription tiers.
// An arbitrary provider field is never copied into a response header.
const CODEX_PLAN_TIERS = new Set([
"free",
"go",
"plus",
"pro",
"prolite",
"promax",
"team",
"chatgptteam",
"business",
"enterprise",
"edu",
"education",
]);
function selectedCodexPlan(value: string | null | undefined): string | null {
if (process.env.OMNIROUTE_CODEX_PLAN_RESPONSE_HEADER !== "1") return null;
if (typeof value !== "string") return null;
const normalized = value.trim().toLowerCase().replace(/[\s_-]+/g, "");
return CODEX_PLAN_TIERS.has(normalized) ? normalized : null;
}
export function withSelectedConnectionHeader(
response: Response,
connectionId: string | null | undefined,
@@ -44,15 +69,24 @@ export function withSelectedConnectionHeader(
): Response {
if (!response) return response;
const fallbacks = nonnegativeCount(telemetry?.accountFallbacks);
const plan = selectedCodexPlan(telemetry?.selectedConnectionPlan);
const retries = nonnegativeCount(telemetry?.sameAccountRetries);
const waitMs = nonnegativeCount(telemetry?.routingWaitMs);
const reasons = fallbackReasons(telemetry?.accountFallbackReasons);
if (!connectionId && fallbacks === 0 && retries === 0 && waitMs === 0 && reasons.length === 0) {
if (
!connectionId &&
fallbacks === 0 &&
retries === 0 &&
waitMs === 0 &&
reasons.length === 0 &&
!plan
) {
return response;
}
const apply = (headers: Headers) => {
if (connectionId) headers.set("X-OmniRoute-Selected-Connection-Id", connectionId);
if (plan) headers.set(OMNIROUTE_RESPONSE_HEADERS.selectedConnectionPlan, plan);
if (fallbacks > 0) headers.set(OMNIROUTE_RESPONSE_HEADERS.accountFallbacks, String(fallbacks));
if (reasons.length) {
headers.set(OMNIROUTE_RESPONSE_HEADERS.accountFallbackReasons, reasons.join(","));

View File

@@ -32,6 +32,38 @@ test("telemetry headers are bounded and never echo untrusted errors", () => {
assert.equal(response.headers.get("X-OmniRoute-Selected-Connection-Plan"), null);
});
test("Codex account plan diagnostics are off by default and allowlisted when enabled", () => {
const key = "OMNIROUTE_CODEX_PLAN_RESPONSE_HEADER";
const before = process.env[key];
try {
delete process.env[key];
const hidden = withSelectedConnectionHeader(new Response("ok"), "c", {
selectedConnectionPlan: "Plus",
});
assert.equal(hidden.headers.get("X-OmniRoute-Selected-Connection-Plan"), null);
process.env[key] = "1";
const visible = withSelectedConnectionHeader(new Response("ok"), "c", {
selectedConnectionPlan: "Plus",
});
assert.equal(visible.headers.get("X-OmniRoute-Selected-Connection-Plan"), "plus");
const normalized = withSelectedConnectionHeader(new Response("ok"), "c", {
selectedConnectionPlan: "Pro Max",
});
assert.equal(normalized.headers.get("X-OmniRoute-Selected-Connection-Plan"), "promax");
for (const unsafe of ["custom-internal-secret", "Plus\r\nInjected: true", ""]) {
const response = withSelectedConnectionHeader(new Response("ok"), "c", {
selectedConnectionPlan: unsafe,
});
assert.equal(response.headers.get("X-OmniRoute-Selected-Connection-Plan"), null);
}
} finally {
if (before === undefined) delete process.env[key];
else process.env[key] = before;
}
});
test("immutable response clone retains trusted response policies alongside telemetry", async () => {
const { markProviderProbeResponse, isProviderProbeResponse } =
await import("../../src/shared/utils/providerProbeResult.ts");