mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-17 01:20:28 +00:00
feat(antigravity): opt-in sanitized physical-send timing telemetry (#19133)
## Problem Antigravity can take a long time to begin responding. We need trustworthy phase evidence without persisting prompts, credentials or response contents. ## Implementation - Opt-in bounded timing tracker **at the actual Antigravity physical-send boundary**, including internal retries and both bounded and unbounded readiness paths. - Report time to headers, first meaningful Gemini model output, account idle time and concurrent physical sends. Report unobservable DNS/TCP/TLS fields as null. - Sanitize emitted account identifiers/model aliases; leave live transport unaffected when disabled. Abandoned successful streams expire **observation state only**. - **Hot operator control:** Settings → Resilience → Antigravity timing diagnostics, persisted as `resilienceSettings.antigravityTimingTelemetry.enabled`. The physical-send executor reads cached settings for each send, so it covers non-chat and retry flows too (5s settings TTL; no restart). A settings-cache read error gracefully falls back to the env flag, not a failed provider send. - Defaults to OFF; `OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY=true` remains an env fallback, overridden by an explicitly saved boolean. - Legacy fork `resilienceSettings.antigravity.timingTelemetryEnabled` is also recognized. ## Coverage Existing physical-send/stream timing tests plus `tests/unit/antigravity-timing-hot-settings.test.ts` for normalized persistence, validated PATCH, explicit false, legacy fallback and physical-send configuration Thanks @Bl0ck154.
This commit is contained in:
@@ -4194,6 +4194,9 @@ QUOTA_STORE_DRIVER=sqlite
|
||||
# Opt-in model-sync phase timings; disabled unless exactly 1.
|
||||
OMNIROUTE_MODEL_SYNC_TIMINGS=0
|
||||
|
||||
# Opt-in sanitized per-physical-send Antigravity phase timings (no request bodies).
|
||||
OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY=false
|
||||
|
||||
# Optional local OpenCode media-analysis tool settings (scripts/cli/omniroute-media-mcp.mjs).
|
||||
# These configure the local tool process, not the hosted gateway.
|
||||
# OMNIROUTE_MEDIA_CONFIG=
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
- **feat(antigravity):** add opt-in, per-account Antigravity physical-send phase timing diagnostics without logging request content or credentials ([#19133](https://github.com/cheaper-inference/OmniRoute/pull/19133)) — thanks @Bl0ck154
|
||||
@@ -0,0 +1 @@
|
||||
- **feat(antigravity):** Expose physical-send timing diagnostics as a saved Settings → Resilience toggle. New sends use updated values through the settings cache without server restart.
|
||||
@@ -651,6 +651,7 @@ detection above).
|
||||
| `OMNIROUTE_BOOTSTRAPPED` | `false` | `src/app/(dashboard)/dashboard/page.tsx` | Set `true` by bootstrap script after initial setup. Controls setup wizard visibility. |
|
||||
| `OMNIROUTE_ALLOW_BODY_PROJECT_OVERRIDE` | `0` | `open-sse/executors/antigravity.ts` | Escape hatch: allow request body to override the Antigravity project field. |
|
||||
| `ANTIGRAVITY_CREDITS` | `off` | `open-sse/services/antigravityCredits.ts` | Google One AI credits policy: `off` never injects credits, `retry` injects once after an eligible quota 429, and `always` injects on the first request. |
|
||||
| `OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY` | `false` | `open-sse/executors/antigravity/executeAttempt.ts` | Opt-in sanitized per-physical-send timing: account idle, in-flight requests, time to headers and first meaningful SSE content. No prompts, credentials, or upstream URLs in logs. |
|
||||
| `ANTIGRAVITY_MAX_429_ATTEMPTS` | `2` | `open-sse/services/antigravity429Guard.ts` | Upstream 429s tolerated per Antigravity (`agy`/`antigravity`) request across all hosts (1–12) before the 429 goes back to the caller. A body that spent it is answered locally on re-send (2 min, doubling to 30 min, or the upstream retry hint), and the chat handler does not restart the request after that 429. |
|
||||
| `ANTIGRAVITY_ALLOW_SIGNATURE_BYPASS` | `0` | `open-sse/translator/request/openai-to-gemini.ts` | Allow the Antigravity request translator to skip its strict CLI request-signature validation when the upstream refuses real signatures (debug/antiquated-CLI mode). Non-zero enables the bypass. |
|
||||
| `AGY_TOKEN_FILE` | `~/.gemini/antigravity-cli/antigravity-oauth-token` | `src/app/api/providers/agy-auth/apply-local/route.ts` | Override the Antigravity CLI (agy) token-file path for the auto-detect local login import. |
|
||||
|
||||
@@ -32,6 +32,10 @@ import {
|
||||
removeHeaderCaseInsensitive,
|
||||
} from "../../services/antigravityClientProfile.ts";
|
||||
import * as prl from "../../utils/providerRequestLogging.ts";
|
||||
import { withAntigravityTiming, timeAntigravityFetch } from "../../services/antigravityTiming.ts";
|
||||
import * as timingLog from "@/sse/utils/logger";
|
||||
import { getCachedSettings } from "@/lib/db/readCache";
|
||||
import { resolveAntigravityTimingTelemetryEnabled } from "../../services/antigravityTimingConfig.ts";
|
||||
import { generateAntigravityRequestId } from "../../services/antigravityIdentity.ts";
|
||||
import {
|
||||
createCreditsExtractionTransform as createCreditsExtractionTransformImpl,
|
||||
@@ -150,6 +154,39 @@ export function isAbortError(error: unknown): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
type PhysicalSendTiming = { correlationId: string | null; ordinal: number };
|
||||
|
||||
async function observeTimedAntigravitySend(
|
||||
metadata: HttpAttemptMetadata,
|
||||
send: () => Promise<Response>,
|
||||
signal?: AbortSignal | null,
|
||||
timing?: PhysicalSendTiming
|
||||
): Promise<Response> {
|
||||
// Resolve cached operator preferences before the HTTP-attempt timer starts:
|
||||
// settings I/O must not be counted as upstream header latency.
|
||||
const envEnabled = process.env.OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY === "true";
|
||||
const enabled = metadata.connectionId
|
||||
? await getCachedSettings()
|
||||
.then((saved) => resolveAntigravityTimingTelemetryEnabled(saved, envEnabled))
|
||||
.catch(() => envEnabled)
|
||||
: false;
|
||||
return prl.observeProviderHttpSend(metadata, () => {
|
||||
if (!enabled || !metadata.connectionId) return send();
|
||||
return withAntigravityTiming(
|
||||
{
|
||||
enabled: true,
|
||||
connectionId: metadata.connectionId,
|
||||
model: metadata.model,
|
||||
correlationId: timing?.correlationId,
|
||||
physicalSendOrdinal: timing?.ordinal,
|
||||
emit: (event) =>
|
||||
timingLog.info("ANTIGRAVITY_TIMING", "Antigravity physical-send timing", event),
|
||||
},
|
||||
() => timeAntigravityFetch(send, signal)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* `fetch()` wrapper that aborts if the upstream never returns response headers
|
||||
* within `timeoutMs` (default FETCH_TIMEOUT_MS, the global startup budget).
|
||||
@@ -160,18 +197,21 @@ export async function fetchAntigravityWithReadinessTimeout(
|
||||
url: string,
|
||||
init: RequestInit,
|
||||
timeoutMs = FETCH_TIMEOUT_MS,
|
||||
metadata?: HttpAttemptMetadata
|
||||
metadata?: HttpAttemptMetadata,
|
||||
timing?: PhysicalSendTiming
|
||||
): Promise<Response> {
|
||||
const boundedTimeoutMs = Math.max(0, Math.floor(timeoutMs));
|
||||
if (boundedTimeoutMs <= 0) {
|
||||
return prl.observeProviderHttpSend(
|
||||
return observeTimedAntigravitySend(
|
||||
metadata ?? {
|
||||
provider: "antigravity",
|
||||
model: "unknown",
|
||||
connectionId: null,
|
||||
endpoint: "antigravity_generate",
|
||||
},
|
||||
() => fetch(url, init)
|
||||
() => fetch(url, init),
|
||||
init.signal,
|
||||
timing
|
||||
);
|
||||
}
|
||||
|
||||
@@ -186,14 +226,16 @@ export async function fetchAntigravityWithReadinessTimeout(
|
||||
: timeoutController.signal;
|
||||
|
||||
try {
|
||||
return await prl.observeProviderHttpSend(
|
||||
return await observeTimedAntigravitySend(
|
||||
metadata ?? {
|
||||
provider: "antigravity",
|
||||
model: "unknown",
|
||||
connectionId: null,
|
||||
endpoint: "antigravity_generate",
|
||||
},
|
||||
() => fetch(url, { ...init, signal: combinedSignal })
|
||||
() => fetch(url, { ...init, signal: combinedSignal }),
|
||||
combinedSignal,
|
||||
timing
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
@@ -405,7 +447,8 @@ export async function sendAntigravityRequest(
|
||||
model: getRequestTargetModel(transformedBody),
|
||||
connectionId: credentials.connectionId ?? null,
|
||||
endpoint: "antigravity_generate",
|
||||
}
|
||||
},
|
||||
{ correlationId, ordinal: physicalSendOrdinal }
|
||||
);
|
||||
|
||||
if (response.status === HTTP_STATUS.FORBIDDEN && finalHeaders["x-goog-user-project"]) {
|
||||
@@ -433,7 +476,8 @@ export async function sendAntigravityRequest(
|
||||
connectionId: credentials.connectionId ?? null,
|
||||
endpoint: "antigravity_generate",
|
||||
retryReason: "quota_header_removed_after_403",
|
||||
}
|
||||
},
|
||||
{ correlationId, ordinal: retryPhysicalSendOrdinal }
|
||||
);
|
||||
finalHeaders = retryHeaders;
|
||||
}
|
||||
@@ -497,7 +541,8 @@ export async function sendAntigravityRequest(
|
||||
model: reResolvedModel,
|
||||
connectionId: credentials.connectionId ?? null,
|
||||
endpoint: "antigravity_generate",
|
||||
}
|
||||
},
|
||||
{ correlationId, ordinal: syncRetryPhysicalSendOrdinal }
|
||||
);
|
||||
finalHeaders = retryHeaders;
|
||||
}
|
||||
@@ -599,7 +644,8 @@ export async function tryCreditsRetry(
|
||||
model: getRequestTargetModel(creditsBody),
|
||||
connectionId: credentials.connectionId ?? null,
|
||||
endpoint: "antigravity_generate",
|
||||
}
|
||||
},
|
||||
{ correlationId, ordinal: creditsPhysicalSendOrdinal }
|
||||
);
|
||||
if (creditsResp.ok || creditsResp.status !== HTTP_STATUS.RATE_LIMITED) {
|
||||
log.info("AG_CREDITS", `Credits retry succeeded: ${creditsResp.status}`);
|
||||
|
||||
341
open-sse/services/antigravityTiming.ts
Normal file
341
open-sse/services/antigravityTiming.ts
Normal file
@@ -0,0 +1,341 @@
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { resolveAntigravityModelId } from "../config/antigravityModelAliases.ts";
|
||||
import { isAntigravityPreResponseTimeout } from "./antigravityTimeout.ts";
|
||||
|
||||
function normalizeAntigravityTimingModel(value: string): string {
|
||||
const normalized = value
|
||||
.trim()
|
||||
.replace(/^(?:antigravity|agy)\//i, "")
|
||||
.toLowerCase();
|
||||
if (!/^[a-z0-9_.:-]{1,160}$/.test(normalized)) return "unknown";
|
||||
return resolveAntigravityModelId(normalized);
|
||||
}
|
||||
|
||||
type TimingContext = {
|
||||
connectionId: string;
|
||||
model: string;
|
||||
requestId: string;
|
||||
sends: number;
|
||||
emit: (event: Record<string, unknown>) => void;
|
||||
};
|
||||
const context = new AsyncLocalStorage<TimingContext>();
|
||||
const MAX_ACCOUNTS = 1000;
|
||||
const MAX_EVENTS_PER_ATTEMPT = 24;
|
||||
// A forgotten/unconsumed successful response must not accumulate as inflight forever.
|
||||
// This timeout affects observation only; it never aborts fetches or response streams.
|
||||
const MAX_TIMING_OBSERVATION_MS = 300_000;
|
||||
type AccountTiming = { inflight: number; lastFinishedAt: number | null };
|
||||
|
||||
/** Only allow actual Gemini output to set first delta; pings/errors/usage do not. */
|
||||
function isMeaningfulPart(value: unknown): boolean {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const part = value as {
|
||||
text?: unknown;
|
||||
functionCall?: { name?: unknown };
|
||||
inlineData?: { data?: unknown };
|
||||
executableCode?: unknown;
|
||||
codeExecutionResult?: unknown;
|
||||
};
|
||||
return (
|
||||
(typeof part.text === "string" && part.text.length > 0) ||
|
||||
(typeof part.functionCall?.name === "string" && part.functionCall.name.length > 0) ||
|
||||
(typeof part.inlineData?.data === "string" && part.inlineData.data.length > 0) ||
|
||||
Boolean(part.executableCode || part.codeExecutionResult)
|
||||
);
|
||||
}
|
||||
|
||||
function candidateHasMeaningfulPart(value: unknown): boolean {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const candidate = value as { content?: { parts?: unknown[] } };
|
||||
return Array.isArray(candidate.content?.parts) && candidate.content.parts.some(isMeaningfulPart);
|
||||
}
|
||||
|
||||
export function hasAntigravityMeaningfulDelta(frame: string): boolean {
|
||||
for (const line of frame.split(/\r?\n/)) {
|
||||
if (!line.startsWith("data:")) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(line.slice(5).trim()) as Record<string, unknown>;
|
||||
const response = (parsed.response ?? parsed) as { candidates?: unknown[] };
|
||||
if (
|
||||
Array.isArray(response.candidates) &&
|
||||
response.candidates.some(candidateHasMeaningfulPart)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
} catch {
|
||||
/* Partial, invalid or terminal frame: no measured delta. */
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function createDeltaInspector(onFirstDelta: () => void): (bytes: Uint8Array) => void {
|
||||
const decoder = new TextDecoder();
|
||||
let pending = "";
|
||||
let skipOversizedFrame = false;
|
||||
let found = false;
|
||||
return (bytes) => {
|
||||
if (found) return;
|
||||
if (bytes.byteLength > 65536) {
|
||||
pending = "";
|
||||
skipOversizedFrame = true;
|
||||
return;
|
||||
}
|
||||
pending += decoder.decode(bytes, { stream: true });
|
||||
for (;;) {
|
||||
const boundary = pending.search(/\r?\n\r?\n/);
|
||||
if (boundary < 0) break;
|
||||
const frame = pending.slice(0, boundary);
|
||||
pending = pending.slice(boundary).replace(/^\r?\n\r?\n/, "");
|
||||
if (!skipOversizedFrame && hasAntigravityMeaningfulDelta(frame)) {
|
||||
found = true;
|
||||
pending = "";
|
||||
onFirstDelta();
|
||||
return;
|
||||
}
|
||||
skipOversizedFrame = false;
|
||||
}
|
||||
if (pending.length > 65536) {
|
||||
pending = "";
|
||||
skipOversizedFrame = true;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function wrapTrackedResponseBody(
|
||||
response: Response,
|
||||
handlers: {
|
||||
finish: (phase: string) => void;
|
||||
onChunk: () => void;
|
||||
onFirstDelta: () => void;
|
||||
isFinished: () => boolean;
|
||||
}
|
||||
): Response {
|
||||
const reader = response.body!.getReader();
|
||||
const inspect = createDeltaInspector(handlers.onFirstDelta);
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
try {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) {
|
||||
handlers.finish("complete");
|
||||
reader.releaseLock();
|
||||
controller.close();
|
||||
} else {
|
||||
handlers.onChunk();
|
||||
if (!handlers.isFinished()) inspect(chunk.value);
|
||||
controller.enqueue(chunk.value);
|
||||
}
|
||||
} catch (error) {
|
||||
handlers.finish("body_error");
|
||||
reader.releaseLock();
|
||||
controller.error(error);
|
||||
}
|
||||
},
|
||||
async cancel(reason) {
|
||||
handlers.finish("cancelled");
|
||||
try {
|
||||
await reader.cancel(reason);
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
},
|
||||
});
|
||||
return new Response(body, {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: response.headers,
|
||||
});
|
||||
}
|
||||
|
||||
/** Reuse bounded per-account telemetry state without altering dispatch eligibility. */
|
||||
function claimTimingAccount(
|
||||
accounts: Map<string, AccountTiming>,
|
||||
connectionId: string
|
||||
): AccountTiming | null {
|
||||
let account = accounts.get(connectionId);
|
||||
if (account) return account;
|
||||
if (accounts.size >= MAX_ACCOUNTS) {
|
||||
const idle = [...accounts].find(([, item]) => item.inflight === 0);
|
||||
if (!idle) return null;
|
||||
accounts.delete(idle[0]);
|
||||
}
|
||||
account = { inflight: 0, lastFinishedAt: null };
|
||||
accounts.set(connectionId, account);
|
||||
return account;
|
||||
}
|
||||
|
||||
function classifyTimingAbort(signal: AbortSignal | null | undefined): string {
|
||||
if (isAntigravityPreResponseTimeout(signal?.reason)) return "pre_response_timeout";
|
||||
if ((signal?.reason as { name?: string } | undefined)?.name === "TimeoutError") {
|
||||
return "upstream_deadline";
|
||||
}
|
||||
return "caller_abort";
|
||||
}
|
||||
|
||||
function measureIdleDuration(account: AccountTiming, started: number): number | null {
|
||||
if (account.inflight > 0) return 0;
|
||||
return account.lastFinishedAt === null ? null : Math.max(0, started - account.lastFinishedAt);
|
||||
}
|
||||
|
||||
type PhysicalSendState = {
|
||||
scope: TimingContext;
|
||||
started: number;
|
||||
idleDurationMs: number | null;
|
||||
concurrentInflight: number;
|
||||
physicalSend: number;
|
||||
headersAt: number | null;
|
||||
firstDeltaAt: number | null;
|
||||
};
|
||||
|
||||
function buildTimingEvent(
|
||||
phase: string,
|
||||
send: PhysicalSendState,
|
||||
now: number,
|
||||
extra: Record<string, unknown>
|
||||
): Record<string, unknown> {
|
||||
const { scope, started, headersAt, firstDeltaAt } = send;
|
||||
// Correlation ids stay log fields, never metric labels. No URLs or payloads.
|
||||
return {
|
||||
event: "antigravity_timing",
|
||||
phase,
|
||||
requestId: scope.requestId,
|
||||
account: createHash("sha256").update(scope.connectionId).digest("hex").slice(0, 16),
|
||||
model: normalizeAntigravityTimingModel(scope.model),
|
||||
physicalSend: send.physicalSend,
|
||||
idleDurationMs: send.idleDurationMs,
|
||||
concurrentInflight: send.concurrentInflight,
|
||||
dnsMs: null,
|
||||
connectMs: null,
|
||||
tlsMs: null,
|
||||
headerLatencyMs: headersAt === null ? null : headersAt - started,
|
||||
firstMeaningfulDeltaFromHeadersMs:
|
||||
firstDeltaAt === null || headersAt === null ? null : firstDeltaAt - headersAt,
|
||||
firstMeaningfulDeltaFromSendMs: firstDeltaAt === null ? null : firstDeltaAt - started,
|
||||
elapsedMs: Math.max(0, now - started),
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
export class AntigravityTimingTracker {
|
||||
private accounts = new Map<string, AccountTiming>();
|
||||
constructor(
|
||||
private now: () => number = () => performance.now(),
|
||||
private maxObservationMs: number = MAX_TIMING_OBSERVATION_MS
|
||||
) {}
|
||||
|
||||
async fetch(send: () => Promise<Response>, signal?: AbortSignal | null): Promise<Response> {
|
||||
const scope = context.getStore();
|
||||
if (!scope) return send();
|
||||
const account = claimTimingAccount(this.accounts, scope.connectionId);
|
||||
if (!account) return send();
|
||||
const started = this.now();
|
||||
const attempt: PhysicalSendState = {
|
||||
scope,
|
||||
started,
|
||||
idleDurationMs: measureIdleDuration(account, started),
|
||||
concurrentInflight: ++account.inflight,
|
||||
physicalSend: ++scope.sends,
|
||||
headersAt: null,
|
||||
firstDeltaAt: null,
|
||||
};
|
||||
let finished = false;
|
||||
let observationTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
const event = (phase: string, extra: Record<string, unknown> = {}) => {
|
||||
if (attempt.physicalSend > MAX_EVENTS_PER_ATTEMPT / 3) return;
|
||||
try {
|
||||
scope.emit(buildTimingEvent(phase, attempt, this.now(), extra));
|
||||
} catch {
|
||||
/* Diagnostics cannot change request behavior. */
|
||||
}
|
||||
};
|
||||
const finish = (phase: string) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
if (observationTimer !== null) clearTimeout(observationTimer);
|
||||
account!.inflight--;
|
||||
account!.lastFinishedAt = this.now();
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
event(phase);
|
||||
};
|
||||
const abortPhase = () => classifyTimingAbort(signal);
|
||||
const onAbort = () => finish(abortPhase());
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
// Bound *telemetry state*, not upstream execution. This also protects HTTP 200
|
||||
// streams that a caller abandons without reading or cancelling their bodies.
|
||||
const armObservationTimer = () => {
|
||||
if (finished) return;
|
||||
if (observationTimer !== null) clearTimeout(observationTimer);
|
||||
observationTimer = setTimeout(() => finish("observation_expired"), this.maxObservationMs);
|
||||
observationTimer.unref?.();
|
||||
};
|
||||
armObservationTimer();
|
||||
try {
|
||||
const response = await send();
|
||||
attempt.headersAt = this.now();
|
||||
event("headers", { status: response.status });
|
||||
if (signal?.aborted) finish(abortPhase());
|
||||
// Antigravity can return an intermediate 403/429 that the executor retries
|
||||
// without consuming the body. Such a physical fetch has already completed
|
||||
// at the HTTP-header boundary; retaining an unread-body wrapper here leaks
|
||||
// the inflight counter and corrupts idle/concurrency measurements.
|
||||
// Return the original response untouched so the executor can still inspect
|
||||
// any error body or retry it exactly as before.
|
||||
if (!response.ok) {
|
||||
finish("http_error");
|
||||
return response;
|
||||
}
|
||||
if (!response.body) {
|
||||
finish("complete");
|
||||
return response;
|
||||
}
|
||||
return wrapTrackedResponseBody(response, {
|
||||
finish,
|
||||
onChunk: armObservationTimer,
|
||||
isFinished: () => finished,
|
||||
onFirstDelta: () => {
|
||||
attempt.firstDeltaAt = this.now();
|
||||
event("first_meaningful_delta");
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
finish(
|
||||
isAntigravityPreResponseTimeout(error)
|
||||
? "pre_response_timeout"
|
||||
: signal?.aborted
|
||||
? abortPhase()
|
||||
: "fetch_failed"
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const tracker = new AntigravityTimingTracker();
|
||||
export const timeAntigravityFetch = (send: () => Promise<Response>, signal?: AbortSignal | null) =>
|
||||
tracker.fetch(send, signal);
|
||||
|
||||
export function withAntigravityTiming<T>(
|
||||
options: {
|
||||
enabled: boolean;
|
||||
connectionId: string;
|
||||
model: string;
|
||||
correlationId?: string | null;
|
||||
physicalSendOrdinal?: number;
|
||||
emit: TimingContext["emit"];
|
||||
},
|
||||
fn: () => T
|
||||
): T {
|
||||
if (!options.enabled) return fn();
|
||||
const requestId = /^[0-9a-f]{8}-[0-9a-f-]{27}$/i.test(options.correlationId ?? "")
|
||||
? options.correlationId!
|
||||
: randomUUID();
|
||||
const sends =
|
||||
options.physicalSendOrdinal &&
|
||||
Number.isInteger(options.physicalSendOrdinal) &&
|
||||
options.physicalSendOrdinal > 0
|
||||
? options.physicalSendOrdinal - 1
|
||||
: 0;
|
||||
return context.run({ ...options, requestId, sends }, fn);
|
||||
}
|
||||
29
open-sse/services/antigravityTimingConfig.ts
Normal file
29
open-sse/services/antigravityTimingConfig.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* Read the current Antigravity timing preference without coupling this
|
||||
* physical-send transport boundary to the dashboard/settings module.
|
||||
* Explicit saved values take precedence; old fork DBs used a nested
|
||||
* resilienceSettings.antigravity.timingTelemetryEnabled field.
|
||||
*/
|
||||
export function resolveAntigravityTimingTelemetryEnabled(
|
||||
settings: unknown,
|
||||
envEnabled: boolean
|
||||
): boolean {
|
||||
if (!settings || typeof settings !== "object" || Array.isArray(settings))
|
||||
return envEnabled;
|
||||
const root = settings as Record<string, unknown>;
|
||||
const resilience = root.resilienceSettings;
|
||||
if (!resilience || typeof resilience !== "object" || Array.isArray(resilience))
|
||||
return envEnabled;
|
||||
const stored = resilience as Record<string, unknown>;
|
||||
const next = stored.antigravityTimingTelemetry;
|
||||
if (next && typeof next === "object" && !Array.isArray(next)) {
|
||||
const enabled = (next as Record<string, unknown>).enabled;
|
||||
if (typeof enabled === "boolean") return enabled;
|
||||
}
|
||||
const legacy = stored.antigravity;
|
||||
if (legacy && typeof legacy === "object" && !Array.isArray(legacy)) {
|
||||
const enabled = (legacy as Record<string, unknown>).timingTelemetryEnabled;
|
||||
if (typeof enabled === "boolean") return enabled;
|
||||
}
|
||||
return envEnabled;
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { Button, Card } from "@/shared/components";
|
||||
import { BooleanField } from "./ResilienceFields";
|
||||
|
||||
export type AntigravityTimingTelemetryValue = { enabled: boolean };
|
||||
|
||||
export default function AntigravityTimingTelemetryCard({
|
||||
value, saving, onSave,
|
||||
}: {
|
||||
value: AntigravityTimingTelemetryValue;
|
||||
saving: boolean;
|
||||
onSave: (value: AntigravityTimingTelemetryValue) => Promise<void>;
|
||||
}) {
|
||||
const [enabled, setEnabled] = useState(value.enabled);
|
||||
const [previous, setPrevious] = useState(value.enabled);
|
||||
if (previous !== value.enabled) {
|
||||
setPrevious(value.enabled);
|
||||
setEnabled(value.enabled);
|
||||
}
|
||||
|
||||
return (
|
||||
<Card className="space-y-4 p-6">
|
||||
<h2 className="text-lg font-bold">Antigravity timing diagnostics</h2>
|
||||
<p className="text-sm text-text-muted">
|
||||
Record sanitized timing for actual Antigravity network sends, including
|
||||
response headers, first meaningful model delta, idle time and
|
||||
concurrency. Raw prompts, credentials and response frames are not logged.
|
||||
Changes apply to new physical sends without restarting the server.
|
||||
</p>
|
||||
<BooleanField
|
||||
label="Enable physical-send timing diagnostics"
|
||||
description="Off by default. Disabled monitoring has no routing effect."
|
||||
checked={enabled}
|
||||
onChange={setEnabled}
|
||||
/>
|
||||
<div className="flex gap-2">
|
||||
<Button size="sm" variant="secondary" disabled={saving}
|
||||
onClick={() => setEnabled(value.enabled)}>Cancel</Button>
|
||||
<Button size="sm" variant="primary" icon="save" disabled={saving || enabled === value.enabled}
|
||||
onClick={() => void onSave({ enabled }).catch(() => {})}>Save</Button>
|
||||
</div>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -11,6 +11,9 @@ import ModelLockoutCard from "./ModelLockoutCard";
|
||||
import TokenRefreshBreakerCard, { type TokenRefreshBreakerValue } from "./TokenRefreshBreakerCard";
|
||||
import { NumberField, BooleanField } from "./ResilienceFields";
|
||||
import { throwIfResilienceSaveFailed } from "./resilienceSaveError";
|
||||
import AntigravityTimingTelemetryCard, {
|
||||
type AntigravityTimingTelemetryValue,
|
||||
} from "./AntigravityTimingTelemetryCard";
|
||||
|
||||
type RequestQueueSettings = {
|
||||
autoEnableApiKeyProviders: boolean;
|
||||
@@ -65,6 +68,7 @@ export type ProviderCooldownSettings = {
|
||||
};
|
||||
|
||||
type ResilienceResponse = {
|
||||
antigravityTimingTelemetry?: AntigravityTimingTelemetryValue;
|
||||
requestQueue: RequestQueueSettings;
|
||||
connectionCooldown: {
|
||||
oauth: ConnectionCooldownProfileSettings;
|
||||
@@ -84,6 +88,7 @@ type ResilienceResponse = {
|
||||
|
||||
function toResilienceResponse(json: ResilienceResponse): ResilienceResponse {
|
||||
return {
|
||||
antigravityTimingTelemetry: json.antigravityTimingTelemetry,
|
||||
requestQueue: json.requestQueue,
|
||||
connectionCooldown: json.connectionCooldown,
|
||||
providerBreaker: json.providerBreaker,
|
||||
@@ -1214,6 +1219,15 @@ export default function ResilienceTab() {
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<AutoDisableCard />
|
||||
{data.antigravityTimingTelemetry && (
|
||||
<AntigravityTimingTelemetryCard
|
||||
value={data.antigravityTimingTelemetry}
|
||||
saving={savingSection === "antigravityTimingTelemetry"}
|
||||
onSave={(antigravityTimingTelemetry) =>
|
||||
savePatch("antigravityTimingTelemetry", { antigravityTimingTelemetry })
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
<RequestQueueCard
|
||||
value={data.requestQueue}
|
||||
|
||||
@@ -144,6 +144,7 @@ export async function GET() {
|
||||
const resilience = resolveResilienceSettings(settings);
|
||||
|
||||
return NextResponse.json({
|
||||
antigravityTimingTelemetry: resilience.antigravityTimingTelemetry,
|
||||
requestQueue: resilience.requestQueue,
|
||||
connectionCooldown: resilience.connectionCooldown,
|
||||
providerBreaker: resilience.providerBreaker,
|
||||
@@ -210,6 +211,12 @@ export async function PATCH(request) {
|
||||
resilienceSettings: storedResilience,
|
||||
});
|
||||
const mergedFull = mergeResilienceSettings(sparseBase, {
|
||||
...(body.antigravityTimingTelemetry
|
||||
? {
|
||||
antigravityTimingTelemetry:
|
||||
body.antigravityTimingTelemetry as ResilienceSettingsPatch["antigravityTimingTelemetry"],
|
||||
}
|
||||
: {}),
|
||||
...(body.requestQueue
|
||||
? { requestQueue: body.requestQueue as ResilienceSettingsPatch["requestQueue"] }
|
||||
: {}),
|
||||
@@ -311,6 +318,7 @@ export async function PATCH(request) {
|
||||
|
||||
return NextResponse.json({
|
||||
ok: true,
|
||||
antigravityTimingTelemetry: nextResilience.antigravityTimingTelemetry,
|
||||
requestQueue: nextResilience.requestQueue,
|
||||
connectionCooldown: nextResilience.connectionCooldown,
|
||||
providerBreaker: nextResilience.providerBreaker,
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
import type { JsonRecord, ResilienceSettings, ResilienceSettingsPatch } from "./settings/types";
|
||||
import {
|
||||
asRecord,
|
||||
toBoolean,
|
||||
toInteger,
|
||||
resolveStreamRecoveryDefaults,
|
||||
normalizeLegacyConnectionCooldownProfile,
|
||||
@@ -83,6 +84,9 @@ export const DEFAULT_REQUEST_QUEUE_MAX_DEPTH = (() => {
|
||||
})();
|
||||
|
||||
export const DEFAULT_RESILIENCE_SETTINGS: ResilienceSettings = {
|
||||
antigravityTimingTelemetry: {
|
||||
enabled: process.env.OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY === "true",
|
||||
},
|
||||
requestQueue: {
|
||||
autoEnableApiKeyProviders: true,
|
||||
requestsPerMinute: DEFAULT_API_LIMITS.requestsPerMinute,
|
||||
@@ -227,6 +231,37 @@ export const DEFAULT_RESILIENCE_SETTINGS: ResilienceSettings = {
|
||||
},
|
||||
};
|
||||
|
||||
// Normalize legacy and modern opt-in flags without inflating the large resolvers.
|
||||
function normalizeAntigravityToggle(
|
||||
section: unknown,
|
||||
legacyEnabled: unknown,
|
||||
fallback: boolean
|
||||
): { enabled: boolean } {
|
||||
return { enabled: toBoolean(asRecord(section).enabled, toBoolean(legacyEnabled, fallback)) };
|
||||
}
|
||||
|
||||
function normalizeCooldownProfiles(
|
||||
source: unknown,
|
||||
fallback: ResilienceSettings["connectionCooldown"]
|
||||
): ResilienceSettings["connectionCooldown"] {
|
||||
const sections = asRecord(source);
|
||||
return {
|
||||
oauth: normalizeConnectionCooldownProfile(sections.oauth, fallback.oauth),
|
||||
apikey: normalizeConnectionCooldownProfile(sections.apikey, fallback.apikey),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeBreakerProfiles(
|
||||
source: unknown,
|
||||
fallback: ResilienceSettings["providerBreaker"]
|
||||
): ResilienceSettings["providerBreaker"] {
|
||||
const sections = asRecord(source);
|
||||
return {
|
||||
oauth: normalizeProviderBreakerProfile(sections.oauth, fallback.oauth),
|
||||
apikey: normalizeProviderBreakerProfile(sections.apikey, fallback.apikey),
|
||||
};
|
||||
}
|
||||
|
||||
function buildLegacyFallback(settings: JsonRecord): ResilienceSettings {
|
||||
const profiles = asRecord(settings.providerProfiles);
|
||||
const defaults = asRecord(settings.rateLimitDefaults);
|
||||
@@ -329,6 +364,7 @@ function buildLegacyFallback(settings: JsonRecord): ResilienceSettings {
|
||||
streamRecovery: streamRecoveryDefaults,
|
||||
providerQuotaOverrides: DEFAULT_RESILIENCE_SETTINGS.providerQuotaOverrides,
|
||||
credentialHealthCheck: DEFAULT_RESILIENCE_SETTINGS.credentialHealthCheck,
|
||||
antigravityTimingTelemetry: DEFAULT_RESILIENCE_SETTINGS.antigravityTimingTelemetry,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -364,30 +400,21 @@ export function resolveResilienceSettings(
|
||||
if (cached) return cached;
|
||||
|
||||
const current = asRecord(record.resilienceSettings);
|
||||
const legacyAntigravitySettings = asRecord(current.antigravity);
|
||||
const fallback = buildLegacyFallback(record);
|
||||
|
||||
const resolved: ResilienceSettings = {
|
||||
antigravityTimingTelemetry: normalizeAntigravityToggle(
|
||||
current.antigravityTimingTelemetry,
|
||||
legacyAntigravitySettings.timingTelemetryEnabled,
|
||||
fallback.antigravityTimingTelemetry.enabled
|
||||
),
|
||||
requestQueue: normalizeRequestQueueSettings(current.requestQueue, fallback.requestQueue),
|
||||
connectionCooldown: {
|
||||
oauth: normalizeConnectionCooldownProfile(
|
||||
asRecord(current.connectionCooldown).oauth,
|
||||
fallback.connectionCooldown.oauth
|
||||
),
|
||||
apikey: normalizeConnectionCooldownProfile(
|
||||
asRecord(current.connectionCooldown).apikey,
|
||||
fallback.connectionCooldown.apikey
|
||||
),
|
||||
},
|
||||
providerBreaker: {
|
||||
oauth: normalizeProviderBreakerProfile(
|
||||
asRecord(current.providerBreaker).oauth,
|
||||
fallback.providerBreaker.oauth
|
||||
),
|
||||
apikey: normalizeProviderBreakerProfile(
|
||||
asRecord(current.providerBreaker).apikey,
|
||||
fallback.providerBreaker.apikey
|
||||
),
|
||||
},
|
||||
connectionCooldown: normalizeCooldownProfiles(
|
||||
current.connectionCooldown,
|
||||
fallback.connectionCooldown
|
||||
),
|
||||
providerBreaker: normalizeBreakerProfiles(current.providerBreaker, fallback.providerBreaker),
|
||||
tokenRefreshBreaker: normalizeTokenRefreshBreakerSettings(
|
||||
current.tokenRefreshBreaker,
|
||||
fallback.tokenRefreshBreaker
|
||||
@@ -440,27 +467,15 @@ export function mergeResilienceSettings(
|
||||
updates: ResilienceSettingsPatch
|
||||
): ResilienceSettings {
|
||||
return {
|
||||
antigravityTimingTelemetry: normalizeAntigravityToggle(
|
||||
updates.antigravityTimingTelemetry, undefined, current.antigravityTimingTelemetry.enabled
|
||||
),
|
||||
requestQueue: normalizeRequestQueueSettings(updates.requestQueue, current.requestQueue),
|
||||
connectionCooldown: {
|
||||
oauth: normalizeConnectionCooldownProfile(
|
||||
updates.connectionCooldown?.oauth,
|
||||
current.connectionCooldown.oauth
|
||||
),
|
||||
apikey: normalizeConnectionCooldownProfile(
|
||||
updates.connectionCooldown?.apikey,
|
||||
current.connectionCooldown.apikey
|
||||
),
|
||||
},
|
||||
providerBreaker: {
|
||||
oauth: normalizeProviderBreakerProfile(
|
||||
updates.providerBreaker?.oauth,
|
||||
current.providerBreaker.oauth
|
||||
),
|
||||
apikey: normalizeProviderBreakerProfile(
|
||||
updates.providerBreaker?.apikey,
|
||||
current.providerBreaker.apikey
|
||||
),
|
||||
},
|
||||
connectionCooldown: normalizeCooldownProfiles(
|
||||
updates.connectionCooldown,
|
||||
current.connectionCooldown
|
||||
),
|
||||
providerBreaker: normalizeBreakerProfiles(updates.providerBreaker, current.providerBreaker),
|
||||
tokenRefreshBreaker: normalizeTokenRefreshBreakerSettings(
|
||||
updates.tokenRefreshBreaker,
|
||||
current.tokenRefreshBreaker
|
||||
|
||||
@@ -277,7 +277,13 @@ export interface StreamThroughputWatchdogSettings {
|
||||
minUsefulBytes: number;
|
||||
}
|
||||
|
||||
/** Controls sanitized timing of the real Antigravity physical HTTP sends. */
|
||||
export interface AntigravityTimingTelemetrySettings {
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface ResilienceSettings {
|
||||
antigravityTimingTelemetry: AntigravityTimingTelemetrySettings;
|
||||
requestQueue: RequestQueueSettings;
|
||||
connectionCooldown: Record<AuthCategory, ConnectionCooldownProfileSettings>;
|
||||
providerBreaker: Record<AuthCategory, ProviderBreakerProfileSettings>;
|
||||
@@ -295,6 +301,7 @@ export interface ResilienceSettings {
|
||||
}
|
||||
|
||||
export interface ResilienceSettingsPatch {
|
||||
antigravityTimingTelemetry?: Partial<AntigravityTimingTelemetrySettings>;
|
||||
requestQueue?: Partial<RequestQueueSettings>;
|
||||
connectionCooldown?: Partial<Record<AuthCategory, Partial<ConnectionCooldownProfileSettings>>>;
|
||||
providerBreaker?: Partial<Record<AuthCategory, Partial<ProviderBreakerProfileSettings>>>;
|
||||
|
||||
@@ -212,6 +212,7 @@ export const tokenRefreshBreakerSettingsSchema = z
|
||||
|
||||
export const updateResilienceSchema = z
|
||||
.object({
|
||||
antigravityTimingTelemetry: z.object({ enabled: z.boolean().optional() }).strict().optional(),
|
||||
requestQueue: requestQueueSettingsSchema.optional(),
|
||||
connectionCooldown: z
|
||||
.object({
|
||||
@@ -270,6 +271,7 @@ export const updateResilienceSchema = z
|
||||
.superRefine((value, ctx) => {
|
||||
const provided = [
|
||||
value.requestQueue,
|
||||
value.antigravityTimingTelemetry,
|
||||
value.connectionCooldown,
|
||||
value.providerBreaker,
|
||||
value.tokenRefreshBreaker,
|
||||
|
||||
@@ -46,6 +46,7 @@
|
||||
"tests/unit/upstream-response-decoded-byte-cap.test.ts",
|
||||
"tests/unit/vision-bridge-model-lockout-18268.test.ts",
|
||||
"tests/unit/antigravity-headers-budget.test.ts",
|
||||
"tests/unit/antigravity-timing-hot-settings.test.ts",
|
||||
"tests/unit/log-truncation-image-ref.test.ts",
|
||||
"tests/unit/m365-native-recovery.test.ts",
|
||||
"tests/unit/m365-native-routing.test.ts",
|
||||
|
||||
66
tests/unit/antigravity-timing-hot-settings.test.ts
Normal file
66
tests/unit/antigravity-timing-hot-settings.test.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
DEFAULT_RESILIENCE_SETTINGS,
|
||||
mergeResilienceSettings,
|
||||
pruneResilienceSettingsToProvided,
|
||||
resolveResilienceSettings,
|
||||
} from "../../src/lib/resilience/settings.ts";
|
||||
import { updateResilienceSchema } from "../../src/shared/validation/schemas/settings.ts";
|
||||
import { resolveAntigravityTimingTelemetryEnabled } from "../../open-sse/services/antigravityTimingConfig.ts";
|
||||
|
||||
test("timing telemetry persisted override applies to new physical sends", () => {
|
||||
const defaults = resolveResilienceSettings(null);
|
||||
const overridden = mergeResilienceSettings(defaults, {
|
||||
antigravityTimingTelemetry: { enabled: !defaults.antigravityTimingTelemetry.enabled },
|
||||
});
|
||||
assert.equal(overridden.antigravityTimingTelemetry.enabled,
|
||||
!defaults.antigravityTimingTelemetry.enabled);
|
||||
assert.deepEqual(overridden.requestQueue, defaults.requestQueue);
|
||||
const sparse = pruneResilienceSettingsToProvided(overridden, defaults);
|
||||
assert.equal(resolveResilienceSettings({ resilienceSettings: sparse })
|
||||
.antigravityTimingTelemetry.enabled, !defaults.antigravityTimingTelemetry.enabled);
|
||||
});
|
||||
|
||||
test("legacy fork Antigravity setting is honored until an explicit new override", () => {
|
||||
const legacy = { resilienceSettings: { antigravity: { timingTelemetryEnabled: true } } };
|
||||
assert.equal(resolveResilienceSettings(legacy).antigravityTimingTelemetry.enabled, true);
|
||||
const explicit = { resilienceSettings: {
|
||||
antigravity: { timingTelemetryEnabled: true },
|
||||
antigravityTimingTelemetry: { enabled: false },
|
||||
} };
|
||||
assert.equal(resolveResilienceSettings(explicit).antigravityTimingTelemetry.enabled, false);
|
||||
});
|
||||
|
||||
test("strict PATCH accepts boolean timing toggle, not strings or arbitrary keys", () => {
|
||||
assert.equal(updateResilienceSchema.safeParse({
|
||||
antigravityTimingTelemetry: { enabled: true },
|
||||
}).success, true);
|
||||
for (const invalid of [
|
||||
{ antigravityTimingTelemetry: { enabled: "true" } },
|
||||
{ antigravityTimingTelemetry: { enabled: 1 } },
|
||||
{ antigravityTimingTelemetry: { diagnosticKey: "raw" } },
|
||||
]) {
|
||||
assert.equal(updateResilienceSchema.safeParse(invalid).success, false);
|
||||
}
|
||||
});
|
||||
|
||||
test("timing feature default remains an opt-in boolean", () => {
|
||||
assert.equal(typeof DEFAULT_RESILIENCE_SETTINGS.antigravityTimingTelemetry.enabled, "boolean");
|
||||
});
|
||||
|
||||
|
||||
test("physical-send flag resolver respects runtime toggle, legacy fork and env fallback", () => {
|
||||
assert.equal(resolveAntigravityTimingTelemetryEnabled({}, true), true);
|
||||
assert.equal(resolveAntigravityTimingTelemetryEnabled({}, false), false);
|
||||
const old = { resilienceSettings: { antigravity: { timingTelemetryEnabled: true } } };
|
||||
assert.equal(resolveAntigravityTimingTelemetryEnabled(old, false), true);
|
||||
const explicit = { resilienceSettings: {
|
||||
antigravity: { timingTelemetryEnabled: true },
|
||||
antigravityTimingTelemetry: { enabled: false },
|
||||
} };
|
||||
assert.equal(resolveAntigravityTimingTelemetryEnabled(explicit, true), false);
|
||||
assert.equal(resolveAntigravityTimingTelemetryEnabled({
|
||||
resilienceSettings: { antigravityTimingTelemetry: { enabled: true } },
|
||||
}, false), true);
|
||||
});
|
||||
408
tests/unit/antigravity-timing.test.ts
Normal file
408
tests/unit/antigravity-timing.test.ts
Normal file
@@ -0,0 +1,408 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { ANTIGRAVITY_PRE_RESPONSE_TIMEOUT_CODE } from "../../open-sse/config/constants.ts";
|
||||
import {
|
||||
AntigravityTimingTracker,
|
||||
withAntigravityTiming,
|
||||
hasAntigravityMeaningfulDelta,
|
||||
} from "../../open-sse/services/antigravityTiming.ts";
|
||||
import {
|
||||
HttpAttemptCollector,
|
||||
selectHttpResponse,
|
||||
} from "../../open-sse/utils/providerHttpAttempts.ts";
|
||||
import { runWithCapture } from "../../open-sse/utils/providerRequestLogging.ts";
|
||||
|
||||
const text = 'data: {"response":{"candidates":[{"content":{"parts":[{"text":"fixture"}]}}]}}\n\n';
|
||||
const encoder = new TextEncoder();
|
||||
const requestId = "24a00000-0000-4000-a000-000000000024";
|
||||
|
||||
test("pings, roles, usage and errors are not meaningful deltas", () => {
|
||||
for (const frame of [
|
||||
": ping\n\n",
|
||||
'data: {"usageMetadata":{"totalTokenCount":1}}\n\n',
|
||||
'data: {"response":{"candidates":[{"content":{"role":"model","parts":[]}}]}}\n\n',
|
||||
'data: {"error":{"message":"failure"}}\n\n',
|
||||
"data: [DONE]\n\n",
|
||||
]) {
|
||||
assert.equal(hasAntigravityMeaningfulDelta(frame), false);
|
||||
}
|
||||
assert.equal(hasAntigravityMeaningfulDelta(text), true);
|
||||
assert.equal(
|
||||
hasAntigravityMeaningfulDelta(
|
||||
'data: {"candidates":[{"content":{"parts":[{"functionCall":{"name":"tool"}}]}}]}\n\n'
|
||||
),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("measures actual header and first delta phases with null unavailable network phases", async () => {
|
||||
let now = 100;
|
||||
const events: Record<string, unknown>[] = [];
|
||||
const tracker = new AntigravityTimingTracker(() => now);
|
||||
let upstream: ReadableStreamDefaultController<Uint8Array>;
|
||||
const response = await withAntigravityTiming(
|
||||
{
|
||||
enabled: true,
|
||||
connectionId: "fixture-account",
|
||||
model: "agy/gemini-3.8-flash",
|
||||
correlationId: requestId,
|
||||
emit: (event) => events.push(event),
|
||||
},
|
||||
() =>
|
||||
tracker.fetch(async () => {
|
||||
now = 125;
|
||||
return new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
upstream = controller;
|
||||
},
|
||||
})
|
||||
);
|
||||
})
|
||||
);
|
||||
assert.equal(events[0].headerLatencyMs, 25);
|
||||
assert.equal(events[0].idleDurationMs, null);
|
||||
assert.equal(events[0].dnsMs, null);
|
||||
assert.equal(events[0].connectMs, null);
|
||||
assert.equal(events[0].tlsMs, null);
|
||||
now = 140;
|
||||
upstream!.enqueue(encoder.encode(": ping\n\n"));
|
||||
const reading = response.text();
|
||||
now = 170;
|
||||
upstream!.enqueue(encoder.encode(text.slice(0, 20)));
|
||||
upstream!.enqueue(encoder.encode(text.slice(20)));
|
||||
upstream!.close();
|
||||
assert.equal(await reading, ": ping\n\n" + text);
|
||||
const first = events.find((event) => event.phase === "first_meaningful_delta")!;
|
||||
assert.equal(first.firstMeaningfulDeltaFromHeadersMs, 45);
|
||||
assert.equal(first.firstMeaningfulDeltaFromSendMs, 70);
|
||||
assert.equal(first.requestId, requestId);
|
||||
assert.equal(first.model, "gemini-3.8-flash-high");
|
||||
assert.equal(events.filter((event) => event.phase === "complete").length, 1);
|
||||
const serialized = JSON.stringify(events);
|
||||
for (const forbidden of [
|
||||
"fixture-account",
|
||||
'fixture"',
|
||||
"Bearer",
|
||||
"https://",
|
||||
"parts",
|
||||
"candidates",
|
||||
]) {
|
||||
assert.equal(serialized.includes(forbidden), false);
|
||||
}
|
||||
});
|
||||
|
||||
test("idle duration and inflight count are per account and end on actual lifecycle", async () => {
|
||||
let now = 10;
|
||||
const tracker = new AntigravityTimingTracker(() => now);
|
||||
const events: Record<string, unknown>[] = [];
|
||||
const send = (id = "a") =>
|
||||
withAntigravityTiming(
|
||||
{
|
||||
enabled: true,
|
||||
connectionId: id,
|
||||
model: "gemini-3.8-flash-high",
|
||||
emit: (event) => events.push(event),
|
||||
},
|
||||
() => tracker.fetch(async () => new Response(new ReadableStream()))
|
||||
);
|
||||
const first = await send();
|
||||
now = 20;
|
||||
const concurrent = await send();
|
||||
assert.equal(events[1].concurrentInflight, 2);
|
||||
assert.equal(events[1].idleDurationMs, 0);
|
||||
await first.body!.cancel();
|
||||
await concurrent.body!.cancel();
|
||||
now = 90;
|
||||
const afterIdle = await send();
|
||||
assert.equal(events.findLast((event) => event.phase === "headers")!.idleDurationMs, 70);
|
||||
const other = await send("b");
|
||||
assert.equal(events.findLast((event) => event.phase === "headers")!.idleDurationMs, null);
|
||||
await afterIdle.body!.cancel();
|
||||
await other.body!.cancel();
|
||||
});
|
||||
|
||||
test("feature off does not wrap or consume the response, and logs nothing", async () => {
|
||||
const tracker = new AntigravityTimingTracker();
|
||||
const original = new Response(text);
|
||||
const events: unknown[] = [];
|
||||
const response = await withAntigravityTiming(
|
||||
{
|
||||
enabled: false,
|
||||
connectionId: "a",
|
||||
model: "m",
|
||||
emit: (event) => events.push(event),
|
||||
},
|
||||
() => tracker.fetch(async () => original)
|
||||
);
|
||||
assert.equal(response, original);
|
||||
assert.equal(response.bodyUsed, false);
|
||||
assert.deepEqual(events, []);
|
||||
});
|
||||
|
||||
test("failed fetch never fabricates header or first delta latency", async () => {
|
||||
let now = 0;
|
||||
const tracker = new AntigravityTimingTracker(() => now);
|
||||
const events: Record<string, unknown>[] = [];
|
||||
await assert.rejects(
|
||||
withAntigravityTiming(
|
||||
{
|
||||
enabled: true,
|
||||
connectionId: "a",
|
||||
model: "m",
|
||||
correlationId: "Bearer secret",
|
||||
emit: (event) => events.push(event),
|
||||
},
|
||||
() =>
|
||||
tracker.fetch(async () => {
|
||||
now = 500;
|
||||
throw new Error("secret upstream URL");
|
||||
})
|
||||
)
|
||||
);
|
||||
assert.equal(events[0].headerLatencyMs, null);
|
||||
assert.equal(events[0].firstMeaningfulDeltaFromSendMs, null);
|
||||
assert.equal(events[0].elapsedMs, 500);
|
||||
assert.equal(JSON.stringify(events).includes("secret"), false);
|
||||
});
|
||||
|
||||
test("unread upstream errors close timing immediately and keep original bodies untouched", async () => {
|
||||
let now = 100;
|
||||
const tracker = new AntigravityTimingTracker(() => now);
|
||||
const events: Record<string, unknown>[] = [];
|
||||
const run = (body: Response) =>
|
||||
withAntigravityTiming(
|
||||
{
|
||||
enabled: true,
|
||||
connectionId: "retry-account",
|
||||
model: "gemini-3.8-flash-tiered",
|
||||
emit: (event) => events.push(event),
|
||||
},
|
||||
() =>
|
||||
tracker.fetch(async () => {
|
||||
now += 10;
|
||||
return body;
|
||||
})
|
||||
);
|
||||
|
||||
for (const status of [403, 429, 503]) {
|
||||
const original = new Response(new ReadableStream<Uint8Array>(), { status });
|
||||
const returned = await run(original);
|
||||
assert.equal(returned, original, "error bodies must remain untouched for retries");
|
||||
assert.equal(returned.bodyUsed, false);
|
||||
assert.equal(events.findLast((event) => event.phase === "headers")!.concurrentInflight, 1);
|
||||
assert.equal(events.findLast((event) => event.phase === "http_error")!.headerLatencyMs, 10);
|
||||
await returned.body!.cancel();
|
||||
}
|
||||
assert.equal(events.filter((event) => event.phase === "http_error").length, 3);
|
||||
assert.equal(events.filter((event) => event.phase === "complete").length, 0);
|
||||
assert.equal(events.findLast((event) => event.phase === "headers")!.idleDurationMs, 0);
|
||||
});
|
||||
|
||||
test("unread successful stream expires telemetry without cancelling upstream transport", async () => {
|
||||
const tracker = new AntigravityTimingTracker(() => performance.now(), 15);
|
||||
const events: Record<string, unknown>[] = [];
|
||||
const run = (response: Response) =>
|
||||
withAntigravityTiming(
|
||||
{
|
||||
enabled: true,
|
||||
connectionId: "unread-ok",
|
||||
model: "gemini-3.8-flash-tiered",
|
||||
emit: (event) => events.push(event),
|
||||
},
|
||||
() => tracker.fetch(async () => response)
|
||||
);
|
||||
let upstreamCancelled = false;
|
||||
const unread = await run(
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
cancel() {
|
||||
upstreamCancelled = true;
|
||||
},
|
||||
})
|
||||
)
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 35));
|
||||
assert.equal(events.filter((event) => event.phase === "observation_expired").length, 1);
|
||||
assert.equal(
|
||||
upstreamCancelled,
|
||||
false,
|
||||
"timing watchdog must never affect HTTP response behavior"
|
||||
);
|
||||
assert.equal(unread.bodyUsed, false);
|
||||
const following = await run(new Response(null, { status: 204 }));
|
||||
assert.equal(following.status, 204);
|
||||
assert.equal(events.filter((event) => event.phase === "headers").at(-1)!.concurrentInflight, 1);
|
||||
await unread.body!.cancel();
|
||||
assert.equal(upstreamCancelled, true);
|
||||
assert.equal(events.filter((event) => event.phase === "observation_expired").length, 1);
|
||||
});
|
||||
|
||||
test("request abort and failed fetch release per-account timing slots exactly once", async () => {
|
||||
const tracker = new AntigravityTimingTracker();
|
||||
const events: Record<string, unknown>[] = [];
|
||||
const abortController = new AbortController();
|
||||
const run = (send: () => Promise<Response>, signal?: AbortSignal) =>
|
||||
withAntigravityTiming(
|
||||
{
|
||||
enabled: true,
|
||||
connectionId: "abort-account",
|
||||
model: "gemini-3.8-flash-tiered",
|
||||
emit: (event) => events.push(event),
|
||||
},
|
||||
() => tracker.fetch(send, signal)
|
||||
);
|
||||
const held = await run(async () => new Response(new ReadableStream()), abortController.signal);
|
||||
abortController.abort(new DOMException("cancelled", "AbortError"));
|
||||
assert.equal(events.filter((event) => event.phase === "caller_abort").length, 1);
|
||||
await held.body!.cancel();
|
||||
await assert.rejects(
|
||||
run(async () => {
|
||||
throw new Error("upstream failed");
|
||||
})
|
||||
);
|
||||
assert.equal(events.filter((event) => event.phase === "fetch_failed").length, 1);
|
||||
await run(async () => new Response(null, { status: 204 }));
|
||||
assert.equal(events.filter((event) => event.phase === "headers").at(-1)!.concurrentInflight, 1);
|
||||
});
|
||||
|
||||
test("a pre-response abort reports its own phase before the fetch rejection", async () => {
|
||||
const controller = new AbortController();
|
||||
const tracker = new AntigravityTimingTracker();
|
||||
const events: Record<string, unknown>[] = [];
|
||||
const timeout = Object.assign(new Error("request header timeout"), {
|
||||
name: "TimeoutError",
|
||||
code: ANTIGRAVITY_PRE_RESPONSE_TIMEOUT_CODE,
|
||||
});
|
||||
const running = withAntigravityTiming(
|
||||
{
|
||||
enabled: true,
|
||||
connectionId: "timeout-account",
|
||||
model: "gemini-3.8-flash",
|
||||
emit: (event) => events.push(event),
|
||||
},
|
||||
() =>
|
||||
tracker.fetch(
|
||||
() =>
|
||||
new Promise<Response>((_resolve, reject) => {
|
||||
controller.signal.addEventListener("abort", () => reject(timeout), { once: true });
|
||||
}),
|
||||
controller.signal
|
||||
)
|
||||
);
|
||||
controller.abort(timeout);
|
||||
await assert.rejects(running);
|
||||
assert.equal(events.filter((event) => event.phase === "pre_response_timeout").length, 1);
|
||||
assert.equal(events.filter((event) => event.phase === "upstream_deadline").length, 0);
|
||||
});
|
||||
|
||||
test("timing events retain the request id and physical retry ordinal", async () => {
|
||||
const events: Record<string, unknown>[] = [];
|
||||
const tracker = new AntigravityTimingTracker();
|
||||
await withAntigravityTiming(
|
||||
{
|
||||
enabled: true,
|
||||
connectionId: "retry-account",
|
||||
model: "gemini-3.8-flash",
|
||||
correlationId: requestId,
|
||||
physicalSendOrdinal: 4,
|
||||
emit: (event) => events.push(event),
|
||||
},
|
||||
() => tracker.fetch(async () => new Response(null, { status: 204 }))
|
||||
);
|
||||
assert.equal(events[0].requestId, requestId);
|
||||
assert.equal(events[0].physicalSend, 4);
|
||||
});
|
||||
|
||||
test("Antigravity physical fetch opt-in wraps actual transport; default stays unchanged", async () => {
|
||||
const { fetchAntigravityWithReadinessTimeout } =
|
||||
await import("../../open-sse/executors/antigravity/executeAttempt.ts");
|
||||
const previousFetch = globalThis.fetch;
|
||||
const previousFlag = process.env.OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY;
|
||||
const response = new Response(text);
|
||||
let sends = 0;
|
||||
try {
|
||||
globalThis.fetch = (async () => {
|
||||
sends++;
|
||||
return response;
|
||||
}) as typeof fetch;
|
||||
delete process.env.OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY;
|
||||
const meta = {
|
||||
provider: "antigravity",
|
||||
model: "gemini-3.8-flash-high",
|
||||
connectionId: "timing-connection",
|
||||
endpoint: "antigravity_generate",
|
||||
};
|
||||
const original = await fetchAntigravityWithReadinessTimeout(
|
||||
"https://example.invalid/generate",
|
||||
{ method: "POST" },
|
||||
0,
|
||||
meta
|
||||
);
|
||||
assert.equal(original, response, "disabled telemetry must preserve original Response identity");
|
||||
process.env.OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY = "true";
|
||||
const timed = await fetchAntigravityWithReadinessTimeout(
|
||||
"https://example.invalid/generate",
|
||||
{ method: "POST" },
|
||||
0,
|
||||
meta
|
||||
);
|
||||
assert.notEqual(
|
||||
timed,
|
||||
response,
|
||||
"enabled telemetry must observe the actual transport response"
|
||||
);
|
||||
assert.equal(await timed.text(), text);
|
||||
assert.equal(sends, 2, "telemetry never resends the upstream request");
|
||||
} finally {
|
||||
globalThis.fetch = previousFetch;
|
||||
if (previousFlag === undefined) delete process.env.OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY;
|
||||
else process.env.OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY = previousFlag;
|
||||
}
|
||||
});
|
||||
|
||||
test("opt-in timing on the bounded path keeps the attempt on the wrapped Response", async () => {
|
||||
const { fetchAntigravityWithReadinessTimeout } =
|
||||
await import("../../open-sse/executors/antigravity/executeAttempt.ts");
|
||||
const previousFetch = globalThis.fetch;
|
||||
const previousFlag = process.env.OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY;
|
||||
const response = new Response(text);
|
||||
const httpAttempts = new HttpAttemptCollector();
|
||||
const capture = { httpAttempts, capture() {}, body: (fallback: unknown) => fallback };
|
||||
const caller = new AbortController();
|
||||
let sentSignal: AbortSignal | null | undefined;
|
||||
try {
|
||||
globalThis.fetch = (async (_url: string, init?: RequestInit) => {
|
||||
sentSignal = init?.signal;
|
||||
return response;
|
||||
}) as typeof fetch;
|
||||
process.env.OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY = "true";
|
||||
const timed = await runWithCapture(capture, () =>
|
||||
fetchAntigravityWithReadinessTimeout(
|
||||
"https://example.invalid/generate",
|
||||
{ method: "POST", signal: caller.signal },
|
||||
60_000,
|
||||
{
|
||||
provider: "antigravity",
|
||||
model: "gemini-3.8-flash-high",
|
||||
connectionId: "bounded-connection",
|
||||
endpoint: "antigravity_generate",
|
||||
},
|
||||
{ correlationId: requestId, ordinal: 1 }
|
||||
)
|
||||
);
|
||||
assert.notEqual(timed, response, "enabled telemetry must wrap the bounded transport response");
|
||||
assert.ok(sentSignal, "the bounded path must keep sending its merged abort signal");
|
||||
assert.notEqual(sentSignal, caller.signal);
|
||||
const attempt = selectHttpResponse(timed)?.current;
|
||||
assert.equal(attempt?.record.outcome, "response");
|
||||
assert.equal(attempt?.record.status, 200);
|
||||
assert.equal(attempt?.record.connectionId, "bounded-connection");
|
||||
assert.equal(selectHttpResponse(response), null, "the unwrapped Response must not hold it");
|
||||
assert.equal(await timed.text(), text);
|
||||
} finally {
|
||||
globalThis.fetch = previousFetch;
|
||||
if (previousFlag === undefined) delete process.env.OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY;
|
||||
else process.env.OMNIROUTE_ANTIGRAVITY_TIMING_TELEMETRY = previousFlag;
|
||||
}
|
||||
});
|
||||
@@ -99,6 +99,7 @@ describe("resilience/settings normalize split-guard", () => {
|
||||
it("host exposes DEFAULT_RESILIENCE_SETTINGS with the full section set", () => {
|
||||
const keys = Object.keys(DEFAULT_RESILIENCE_SETTINGS).sort();
|
||||
assert.deepEqual(keys, [
|
||||
"antigravityTimingTelemetry",
|
||||
"comboCooldownWait",
|
||||
"connectionCooldown",
|
||||
// Global default cadence for the background credential health check sweep.
|
||||
|
||||
Reference in New Issue
Block a user