Files
OmniRoute/tests/unit/cli-usage.test.ts
Gaurav Dubey 0f4b560583 fix(api): report an unknown provider quota as null, not 100% remaining (#18847)
## Summary

- `GET /api/usage/quota` starts every entry at `percentRemaining = 100` (`buildQuotaEntry` in `src/app/api/usage/quota/route.ts`), and `sanitizeQuotaProvider()` in `src/shared/contracts/quota.ts` also falls back to `100` whenever there is no total. So a connection with no quota signal (no learned rate-limit headers, no active `rateLimitedUntil`, no queue pressure) shows as fully available. The #18745 report shows this: every one of 15 connections came back `quotaTotal: null, percentRemaining: 100`.
- An unknown quota is now `percentRemaining: null`. Known totals, explicit percentages, the cooldown branch (`0`), and the queue-pressure signal are unchanged.
- `QuotaProviderEntry.percentRemaining` is now `number | null`. These in-repo consumers needed changes so the null doesn't turn into a different wrong answer:
  - MCP `omniroute_check_quota`: `checkQuotaOutput.percentRemaining` is now `.nullable()`.
  - A2A quota-management skill: `null <= 10` is `true` in JS, so without a guard every unknown provider would be listed as "at or below 10% remaining". The summary now skips unknown quotas, and the ranking label shows `n/a`.
  - `omniroute usage quota`: `null > 0` is `false`, so the state would have read `exhausted`. It now reads `unknown`.
- `tests/e2e/ecosystem.test.ts` accepts `null` for `percentRemaining` and keeps the 0..100 range check for numeric values.

## Related Issues

- Related to #18745. This covers only the "unknown quota shown as 100% remaining" contra

Thanks @gaurav0107.
2026-11-08 20:04:49 -03:00

248 lines
8.2 KiB
TypeScript

import test from "node:test";
import assert from "node:assert/strict";
const ANALYTICS_DATA = {
byProvider: [
{
provider: "openai",
totalRequests: 100,
totalTokensIn: 40000,
totalTokensOut: 16000,
totalCost: 0.35,
},
{
provider: "anthropic",
totalRequests: 50,
totalTokensIn: 20000,
totalTokensOut: 8000,
totalCost: 0.15,
},
],
};
const BUDGET_DATA = {
budgets: [
{ scope: "global", period: "monthly", limit: 100, used: 42.5, remaining: 57.5, pct: 0.425 },
],
};
const QUOTA_DATA = {
providers: [
{ provider: "openai", limit: 1000000, used: 500000, remaining: 500000, state: "available" },
],
};
const LOGS_DATA = {
logs: [
{
id: "1",
createdAt: "2026-05-15T10:00:00Z",
apiKey: "sk-test-key",
provider: "openai",
model: "gpt-4o",
tokensIn: 100,
tokensOut: 50,
cost: 0.001,
latencyMs: 500,
status: 200,
},
{
id: "2",
createdAt: "2026-05-15T10:01:00Z",
apiKey: "sk-test-key",
provider: "anthropic",
model: "claude-3-5-sonnet",
tokensIn: 80,
tokensOut: 40,
cost: 0.0008,
latencyMs: 400,
status: 200,
},
],
};
const UTILIZATION_DATA = [{ apiKey: "sk-test-key", requests: 150, cost: 0.5, avgLatency: 450 }];
const HISTORY_DATA = { items: [{ id: "a", model: "gpt-4o", provider: "openai", cost: 0.01 }] };
const PROXY_LOGS_DATA = {
logs: [{ id: "p1", path: "/v1/chat/completions", method: "POST", status: 200 }],
};
function makeResp(data: unknown, status = 200) {
const obj = {
ok: status < 400,
status,
exitCode: status < 400 ? 0 : 1,
json: () => Promise.resolve(data),
text: () => Promise.resolve(JSON.stringify(data)),
headers: new Headers(),
};
obj.json = obj.json.bind(obj);
obj.text = obj.text.bind(obj);
return obj;
}
function mockFetch(overrides: Record<string, unknown> = {}) {
return (url: string) => {
if (url.includes("/api/usage/analytics"))
return Promise.resolve(makeResp(overrides.analytics ?? ANALYTICS_DATA));
if (url.includes("/api/usage/budget"))
return Promise.resolve(makeResp(overrides.budget ?? BUDGET_DATA));
if (url.includes("/api/usage/quota"))
return Promise.resolve(makeResp(overrides.quota ?? QUOTA_DATA));
if (url.includes("/api/usage/call-logs"))
return Promise.resolve(makeResp(overrides.logs ?? LOGS_DATA));
if (url.includes("/api/usage/utilization"))
return Promise.resolve(makeResp(overrides.utilization ?? UTILIZATION_DATA));
if (url.includes("/api/usage/history"))
return Promise.resolve(makeResp(overrides.history ?? HISTORY_DATA));
if (url.includes("/api/usage/proxy-logs"))
return Promise.resolve(makeResp(overrides.proxyLogs ?? PROXY_LOGS_DATA));
return Promise.resolve(makeResp({}, 404));
};
}
async function captureStdout(fn: () => Promise<void>): Promise<string> {
const chunks: string[] = [];
const orig = process.stdout.write.bind(process.stdout);
process.stdout.write = (c: string | Uint8Array) => {
if (typeof c === "string") chunks.push(c);
return true;
};
try {
await fn();
} finally {
process.stdout.write = orig;
}
return chunks.join("");
}
test("runUsageAnalytics exibe providers em json", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = mockFetch() as any;
const { runUsageAnalytics } = await import("../../bin/cli/commands/usage.mjs");
const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) };
const out = await captureStdout(() => runUsageAnalytics({ period: "30d" }, cmd as any));
globalThis.fetch = origFetch;
const parsed = JSON.parse(out);
assert.ok(Array.isArray(parsed));
assert.equal(parsed[0].provider, "openai");
assert.ok(parsed[0].costUsd > 0);
});
test("runBudgetList exibe budgets", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = mockFetch() as any;
const { runBudgetList } = await import("../../bin/cli/commands/usage.mjs");
const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) };
const out = await captureStdout(() => runBudgetList({}, cmd as any));
globalThis.fetch = origFetch;
const parsed = JSON.parse(out);
assert.ok(Array.isArray(parsed));
assert.equal(parsed[0].scope, "global");
assert.ok(parsed[0].limit > 0);
});
test("runUsageQuota exibe providers de quota", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = mockFetch() as any;
const { runUsageQuota } = await import("../../bin/cli/commands/usage.mjs");
const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) };
const out = await captureStdout(() => runUsageQuota({}, cmd as any));
globalThis.fetch = origFetch;
const parsed = JSON.parse(out);
assert.ok(Array.isArray(parsed));
assert.equal(parsed[0].provider, "openai");
});
test("runUsageQuota reports an unknown percentRemaining as unknown, not exhausted (#18745)", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = mockFetch({
quota: {
providers: [
{ provider: "unknown-quota", quotaTotal: null, percentRemaining: null },
{ provider: "empty-quota", quotaTotal: 100, percentRemaining: 0 },
{ provider: "open-quota", quotaTotal: 100, percentRemaining: 40 },
],
},
}) as unknown as typeof fetch;
const { runUsageQuota } = await import("../../bin/cli/commands/usage.mjs");
const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) };
const out = await captureStdout(() => runUsageQuota({}, cmd));
globalThis.fetch = origFetch;
const parsed = JSON.parse(out);
assert.deepEqual(
parsed.map((r: { provider: string; state: string }) => [r.provider, r.state]),
[
["unknown-quota", "unknown"],
["empty-quota", "exhausted"],
["open-quota", "available"],
]
);
});
test("runUsageLogs exibe logs com mascaramento de API key", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = mockFetch() as any;
const { runUsageLogs } = await import("../../bin/cli/commands/usage.mjs");
const cmd = { optsWithGlobals: () => ({ output: "table", quiet: false }) };
const out = await captureStdout(() => runUsageLogs({ limit: 10 }, cmd as any));
globalThis.fetch = origFetch;
assert.ok(!out.includes("sk-test-key") || out.includes("***"));
});
test("runUsageLogs --output json retorna rows com campos esperados", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = mockFetch() as any;
const { runUsageLogs } = await import("../../bin/cli/commands/usage.mjs");
const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) };
const out = await captureStdout(() => runUsageLogs({ limit: 10 }, cmd as any));
globalThis.fetch = origFetch;
const parsed = JSON.parse(out);
assert.ok(Array.isArray(parsed));
assert.equal(parsed.length, 2);
assert.ok(typeof parsed[0].provider === "string");
assert.ok(typeof parsed[0].tokens === "number");
});
test("runUsageHistory exibe histórico", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = mockFetch() as any;
const { runUsageHistory } = await import("../../bin/cli/commands/usage.mjs");
const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) };
const out = await captureStdout(() => runUsageHistory({ limit: 50 }, cmd as any));
globalThis.fetch = origFetch;
const parsed = JSON.parse(out);
assert.ok(Array.isArray(parsed));
assert.ok(parsed.length >= 1);
});
test("runBudgetSet envia POST com amount, scope e period", async () => {
let capturedBody: unknown = null;
const origFetch = globalThis.fetch;
globalThis.fetch = ((url: string, init: any) => {
if (url.includes("/api/usage/budget") && init?.method === "POST") {
capturedBody = JSON.parse(init.body);
}
return Promise.resolve(makeResp({ ok: true }));
}) as any;
const { runBudgetSet } = await import("../../bin/cli/commands/usage.mjs");
const cmd = { optsWithGlobals: () => ({ output: "table", quiet: false }) };
await captureStdout(() => runBudgetSet("50", { scope: "global", period: "monthly" }, cmd as any));
globalThis.fetch = origFetch;
assert.ok(capturedBody !== null);
assert.equal((capturedBody as any).amount, 50);
assert.equal((capturedBody as any).scope, "global");
});