Merge remote-tracking branch 'origin/release/v3.8.54' into fix/truthful-streaming-usage-20054

This commit is contained in:
diegosouzapw
2026-11-15 10:46:16 -03:00
19 changed files with 812 additions and 37 deletions

View File

@@ -0,0 +1 @@
- **fix(mcp):** `npm run test:vitest` no longer writes MCP audit rows into the developer's `~/.omniroute/storage.sqlite`: both Vitest configs run with a throwaway `DATA_DIR`, and the MCP audit refuses the home database in a test context without `DATA_DIR` (opt back in with `OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1`) ([#20252](https://github.com/cheaper-inference/OmniRoute/pull/20252))

View File

@@ -54,16 +54,24 @@ traffic is never affected by it.
`400 — "Chaos Mode is not enabled. Enable it in Dashboard → Chaos Mode."`.
3. Pick the participants and defaults (persisted per instance via the settings store):
| Field | Meaning | Default / limits |
| ------------------- | ------------------------------------------------------------------- | --------------------------------------- |
| `enabled` | Master switch | `false` |
| `defaultMode` | `parallel` or `collaborative` (see below) | `parallel` |
| Field | Meaning | Default / limits |
| ------------------- | ------------------------------------------------------------------------ | -------------------------------------- |
| `enabled` | Master switch | `false` |
| `defaultMode` | `parallel` or `collaborative` (see below) | `parallel` |
| `providerOverrides` | Per-provider participation (`providerId`, optional `modelId`, `enabled`) | empty = every active provider, max 200 |
| `systemPrompt` | Override for the built-in Chaos system prompt | optional, max 10 000 chars |
| `timeoutMs` | Max time per model call | `120000` (5 000–600 000) |
| `maxTokens` | `max_tokens` per model call | `4096` (256–128 000) |
| `systemPrompt` | Override for the built-in Chaos system prompt | optional, max 10 000 chars |
| `timeoutMs` | Max time per model call | `120000` (5 000–600 000) |
| `maxTokens` | `max_tokens` per model call | `4096` (256–128 000) |
4. Run a **test from the page itself** — the results panel shows each provider's answer,
4. Give every participating provider a **model**. A provider's model is the `modelId` of its
enabled override on the Chaos Mode page (it wins), otherwise the **default model** of its
connection. A provider id is never used as a model name. A provider with neither is **not
called**: it shows up in the results as `status: "error"` with the message _"No model set for
this provider. Set a model for it on the Chaos Mode page or a default model on the
connection."_ and an empty `modelId`, while the other providers run normally. If no selected
provider has a model, the run fails before any upstream request with _"No provider has a
model set. …"_ (HTTP 500 on both endpoints).
5. Run a **test from the page itself** — the results panel shows each provider's answer,
status and duration.
## Execution modes
@@ -84,11 +92,11 @@ Cookie-authenticated (the management session — see
```jsonc
// body
{
"task": "Compare approaches to X", // required
"providers": ["glm", "kimi"], // optional filter
"mode": "parallel", // optional — overrides defaultMode
"systemPrompt": "…", // optional override
"maxTokens": 4096 // optional override
"task": "Compare approaches to X", // required
"providers": ["glm", "kimi"], // optional filter
"mode": "parallel", // optional — overrides defaultMode
"systemPrompt": "…", // optional override
"maxTokens": 4096, // optional override
}
```
@@ -115,10 +123,16 @@ Both endpoints return the same shape:
"totalProviders": 3,
"totalResults": 3,
"models": [
{ "providerId": "glm", "providerName": "GLM", "modelId": "glm-4.7",
"status": "success", "content": "…", "durationMs": 3210 }
{
"providerId": "glm",
"providerName": "GLM",
"modelId": "glm-4.7",
"status": "success",
"content": "…",
"durationMs": 3210,
},
],
"summary": "…" // collaborative mode only
"summary": "…", // collaborative mode only
}
```
@@ -127,6 +141,8 @@ Both endpoints return the same shape:
- **`400 Chaos Mode is not enabled`** — step 2 above: the global switch is off.
- **API key gets rejected on `/api/skills/collect/chaos`** — the key lacks the per-key
`chaosModeEnabled` permission (off by default; this is a setting, not an error).
- **A provider shows `No model set for this provider`** — set its model on the Chaos Mode page
or a default model on its connection (see Setup, step 4).
- **A provider you expected is missing from the results** — check `providerOverrides` on the
Chaos Mode page (a disabled override excludes it) and whether the provider connection is
active.

View File

@@ -756,7 +756,10 @@ These keys go in a connection's `providerSpecificData` (for example via
- `toolNameMaxLength` (integer, 1–1024) — for OpenAI chat-format compatible connections whose
gateway rejects long tool names. Longer names are sent as short aliases and restored in the
response.
response. OpenAI, Azure OpenAI, Azure AI Foundry, Mistral, NVIDIA, OpenCode Zen/Go and Muse
Code already alias names over 64 characters with no setting; a connection value lowers the
limit only where the provider has no built-in one (an Azure connection behind a stricter
gateway, or a custom compatible node).
- `quotaEndpoint` — the gateway's quota/credits endpoint, read for the dashboard quota strip and
quota-aware routing. Works on both OpenAI- and Anthropic-compatible connections; the full shape
is documented in `open-sse/services/usage/openaiCompatible.ts`.

View File

@@ -10,6 +10,8 @@ export const openaiProvider: RegistryEntry = {
reasoningTransport: "opaque",
authType: "apikey",
authHeader: "bearer",
// OpenAI Chat Completions rejects function names over 64 chars (MCP tool names hit this).
toolNameMaxLength: 64,
defaultContextLength: 128000,
models: [
// https://developers.openai.com/api/docs/models/gpt-6.1-sol

View File

@@ -67,10 +67,23 @@ export function applyRegistryToolNameLengthAliases(
return applyToolNameLengthAliases(body, getRegistryEntry(provider)?.toolNameMaxLength, normalize);
}
/**
* Limits for providers that have no registry entry to declare `toolNameMaxLength` on:
* Azure OpenAI and Azure AI Foundry are keyed connections handled by their own executors
* (`azure-openai`, `azure-ai`), and both speak the OpenAI Chat Completions schema, which
* rejects a function name over 64 chars.
*/
const BUILT_IN_TOOL_NAME_MAX_LENGTH: Readonly<Record<string, number>> = {
"azure-openai": 64,
"azure-ai": 64,
};
/**
* The registry `toolNameMaxLength`, else the connection's
* `providerSpecificData.toolNameMaxLength`: custom OpenAI chat-format compatible nodes never
* have a registry entry, so this is how they opt in for a stricter upstream.
* `providerSpecificData.toolNameMaxLength`, else a built-in limit for registry-less
* providers. Custom OpenAI chat-format compatible nodes never have a registry entry, so the
* connection value is how they opt in for a stricter upstream; it also lowers the built-in
* limit of an Azure connection sitting behind a stricter gateway.
*/
export function resolveToolNameMaxLength(
provider: string,
@@ -79,7 +92,11 @@ export function resolveToolNameMaxLength(
const connectionLimit = isRecord(providerSpecificData)
? providerSpecificData.toolNameMaxLength
: undefined;
return getRegistryEntry(provider)?.toolNameMaxLength ?? (connectionLimit as number | undefined);
return (
getRegistryEntry(provider)?.toolNameMaxLength ??
(connectionLimit as number | undefined) ??
BUILT_IN_TOOL_NAME_MAX_LENGTH[provider]
);
}
/**

View File

@@ -0,0 +1,101 @@
// The MCP audit resolves its own database path: DATA_DIR, else ~/.omniroute/storage.sqlite. It
// writes there whenever that file exists. A Vitest run without DATA_DIR therefore appended
// `omniroute_test_combo` rows to the developer's real database on every run (202 of them had
// piled up in one install). Two guards close that:
// - tests/_setup/vitestDataDir.ts gives every Vitest run a throwaway DATA_DIR;
// - in a test context with no DATA_DIR, the audit refuses the home database (the same rule
// src/lib/dataPaths.ts applies to the main database, #10428), unless
// OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1 opts back in.
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
it("runs Vitest with a DATA_DIR outside the developer's ~/.omniroute", () => {
const dataDir = process.env.DATA_DIR;
expect(dataDir, "tests/_setup/vitestDataDir.ts must set DATA_DIR").toBeTruthy();
expect(path.resolve(dataDir as string)).not.toBe(path.resolve(os.homedir(), ".omniroute"));
});
describe("MCP audit database path in a test context", () => {
const saved = {
DATA_DIR: process.env.DATA_DIR,
OMNIROUTE_ALLOW_DEFAULT_DATA_DIR: process.env.OMNIROUTE_ALLOW_DEFAULT_DATA_DIR,
};
let fakeHome: string;
let homeDb: string;
beforeEach(() => {
vi.resetModules();
globalThis.__omnirouteMcpAuditDb = undefined;
fakeHome = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-audit-home-"));
fs.mkdirSync(path.join(fakeHome, ".omniroute"));
homeDb = path.join(fakeHome, ".omniroute", "storage.sqlite");
fs.writeFileSync(homeDb, "");
// Vitest runs files in worker threads, where os.homedir() reads the real process environment
// and ignores a changed process.env.HOME, so the home directory is mocked instead.
vi.doMock("node:os", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:os")>();
const homedir = () => fakeHome;
return { ...actual, homedir, default: { ...actual, homedir } };
});
delete process.env.DATA_DIR;
delete process.env.OMNIROUTE_ALLOW_DEFAULT_DATA_DIR;
});
afterEach(() => {
for (const [key, value] of Object.entries(saved)) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
globalThis.__omnirouteMcpAuditDb = undefined;
vi.doUnmock("node:os");
fs.rmSync(fakeHome, { recursive: true, force: true });
vi.restoreAllMocks();
});
async function auditWithRecordingDriver() {
const opened: string[] = [];
class RecordingDatabase {
open = true;
constructor(file: string) {
opened.push(file);
}
prepare() {
return { run: () => ({ changes: 1 }), get: () => undefined, all: () => [] };
}
pragma() {}
close() {
this.open = false;
}
}
const audit = await import("../audit.ts");
audit.__setBetterSqliteLoaderForTests(() => RecordingDatabase);
audit.__setAuditCallerIdResolverForTests(async () => undefined);
return { audit, opened };
}
it("never opens the home database in a test context without DATA_DIR, even when it exists", async () => {
const { audit, opened } = await auditWithRecordingDriver();
try {
await audit.logToolCall("omniroute_test_combo", { combo: "x" }, { ok: true }, 3, true);
expect(opened).toEqual([]);
} finally {
audit.__setBetterSqliteLoaderForTests(null);
audit.__setAuditCallerIdResolverForTests(null);
}
}, 30000);
it("uses the home database only with OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1, like the main database", async () => {
process.env.OMNIROUTE_ALLOW_DEFAULT_DATA_DIR = "1";
const { audit, opened } = await auditWithRecordingDriver();
try {
await audit.logToolCall("omniroute_test_combo", { combo: "x" }, { ok: true }, 3, true);
expect(opened).toEqual([homeDb]);
} finally {
audit.__setBetterSqliteLoaderForTests(null);
audit.__setAuditCallerIdResolverForTests(null);
audit.closeAuditDb();
}
}, 30000);
});

View File

@@ -9,6 +9,7 @@
import { hashInput, summarizeOutput } from "./schemas/audit.ts";
import { runtimeRequire } from "../../src/lib/db/adapters/runtimeRequire.ts";
import { isNativeSqliteLoadError } from "../../src/lib/db/core.ts";
import { isTestContext } from "../../src/lib/dataPaths.ts";
import { resolveMcpCallerApiKeyId } from "./mcpCallerIdentity.ts";
// ============ Database Connection ============
@@ -308,6 +309,20 @@ async function getDb(): Promise<AuditDatabase | null> {
const { join } = await import("node:path");
const { existsSync } = await import("node:fs");
// A test run that never chose a DATA_DIR must not write to the developer's real database:
// the same rule src/lib/dataPaths.ts applies to the main database (#10428), with the same
// opt-in. Not cached, so a DATA_DIR set later in the process still enables auditing.
if (
!process.env.DATA_DIR &&
isTestContext() &&
process.env.OMNIROUTE_ALLOW_DEFAULT_DATA_DIR !== "1"
) {
console.error(
"[MCP Audit] No DATA_DIR in a test context — audit logging disabled instead of using ~/.omniroute"
);
return null;
}
const dbPath = process.env.DATA_DIR
? join(process.env.DATA_DIR, "storage.sqlite")
: join(homedir(), ".omniroute", "storage.sqlite");

View File

@@ -52,6 +52,8 @@ export function classifyFile(file, { claimFiles, instructionFiles }) {
/**
* Each managed value the head changed goes back to the base value of the same claim (same check,
* same position). A check whose claim count differs between base and head is left to a human.
* The replacement is the base's own text for that claim, not the parsed value: a mirror that
* writes "۲۳۳" must get "۲۳۳" back, or the line still differs from the base and keeps conflicting.
*/
export function restoreClaims(base, head, checks) {
const edits = [];
@@ -72,6 +74,7 @@ export function restoreClaims(base, head, checks) {
label: check.label,
from: hv[i],
to: bv[i],
text: b[i].start >= 0 ? (base ?? "").slice(b[i].start, b[i].end) : bv[i],
start: claim.start,
end: claim.end,
});
@@ -84,7 +87,7 @@ export function restoreClaims(base, head, checks) {
const key = `${e.start}:${e.end}`;
if (seen.has(key)) continue;
seen.add(key);
content = content.slice(0, e.start) + e.to + content.slice(e.end);
content = content.slice(0, e.start) + e.text + content.slice(e.end);
}
for (const check of checks) {
if (!edits.some((e) => e.label === check.label)) continue;
@@ -279,6 +282,24 @@ function tryGit(args, opts) {
}
}
/**
* A file's exact bytes at `ref` (null when the path does not exist there). Never trimmed: the
* restore must reproduce the merge-base blob byte for byte — a trimmed read drops the final
* newline, leaves the file "modified" and the PR still conflicting.
*/
export function readBlob(ref, file, { cwd = ROOT } = {}) {
try {
return execFileSync("git", ["show", `${ref}:${file}`], {
cwd,
encoding: "utf8",
stdio: ["ignore", "pipe", "ignore"],
maxBuffer: 256 << 20,
});
} catch {
return null;
}
}
export const RESTORE_MESSAGE = [
"chore: drop derived values the release refresh now owns",
"",
@@ -363,8 +384,8 @@ async function main() {
const changed = git(["diff", "--name-only", mb, ref]).split("\n").filter(Boolean);
const plan = planPrRestore({
changed,
readBase: (f) => tryGit(["show", `${mb}:${f}`]),
readHead: (f) => tryGit(["show", `${ref}:${f}`]),
readBase: (f) => readBlob(mb, f),
readHead: (f) => readBlob(ref, f),
ctx,
});
const entry = {

View File

@@ -117,21 +117,41 @@ function resolveEffectiveRunParams(
}
/**
* Resolve the best model ID for a provider connection.
* Applies provider overrides from global chaos config if present.
* Resolve the model ID for a provider connection: an enabled override's `modelId`
* wins, then the connection's `defaultModel`. Returns `null` when neither is set —
* a provider id is not a model name, so it is never used as a fallback.
*/
function resolveModelId(
conn: { provider?: string; id?: string; defaultModel?: string | null },
overrides: ChaosConfig["providerOverrides"]
): string {
): string | null {
const override = overrides.find(
(o) =>
o.enabled &&
(o.providerId.toLowerCase() === (conn.provider || "").toLowerCase() ||
o.providerId.toLowerCase() === (conn.id || "").toLowerCase())
);
if (override?.modelId) return override.modelId;
return conn.defaultModel || conn.provider || conn.id || "unknown";
const overrideModel = override?.modelId?.trim();
if (overrideModel) return overrideModel;
return conn.defaultModel?.trim() || null;
}
export const NO_MODEL_PARTICIPANT_ERROR =
"No model set for this provider. Set a model for it on the Chaos Mode page or a default model on the connection.";
export const NO_MODEL_RUN_ERROR =
"No provider has a model set. Set a model for each provider on the Chaos Mode page or a default model on its connection.";
/** A participant that could not be dispatched because it has no model; nothing is sent upstream. */
function buildNoModelResult(provider: ProviderInfo): ModelResult {
return {
providerId: provider.id,
providerName: provider.name,
modelId: "",
status: "error",
content: null,
error: NO_MODEL_PARTICIPANT_ERROR,
durationMs: 0,
};
}
// ── Internal helpers — provider selection ───────────────────────────────────
@@ -293,8 +313,7 @@ async function dispatchToModel(
const ctx = { providerId, providerName, modelId };
try {
const model = modelId || providerId;
const request = buildDispatchRequest(model, messages, maxTokens, timeoutMs, apiKey);
const request = buildDispatchRequest(modelId, messages, maxTokens, timeoutMs, apiKey);
const res = await chatDispatch.postChatCompletion(request);
return await parseDispatchResponse(res, ctx, start);
} catch (err: unknown) {
@@ -344,8 +363,9 @@ async function dispatchParallel(
overrides: ChaosConfig["providerOverrides"],
params: EffectiveRunParams
): Promise<ModelResult[]> {
const tasks = providers.map((p) => () => {
const tasks = providers.map((p) => async () => {
const modelId = resolveModelId(p, overrides);
if (!modelId) return buildNoModelResult(p);
const messages = buildMessages(params.effectiveSystemPrompt, input.task);
return dispatchToModel(
p.id,
@@ -381,6 +401,10 @@ async function dispatchCollaborative(
for (const p of providers) {
const modelId = resolveModelId(p, overrides);
if (!modelId) {
results.push(buildNoModelResult(p));
continue;
}
const messages = buildMessages(params.effectiveSystemPrompt, context);
const result = await dispatchToModel(
@@ -424,6 +448,10 @@ export async function executeChaosRun(input: ChaosRunInput): Promise<ChaosRunRes
const params = resolveEffectiveRunParams(input, globalConfig);
const { providers, enabledOverrides } = await selectChaosProviders(input, globalConfig);
if (!providers.some((p) => resolveModelId(p, enabledOverrides))) {
throw new Error(NO_MODEL_RUN_ERROR);
}
const startedAt = new Date().toISOString();
const results =
params.mode === "parallel"

View File

@@ -0,0 +1,46 @@
// Vitest: a throwaway DATA_DIR for the whole run.
//
// The node:test runner gets one from tests/_setup/isolateDataDir.ts, but Vitest never loaded an
// equivalent. Code that resolves its own database path (the MCP audit did: DATA_DIR, else
// ~/.omniroute/storage.sqlite) then wrote into the developer's real database on every
// `npm run test:vitest` (202 audit rows of one test fixture had piled up in a developer's
// database).
//
// Wiring, in both Vitest configs:
// - `test.env: isolatedDataDirEnv()` hands the variables to every worker (a change to
// process.env inside a global setup does not reach the worker threads);
// - `test.globalSetup: ["./tests/_setup/vitestDataDir.ts"]` removes the directory at the end.
// A DATA_DIR chosen by the caller always wins, and is never removed.
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
// Set only when this file created the directory, so teardown never deletes a caller's DATA_DIR.
const CREATED_KEY = "OMNIROUTE_VITEST_DATA_DIR";
export function isolatedDataDirEnv(): Record<string, string> {
if (!process.env[CREATED_KEY]) {
if (process.env.DATA_DIR?.trim()) return {};
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-vitest-"));
process.env[CREATED_KEY] = dir;
process.env.DATA_DIR = dir;
}
const dir = process.env[CREATED_KEY] as string;
return {
DATA_DIR: dir,
// Same guard as isolateDataDir.ts: the plugin scanner would otherwise read ~/.omniroute/plugins.
OMNIROUTE_PLUGINS_DIR: process.env.OMNIROUTE_PLUGINS_DIR?.trim() || path.join(dir, "plugins"),
};
}
export default function setup(): () => void {
return () => {
const dir = process.env[CREATED_KEY];
if (!dir) return;
try {
fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
} catch {
// The OS reaps its temp dir eventually.
}
};
}

View File

@@ -223,6 +223,36 @@ test("POST /api/chaos/run — 200 happy path dispatches with internal Authorizat
assert.equal(capturedAuth?.startsWith("Bearer "), true);
});
test("POST /api/chaos/run — reports a clear error without any upstream request when no provider has a model", async () => {
await chaosConfig.setChaosConfig({
enabled: true,
defaultMode: "parallel",
providerOverrides: [],
timeoutMs: 120_000,
maxTokens: 4096,
});
await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "No model route provider",
apiKey: "sk-run-route-no-model",
});
let calls = 0;
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async () => {
calls++;
return jsonResponse({ choices: [{ message: { content: "should not run" } }] });
});
const res = await runRoute.POST(
makeRequest("POST", "http://localhost/api/chaos/run", { task: "Summarize" })
);
assert.equal(res.status, 500);
const body = (await res.json()) as { error: { message: string } };
assert.match(body.error.message, /No provider has a model set/);
assert.equal(calls, 0);
});
// ═════════════════════════════════════════════════════════════════════════════
// POST /api/skills/collect/chaos
// ═════════════════════════════════════════════════════════════════════════════

View File

@@ -28,6 +28,7 @@ const chaosExecutor = await import("../../src/lib/chaos/chaosExecutor.ts");
async function resetStorage() {
core.resetDbInstance();
chaosConfig.invalidateChaosConfigCache();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
}
@@ -241,3 +242,209 @@ test("executeChaosRun applies global config provider overrides for model resolut
assert.equal(capturedModel, "gpt-4-turbo");
});
const NO_MODEL_MESSAGE =
"No model set for this provider. Set a model for it on the Chaos Mode page or a default model on the connection.";
async function setOverrides(
providerOverrides: { providerId: string; modelId?: string; enabled: boolean }[]
) {
await chaosConfig.setChaosConfig({
enabled: true,
defaultMode: "parallel",
providerOverrides,
timeoutMs: 120_000,
maxTokens: 4096,
});
}
test("executeChaosRun skips a provider without a model: no upstream request, clear error, others still run", async () => {
await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "Has default model",
apiKey: "sk-test-has-model",
defaultModel: "gpt-4o-mini",
});
await providersDb.createProviderConnection({
provider: "anthropic",
authType: "apikey",
name: "No model",
apiKey: "sk-test-no-model",
});
const dispatchedModels: string[] = [];
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => {
const body = (await req.json()) as { model: string };
dispatchedModels.push(body.model);
return jsonResponse({ choices: [{ message: { content: "ok" } }] });
});
const result = await chaosExecutor.executeChaosRun({ task: "task" });
assert.deepEqual(dispatchedModels, ["gpt-4o-mini"]);
assert.equal(result.totalProviders, 2);
assert.equal(result.models.length, 2);
const ok = result.models.find((m) => m.providerName === "Has default model");
const skipped = result.models.find((m) => m.providerName === "No model");
assert.equal(ok?.status, "success");
assert.equal(skipped?.status, "error");
assert.equal(skipped?.content, null);
assert.equal(skipped?.error, NO_MODEL_MESSAGE);
assert.equal(skipped?.modelId, "");
});
test("executeChaosRun skips a provider without a model in collaborative mode and keeps the chain going", async () => {
await providersDb.createProviderConnection({
provider: "anthropic",
authType: "apikey",
name: "No model",
apiKey: "sk-test-collab-no-model",
});
await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "With model",
apiKey: "sk-test-collab-model",
defaultModel: "gpt-4o-mini",
});
let calls = 0;
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async () => {
calls++;
return jsonResponse({ choices: [{ message: { content: "chain-reply" } }] });
});
const result = await chaosExecutor.executeChaosRun({ task: "task", mode: "collaborative" });
assert.equal(calls, 1);
assert.equal(result.models.length, 2);
const skipped = result.models.find((m) => m.providerName === "No model");
const ran = result.models.find((m) => m.providerName === "With model");
assert.equal(skipped?.status, "error");
assert.equal(skipped?.error, NO_MODEL_MESSAGE);
assert.equal(ran?.status, "success");
assert.equal(result.summary, "chain-reply");
});
test("executeChaosRun: an enabled override modelId supplies the model for a connection without a default", async () => {
await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "Override only",
apiKey: "sk-test-override-only",
});
await setOverrides([{ providerId: "openai", modelId: "gpt-4-turbo", enabled: true }]);
const dispatchedModels: string[] = [];
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => {
dispatchedModels.push(((await req.json()) as { model: string }).model);
return jsonResponse({ choices: [{ message: { content: "ok" } }] });
});
const result = await chaosExecutor.executeChaosRun({ task: "task" });
assert.deepEqual(dispatchedModels, ["gpt-4-turbo"]);
assert.equal(result.models[0].status, "success");
});
test("executeChaosRun: a blank override modelId does not count as a model", async () => {
await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "Blank override",
apiKey: "sk-test-blank-override",
});
await providersDb.createProviderConnection({
provider: "anthropic",
authType: "apikey",
name: "Has default",
apiKey: "sk-test-blank-override-2",
defaultModel: "claude-3-5-sonnet",
});
await setOverrides([
{ providerId: "openai", modelId: " ", enabled: true },
{ providerId: "anthropic", enabled: true },
]);
const dispatchedModels: string[] = [];
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => {
dispatchedModels.push(((await req.json()) as { model: string }).model);
return jsonResponse({ choices: [{ message: { content: "ok" } }] });
});
const result = await chaosExecutor.executeChaosRun({ task: "task" });
assert.deepEqual(dispatchedModels, ["claude-3-5-sonnet"]);
const blank = result.models.find((m) => m.providerName === "Blank override");
assert.equal(blank?.status, "error");
assert.equal(blank?.error, NO_MODEL_MESSAGE);
});
test("executeChaosRun: a disabled override modelId is ignored in favor of the connection default", async () => {
await providersDb.createProviderConnection({
provider: "anthropic",
authType: "apikey",
name: "Disabled override",
apiKey: "sk-test-disabled-override",
defaultModel: "claude-3-5-sonnet",
});
await setOverrides([{ providerId: "anthropic", modelId: "ignored-model", enabled: false }]);
const dispatchedModels: string[] = [];
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => {
dispatchedModels.push(((await req.json()) as { model: string }).model);
return jsonResponse({ choices: [{ message: { content: "ok" } }] });
});
await chaosExecutor.executeChaosRun({ task: "task" });
assert.deepEqual(dispatchedModels, ["claude-3-5-sonnet"]);
});
test("executeChaosRun: an override modelId wins over the connection default model", async () => {
await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "Both",
apiKey: "sk-test-both",
defaultModel: "gpt-4o-mini",
});
await setOverrides([{ providerId: "openai", modelId: "gpt-4-turbo", enabled: true }]);
const dispatchedModels: string[] = [];
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => {
dispatchedModels.push(((await req.json()) as { model: string }).model);
return jsonResponse({ choices: [{ message: { content: "ok" } }] });
});
await chaosExecutor.executeChaosRun({ task: "task" });
assert.deepEqual(dispatchedModels, ["gpt-4-turbo"]);
});
test("executeChaosRun fails fast without any upstream request when no provider has a model", async () => {
for (const provider of ["openai", "anthropic", "gemini"]) {
await providersDb.createProviderConnection({
provider,
authType: "apikey",
name: `No model ${provider}`,
apiKey: `sk-test-nomodel-${provider}`,
});
}
let calls = 0;
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async () => {
calls++;
return jsonResponse({ choices: [{ message: { content: "ok" } }] });
});
for (const mode of ["parallel", "collaborative"] as const) {
await assert.rejects(
() => chaosExecutor.executeChaosRun({ task: "task", mode }),
/No provider has a model set\. Set a model for each provider on the Chaos Mode page or a default model on its connection\./
);
}
assert.equal(calls, 0);
});

View File

@@ -1,5 +1,8 @@
import { afterEach, describe, it } from "node:test";
import { after, afterEach, describe, it } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import {
buildChecks,
runCountsGate,
@@ -86,10 +89,18 @@ describe("readCodeFacts failure is reported as a strict failure", () => {
// The gate itself no longer reports that failure as documentation drift: facts that cannot be
// collected are an INFRASTRUCTURE failure (exit 2), so a host too loaded to run the tsx probe
// can never be mistaken for docs that need editing.
//
// The gate runs against an empty root: a real STRICT drift still wins over the infrastructure
// verdict (exit 1), and a feature branch that adds a migration or a provider carries such a
// drift by design (derived values are refreshed at release). Against the working tree, these
// assertions would depend on the branch they run on.
const emptyRoot = fs.mkdtempSync(path.join(os.tmpdir(), "docs-counts-err-seam-"));
after(() => fs.rmSync(emptyRoot, { recursive: true, force: true }));
describe("the release gate classifies a facts failure as infrastructure", () => {
it("exits 2 with an infrastructure verdict, not a STRICT drift", () => {
__setSpawnSyncForTest(failingProbe);
const r = runCountsGate({ mode: "release" }) as GateResult;
const r = runCountsGate({ mode: "release", root: emptyRoot }) as GateResult;
assert.equal(r.mode, "release");
assert.equal(r.exitCode, 2, `expected exit 2 (infrastructure), got ${r.exitCode}`);
assert.ok(r.infra, "the result must carry the infrastructure failure");
@@ -111,7 +122,7 @@ describe("the release gate classifies a facts failure as infrastructure", () =>
signal: "SIGKILL",
}) as never
);
const r = runCountsGate({ mode: "release" }) as GateResult;
const r = runCountsGate({ mode: "release", root: emptyRoot }) as GateResult;
assert.equal(r.exitCode, 2);
assert.match(JSON.stringify(r.infra), /timeout/i);
});

View File

@@ -114,7 +114,7 @@ test("NVIDIA aliases every OpenAI tool-name surface and can restore the provider
assert.notEqual(collisions.tools[0].function.name, collisions.tools[1].function.name);
});
test("NVIDIA executor applies its registry-scoped tool-name limit without changing OpenAI", () => {
test("NVIDIA executor applies its registry-scoped tool-name limit without changing a provider that has no limit", () => {
const original = "mcp__plugin_chrome-devtools-mcp_chrome-devtools__get.console/message";
const request = {
tools: [{ type: "function", function: { name: original } }],
@@ -134,7 +134,7 @@ test("NVIDIA executor applies its registry-scoped tool-name limit without changi
false,
null
);
const openai = new DefaultExecutor("openai").transformRequest(
const openai = new DefaultExecutor("groq").transformRequest(
"gpt-5.4",
structuredClone(request),
false,

View File

@@ -0,0 +1,196 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
import test from "node:test";
// Isolated DATA_DIR under the worktree's _artifacts/ (never the real ~/.omniroute).
const ARTIFACTS_DIR = path.resolve(import.meta.dirname, "../../_artifacts");
fs.mkdirSync(ARTIFACTS_DIR, { recursive: true });
process.env.DATA_DIR = fs.mkdtempSync(path.join(ARTIFACTS_DIR, "openai-tool-name-limit-"));
const core = await import("../../src/lib/db/core.ts");
const { getRegistryEntry } = await import("../../open-sse/config/providerRegistry.ts");
const { DefaultExecutor } = await import("../../open-sse/executors/default.ts");
const { AzureOpenAIExecutor } = await import("../../open-sse/executors/azure-openai.ts");
const { AzureAiExecutor } = await import("../../open-sse/executors/azure-ai.ts");
const { resolveToolNameMaxLength } =
await import("../../open-sse/executors/toolNameLengthAlias.ts");
const { restoreOpenAIToolNames } =
await import("../../open-sse/translator/helpers/toolCallHelper.ts");
const { createSSEStream } = await import("../../open-sse/utils/stream.ts");
const { FORMATS } = await import("../../open-sse/translator/formats.ts");
// OpenAI Chat Completions (and Azure OpenAI, which implements the same schema) reject a
// function name over 64 chars. MCP tool names from VS Code / Claude Code routinely exceed
// it (cheaper-inference/OmniCopilot#27).
const LONG = "mcp__example-observability-prod__trace_top_time_consuming_operations_by_service"; // 78
const SHORT = "mcp__github__get_issue";
const EXACT_64 = "a".repeat(64);
type ToolCall = { id: string; type: string; function: { name: string; arguments: string } };
type ChatBody = {
model: string;
tools: Array<{ type: string; function: { name: string; parameters: object } }>;
tool_choice?: { type: string; function: { name: string } };
messages: Array<{
role: string;
content: string | null;
tool_calls?: ToolCall[];
tool_call_id?: string;
name?: string;
}>;
_toolNameMap?: Map<string, string>;
};
function chatBody(toolName: string): ChatBody {
return {
model: "m",
tools: [
{ type: "function", function: { name: toolName, parameters: { type: "object" } } },
{ type: "function", function: { name: SHORT, parameters: { type: "object" } } },
],
tool_choice: { type: "function", function: { name: toolName } },
messages: [
{ role: "user", content: "hi" },
{
role: "assistant",
content: null,
tool_calls: [{ id: "c1", type: "function", function: { name: toolName, arguments: "{}" } }],
},
{ role: "tool", tool_call_id: "c1", name: toolName, content: "ok" },
],
};
}
const EXECUTORS: Record<string, () => InstanceType<typeof DefaultExecutor>> = {
openai: () => new DefaultExecutor("openai"),
"azure-openai": () => new AzureOpenAIExecutor(),
"azure-ai": () => new AzureAiExecutor(),
};
function transform(
provider: string,
toolName: string,
providerSpecificData?: Record<string, unknown>
): ChatBody {
return EXECUTORS[provider]().transformRequest("m", chatBody(toolName), false, {
apiKey: "k",
providerSpecificData,
}) as ChatBody;
}
for (const provider of Object.keys(EXECUTORS)) {
test(`${provider}: a ${LONG.length}-char tool name goes upstream at most 64 chars everywhere`, () => {
assert.ok(LONG.length > 64);
const out = transform(provider, LONG);
const sent = out.tools[0].function.name;
assert.ok(sent.length <= 64, `tools[] name is ${sent.length} chars`);
assert.match(sent, /^[A-Za-z0-9_-]{1,64}$/);
assert.equal(out.tools[1].function.name, SHORT);
assert.equal(out.tool_choice?.function.name, sent);
assert.equal(out.messages[1].tool_calls?.[0].function.name, sent);
assert.equal(out.messages[2].name, sent);
assert.equal(out._toolNameMap?.get(sent), LONG);
assert.equal(JSON.stringify(out).includes("_toolNameMap"), false);
});
test(`${provider}: a non-streaming tool call comes back under its original name`, () => {
const out = transform(provider, LONG);
const sent = out.tools[0].function.name;
const response = {
choices: [
{
message: {
role: "assistant",
content: null,
tool_calls: [{ id: "c2", type: "function", function: { name: sent, arguments: "{}" } }],
},
finish_reason: "tool_calls",
},
],
};
assert.equal(restoreOpenAIToolNames(response, out._toolNameMap), true);
assert.equal(response.choices[0].message.tool_calls[0].function.name, LONG);
});
test(`${provider}: a streamed tool call comes back under its original name`, async () => {
const out = transform(provider, LONG);
const sent = out.tools[0].function.name;
const chunk = {
id: "chatcmpl-1",
object: "chat.completion.chunk",
created: 1,
model: "m",
choices: [
{
index: 0,
delta: {
tool_calls: [
{ index: 0, id: "c3", type: "function", function: { name: sent, arguments: "{}" } },
],
},
finish_reason: null,
},
],
};
const source = new ReadableStream({
start(controller) {
const encoder = new TextEncoder();
controller.enqueue(encoder.encode(`data: ${JSON.stringify(chunk)}\n\n`));
controller.enqueue(encoder.encode("data: [DONE]\n\n"));
controller.close();
},
});
const text = await new Response(
source.pipeThrough(
createSSEStream({
mode: "passthrough",
sourceFormat: FORMATS.OPENAI,
provider,
model: "m",
toolNameMap: out._toolNameMap,
body: { messages: [{ role: "user", content: "hi" }] },
})
)
).text();
assert.ok(text.includes(JSON.stringify(LONG)), "original name missing from the stream");
assert.equal(text.includes(JSON.stringify(sent)), false, "alias leaked to the client");
});
test(`${provider}: names of 64 chars or fewer are untouched`, () => {
for (const name of [EXACT_64, SHORT]) {
const out = transform(provider, name);
assert.equal(out.tools[0].function.name, name);
assert.equal(out.messages[1].tool_calls?.[0].function.name, name);
assert.equal(out._toolNameMap, undefined);
}
});
}
test("openai declares toolNameMaxLength 64 in the registry", () => {
assert.equal(getRegistryEntry("openai")?.toolNameMaxLength, 64);
});
test("azure providers have no registry entry, so their 64 is a built-in limit", () => {
assert.equal(getRegistryEntry("azure-openai"), null);
assert.equal(getRegistryEntry("azure-ai"), null);
assert.equal(resolveToolNameMaxLength("azure-openai", undefined), 64);
assert.equal(resolveToolNameMaxLength("azure-ai", undefined), 64);
});
test("a connection toolNameMaxLength still applies where the registry declares none", () => {
// Azure: a stricter gateway in front of the deployment lowers the built-in 64.
const azure = transform("azure-openai", "x".repeat(50), { toolNameMaxLength: 40 });
assert.ok(azure.tools[0].function.name.length <= 40);
// A provider with neither a registry nor a built-in limit keeps honoring the connection.
assert.equal(resolveToolNameMaxLength("openai-compatible-gw", { toolNameMaxLength: 48 }), 48);
assert.equal(resolveToolNameMaxLength("openai-compatible-gw", undefined), undefined);
});
test.after(() => {
core.resetDbInstance();
fs.rmSync(process.env.DATA_DIR as string, { recursive: true, force: true });
});

View File

@@ -210,7 +210,7 @@ test("applyToolNameLengthAliases accepts a custom normalizer and ignores invalid
test("applyRegistryToolNameLengthAliases is a no-op for providers without a limit", () => {
const body: Record<string, unknown> = chatBody([LONG_A]);
assert.equal(applyRegistryToolNameLengthAliases("openai", body).size, 0);
assert.equal(applyRegistryToolNameLengthAliases("groq", body).size, 0);
assert.equal((body as unknown as ChatBody).tools[0].function.name, LONG_A);
assert.equal(body._toolNameMap, undefined);
assert.equal(applyRegistryToolNameLengthAliases("nvidia", body).size, 1);

View File

@@ -30,6 +30,7 @@ const strip = (await import("../../scripts/ad-hoc/strip-derived-from-pr.mjs")) a
ctx: Ctx;
}) => { files: PlannedFile[]; manual: string[] };
loadContext: () => Promise<Ctx>;
readBlob: (ref: string, file: string, opts?: { cwd?: string }) => string | null;
buildRestoreCommit: (args: {
cwd: string;
head: string;
@@ -83,6 +84,28 @@ test("claims: each value the PR changed goes back to the base value; prose edits
assert.deepEqual(r.manual, []);
});
test("claims: the base's own digits come back, not a re-rendered number", () => {
// A translated mirror writes the count in Persian digits; the head rewrote it in ASCII.
const toAscii = (t: string) => t.replace(/[۰-۹]/g, (d) => String("۰۱۲۳۴۵۶۷۸۹".indexOf(d)));
const nativeCheck: Check = {
label: "DB migrations count (README mirrors)",
extract: (content) =>
[...content.matchAll(/([0-9۰-۹]+) مهاجرت/dg)].map((m) => ({
value: Number(toAscii(m[1])),
start: (m as unknown as { indices: [number, number][] }).indices[1][0],
end: (m as unknown as { indices: [number, number][] }).indices[1][1],
})),
};
const base = "<td>۱۴۷ ماژول، ۲۳۳ مهاجرت</td>\n";
const head = "<td>۱۴۷ ماژول، 234 مهاجرت</td>\n";
const r = strip.restoreClaims(base, head, [nativeCheck]);
assert.equal(r.content, base, "byte-identical to the base line");
assert.deepEqual(r.edits, [
{ label: "DB migrations count (README mirrors)", from: "234", to: "233" },
]);
assert.deepEqual(r.manual, []);
});
test("claims: a PR that adds or removes a claim is left to a human", () => {
const r = strip.restoreClaims("387 providers.\n", "388 providers. Also 388 providers.\n", [
providerCheck,
@@ -210,3 +233,53 @@ test("buildRestoreCommit: one commit on top of the head, only the planned files
fs.rmSync(dir, { recursive: true, force: true });
}
});
test("restore from real git blobs is byte-exact: the final newline survives", () => {
// Regression: contents read through a trimming helper lost the last "\n", so the "restored"
// file still differed from the merge base and the PR stayed conflicting on it.
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "strip-bytes-"));
const git = (...args: string[]) =>
execFileSync("git", args, { cwd: dir, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
const write = (file: string, content: string) => fs.writeFileSync(path.join(dir, file), content);
try {
git("init", "-q");
git("config", "user.name", "test");
git("config", "user.email", "test@example.invalid");
write("stryker.conf.json", '{\n "testFiles": ["a"]\n}\n');
write("AGENTS.md", "# Agents\nOne endpoint, 387 providers.\n");
git("add", "-A");
git("commit", "-q", "-m", "merge base");
const base = git("rev-parse", "HEAD").trim();
write("stryker.conf.json", '{\n "testFiles": ["a", "b"]\n}\n');
write("AGENTS.md", "# Agents\nOne endpoint, 388 providers.\nNew rule.\n");
git("commit", "-q", "-am", "PR head");
const head = git("rev-parse", "HEAD").trim();
assert.equal(
strip.readBlob(base, "AGENTS.md", { cwd: dir }),
"# Agents\nOne endpoint, 387 providers.\n"
);
assert.equal(strip.readBlob(base, "missing.txt", { cwd: dir }), null);
const plan = strip.planPrRestore({
changed: ["stryker.conf.json", "AGENTS.md"],
readBase: (f) => strip.readBlob(base, f, { cwd: dir }),
readHead: (f) => strip.readBlob(head, f, { cwd: dir }),
ctx,
});
const commit = strip.buildRestoreCommit({ cwd: dir, head, files: plan.files }).trim();
assert.equal(
git("rev-parse", `${commit}:stryker.conf.json`),
git("rev-parse", `${base}:stryker.conf.json`),
"the restored blob is the merge-base blob, byte for byte"
);
assert.equal(git("diff", "--name-only", base, commit, "--", "stryker.conf.json"), "");
assert.equal(
git("show", `${commit}:AGENTS.md`),
"# Agents\nOne endpoint, 387 providers.\nNew rule.\n",
"digit-only revert keeps the PR's prose and the final newline"
);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});

View File

@@ -1,6 +1,7 @@
import { defineConfig } from "vitest/config";
import react from "@vitejs/plugin-react";
import path from "path";
import { isolatedDataDirEnv } from "./tests/_setup/vitestDataDir";
export default defineConfig({
test: {
@@ -10,6 +11,9 @@ export default defineConfig({
testTimeout: 20000,
globals: true,
setupFiles: ["./tests/_setup/vitestUiPolyfills.ts"],
// A throwaway DATA_DIR for the run, so no suite can reach the developer's ~/.omniroute.
env: isolatedDataDirEnv(),
globalSetup: ["./tests/_setup/vitestDataDir.ts"],
pool: "threads",
maxWorkers: 20,
fileParallelism: true,

View File

@@ -1,10 +1,14 @@
import { defineConfig } from "vitest/config";
import path from "path";
import { isolatedDataDirEnv } from "./tests/_setup/vitestDataDir";
export default defineConfig({
test: {
environment: "node",
globals: true,
// A throwaway DATA_DIR for the run, so no suite can reach the developer's ~/.omniroute.
env: isolatedDataDirEnv(),
globalSetup: ["./tests/_setup/vitestDataDir.ts"],
pool: "threads",
// MCP suites initialize isolated SQLite databases; Vitest 5 at 20 workers
// makes native migration/close operations contend and flakes audit tests.