mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-16 17:09:09 +00:00
Merge remote-tracking branch 'origin/release/v3.8.54' into fix/truthful-streaming-usage-20054
This commit is contained in:
1
changelog.d/fixes/20252-vitest-isolate-data-dir.md
Normal file
1
changelog.d/fixes/20252-vitest-isolate-data-dir.md
Normal file
@@ -0,0 +1 @@
|
||||
- **fix(mcp):** `npm run test:vitest` no longer writes MCP audit rows into the developer's `~/.omniroute/storage.sqlite`: both Vitest configs run with a throwaway `DATA_DIR`, and the MCP audit refuses the home database in a test context without `DATA_DIR` (opt back in with `OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1`) ([#20252](https://github.com/cheaper-inference/OmniRoute/pull/20252))
|
||||
@@ -54,16 +54,24 @@ traffic is never affected by it.
|
||||
`400 — "Chaos Mode is not enabled. Enable it in Dashboard → Chaos Mode."`.
|
||||
3. Pick the participants and defaults (persisted per instance via the settings store):
|
||||
|
||||
| Field | Meaning | Default / limits |
|
||||
| ------------------- | ------------------------------------------------------------------- | --------------------------------------- |
|
||||
| `enabled` | Master switch | `false` |
|
||||
| `defaultMode` | `parallel` or `collaborative` (see below) | `parallel` |
|
||||
| Field | Meaning | Default / limits |
|
||||
| ------------------- | ------------------------------------------------------------------------ | -------------------------------------- |
|
||||
| `enabled` | Master switch | `false` |
|
||||
| `defaultMode` | `parallel` or `collaborative` (see below) | `parallel` |
|
||||
| `providerOverrides` | Per-provider participation (`providerId`, optional `modelId`, `enabled`) | empty = every active provider, max 200 |
|
||||
| `systemPrompt` | Override for the built-in Chaos system prompt | optional, max 10 000 chars |
|
||||
| `timeoutMs` | Max time per model call | `120000` (5 000–600 000) |
|
||||
| `maxTokens` | `max_tokens` per model call | `4096` (256–128 000) |
|
||||
| `systemPrompt` | Override for the built-in Chaos system prompt | optional, max 10 000 chars |
|
||||
| `timeoutMs` | Max time per model call | `120000` (5 000–600 000) |
|
||||
| `maxTokens` | `max_tokens` per model call | `4096` (256–128 000) |
|
||||
|
||||
4. Run a **test from the page itself** — the results panel shows each provider's answer,
|
||||
4. Give every participating provider a **model**. A provider's model is the `modelId` of its
|
||||
enabled override on the Chaos Mode page (it wins), otherwise the **default model** of its
|
||||
connection. A provider id is never used as a model name. A provider with neither is **not
|
||||
called**: it shows up in the results as `status: "error"` with the message _"No model set for
|
||||
this provider. Set a model for it on the Chaos Mode page or a default model on the
|
||||
connection."_ and an empty `modelId`, while the other providers run normally. If no selected
|
||||
provider has a model, the run fails before any upstream request with _"No provider has a
|
||||
model set. …"_ (HTTP 500 on both endpoints).
|
||||
5. Run a **test from the page itself** — the results panel shows each provider's answer,
|
||||
status and duration.
|
||||
|
||||
## Execution modes
|
||||
@@ -84,11 +92,11 @@ Cookie-authenticated (the management session — see
|
||||
```jsonc
|
||||
// body
|
||||
{
|
||||
"task": "Compare approaches to X", // required
|
||||
"providers": ["glm", "kimi"], // optional filter
|
||||
"mode": "parallel", // optional — overrides defaultMode
|
||||
"systemPrompt": "…", // optional override
|
||||
"maxTokens": 4096 // optional override
|
||||
"task": "Compare approaches to X", // required
|
||||
"providers": ["glm", "kimi"], // optional filter
|
||||
"mode": "parallel", // optional — overrides defaultMode
|
||||
"systemPrompt": "…", // optional override
|
||||
"maxTokens": 4096, // optional override
|
||||
}
|
||||
```
|
||||
|
||||
@@ -115,10 +123,16 @@ Both endpoints return the same shape:
|
||||
"totalProviders": 3,
|
||||
"totalResults": 3,
|
||||
"models": [
|
||||
{ "providerId": "glm", "providerName": "GLM", "modelId": "glm-4.7",
|
||||
"status": "success", "content": "…", "durationMs": 3210 }
|
||||
{
|
||||
"providerId": "glm",
|
||||
"providerName": "GLM",
|
||||
"modelId": "glm-4.7",
|
||||
"status": "success",
|
||||
"content": "…",
|
||||
"durationMs": 3210,
|
||||
},
|
||||
],
|
||||
"summary": "…" // collaborative mode only
|
||||
"summary": "…", // collaborative mode only
|
||||
}
|
||||
```
|
||||
|
||||
@@ -127,6 +141,8 @@ Both endpoints return the same shape:
|
||||
- **`400 Chaos Mode is not enabled`** — step 2 above: the global switch is off.
|
||||
- **API key gets rejected on `/api/skills/collect/chaos`** — the key lacks the per-key
|
||||
`chaosModeEnabled` permission (off by default; this is a setting, not an error).
|
||||
- **A provider shows `No model set for this provider`** — set its model on the Chaos Mode page
|
||||
or a default model on its connection (see Setup, step 4).
|
||||
- **A provider you expected is missing from the results** — check `providerOverrides` on the
|
||||
Chaos Mode page (a disabled override excludes it) and whether the provider connection is
|
||||
active.
|
||||
|
||||
@@ -756,7 +756,10 @@ These keys go in a connection's `providerSpecificData` (for example via
|
||||
|
||||
- `toolNameMaxLength` (integer, 1–1024) — for OpenAI chat-format compatible connections whose
|
||||
gateway rejects long tool names. Longer names are sent as short aliases and restored in the
|
||||
response.
|
||||
response. OpenAI, Azure OpenAI, Azure AI Foundry, Mistral, NVIDIA, OpenCode Zen/Go and Muse
|
||||
Code already alias names over 64 characters with no setting; a connection value lowers the
|
||||
limit only where the provider has no built-in one (an Azure connection behind a stricter
|
||||
gateway, or a custom compatible node).
|
||||
- `quotaEndpoint` — the gateway's quota/credits endpoint, read for the dashboard quota strip and
|
||||
quota-aware routing. Works on both OpenAI- and Anthropic-compatible connections; the full shape
|
||||
is documented in `open-sse/services/usage/openaiCompatible.ts`.
|
||||
|
||||
@@ -10,6 +10,8 @@ export const openaiProvider: RegistryEntry = {
|
||||
reasoningTransport: "opaque",
|
||||
authType: "apikey",
|
||||
authHeader: "bearer",
|
||||
// OpenAI Chat Completions rejects function names over 64 chars (MCP tool names hit this).
|
||||
toolNameMaxLength: 64,
|
||||
defaultContextLength: 128000,
|
||||
models: [
|
||||
// https://developers.openai.com/api/docs/models/gpt-6.1-sol
|
||||
|
||||
@@ -67,10 +67,23 @@ export function applyRegistryToolNameLengthAliases(
|
||||
return applyToolNameLengthAliases(body, getRegistryEntry(provider)?.toolNameMaxLength, normalize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Limits for providers that have no registry entry to declare `toolNameMaxLength` on:
|
||||
* Azure OpenAI and Azure AI Foundry are keyed connections handled by their own executors
|
||||
* (`azure-openai`, `azure-ai`), and both speak the OpenAI Chat Completions schema, which
|
||||
* rejects a function name over 64 chars.
|
||||
*/
|
||||
const BUILT_IN_TOOL_NAME_MAX_LENGTH: Readonly<Record<string, number>> = {
|
||||
"azure-openai": 64,
|
||||
"azure-ai": 64,
|
||||
};
|
||||
|
||||
/**
|
||||
* The registry `toolNameMaxLength`, else the connection's
|
||||
* `providerSpecificData.toolNameMaxLength`: custom OpenAI chat-format compatible nodes never
|
||||
* have a registry entry, so this is how they opt in for a stricter upstream.
|
||||
* `providerSpecificData.toolNameMaxLength`, else a built-in limit for registry-less
|
||||
* providers. Custom OpenAI chat-format compatible nodes never have a registry entry, so the
|
||||
* connection value is how they opt in for a stricter upstream; it also lowers the built-in
|
||||
* limit of an Azure connection sitting behind a stricter gateway.
|
||||
*/
|
||||
export function resolveToolNameMaxLength(
|
||||
provider: string,
|
||||
@@ -79,7 +92,11 @@ export function resolveToolNameMaxLength(
|
||||
const connectionLimit = isRecord(providerSpecificData)
|
||||
? providerSpecificData.toolNameMaxLength
|
||||
: undefined;
|
||||
return getRegistryEntry(provider)?.toolNameMaxLength ?? (connectionLimit as number | undefined);
|
||||
return (
|
||||
getRegistryEntry(provider)?.toolNameMaxLength ??
|
||||
(connectionLimit as number | undefined) ??
|
||||
BUILT_IN_TOOL_NAME_MAX_LENGTH[provider]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
// The MCP audit resolves its own database path: DATA_DIR, else ~/.omniroute/storage.sqlite. It
|
||||
// writes there whenever that file exists. A Vitest run without DATA_DIR therefore appended
|
||||
// `omniroute_test_combo` rows to the developer's real database on every run (202 of them had
|
||||
// piled up in one install). Two guards close that:
|
||||
// - tests/_setup/vitestDataDir.ts gives every Vitest run a throwaway DATA_DIR;
|
||||
// - in a test context with no DATA_DIR, the audit refuses the home database (the same rule
|
||||
// src/lib/dataPaths.ts applies to the main database, #10428), unless
|
||||
// OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1 opts back in.
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
it("runs Vitest with a DATA_DIR outside the developer's ~/.omniroute", () => {
|
||||
const dataDir = process.env.DATA_DIR;
|
||||
expect(dataDir, "tests/_setup/vitestDataDir.ts must set DATA_DIR").toBeTruthy();
|
||||
expect(path.resolve(dataDir as string)).not.toBe(path.resolve(os.homedir(), ".omniroute"));
|
||||
});
|
||||
|
||||
describe("MCP audit database path in a test context", () => {
|
||||
const saved = {
|
||||
DATA_DIR: process.env.DATA_DIR,
|
||||
OMNIROUTE_ALLOW_DEFAULT_DATA_DIR: process.env.OMNIROUTE_ALLOW_DEFAULT_DATA_DIR,
|
||||
};
|
||||
let fakeHome: string;
|
||||
let homeDb: string;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
globalThis.__omnirouteMcpAuditDb = undefined;
|
||||
fakeHome = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-audit-home-"));
|
||||
fs.mkdirSync(path.join(fakeHome, ".omniroute"));
|
||||
homeDb = path.join(fakeHome, ".omniroute", "storage.sqlite");
|
||||
fs.writeFileSync(homeDb, "");
|
||||
// Vitest runs files in worker threads, where os.homedir() reads the real process environment
|
||||
// and ignores a changed process.env.HOME, so the home directory is mocked instead.
|
||||
vi.doMock("node:os", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("node:os")>();
|
||||
const homedir = () => fakeHome;
|
||||
return { ...actual, homedir, default: { ...actual, homedir } };
|
||||
});
|
||||
delete process.env.DATA_DIR;
|
||||
delete process.env.OMNIROUTE_ALLOW_DEFAULT_DATA_DIR;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const [key, value] of Object.entries(saved)) {
|
||||
if (value === undefined) delete process.env[key];
|
||||
else process.env[key] = value;
|
||||
}
|
||||
globalThis.__omnirouteMcpAuditDb = undefined;
|
||||
vi.doUnmock("node:os");
|
||||
fs.rmSync(fakeHome, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
async function auditWithRecordingDriver() {
|
||||
const opened: string[] = [];
|
||||
class RecordingDatabase {
|
||||
open = true;
|
||||
constructor(file: string) {
|
||||
opened.push(file);
|
||||
}
|
||||
prepare() {
|
||||
return { run: () => ({ changes: 1 }), get: () => undefined, all: () => [] };
|
||||
}
|
||||
pragma() {}
|
||||
close() {
|
||||
this.open = false;
|
||||
}
|
||||
}
|
||||
const audit = await import("../audit.ts");
|
||||
audit.__setBetterSqliteLoaderForTests(() => RecordingDatabase);
|
||||
audit.__setAuditCallerIdResolverForTests(async () => undefined);
|
||||
return { audit, opened };
|
||||
}
|
||||
|
||||
it("never opens the home database in a test context without DATA_DIR, even when it exists", async () => {
|
||||
const { audit, opened } = await auditWithRecordingDriver();
|
||||
try {
|
||||
await audit.logToolCall("omniroute_test_combo", { combo: "x" }, { ok: true }, 3, true);
|
||||
expect(opened).toEqual([]);
|
||||
} finally {
|
||||
audit.__setBetterSqliteLoaderForTests(null);
|
||||
audit.__setAuditCallerIdResolverForTests(null);
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
it("uses the home database only with OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1, like the main database", async () => {
|
||||
process.env.OMNIROUTE_ALLOW_DEFAULT_DATA_DIR = "1";
|
||||
const { audit, opened } = await auditWithRecordingDriver();
|
||||
try {
|
||||
await audit.logToolCall("omniroute_test_combo", { combo: "x" }, { ok: true }, 3, true);
|
||||
expect(opened).toEqual([homeDb]);
|
||||
} finally {
|
||||
audit.__setBetterSqliteLoaderForTests(null);
|
||||
audit.__setAuditCallerIdResolverForTests(null);
|
||||
audit.closeAuditDb();
|
||||
}
|
||||
}, 30000);
|
||||
});
|
||||
@@ -9,6 +9,7 @@
|
||||
import { hashInput, summarizeOutput } from "./schemas/audit.ts";
|
||||
import { runtimeRequire } from "../../src/lib/db/adapters/runtimeRequire.ts";
|
||||
import { isNativeSqliteLoadError } from "../../src/lib/db/core.ts";
|
||||
import { isTestContext } from "../../src/lib/dataPaths.ts";
|
||||
import { resolveMcpCallerApiKeyId } from "./mcpCallerIdentity.ts";
|
||||
|
||||
// ============ Database Connection ============
|
||||
@@ -308,6 +309,20 @@ async function getDb(): Promise<AuditDatabase | null> {
|
||||
const { join } = await import("node:path");
|
||||
const { existsSync } = await import("node:fs");
|
||||
|
||||
// A test run that never chose a DATA_DIR must not write to the developer's real database:
|
||||
// the same rule src/lib/dataPaths.ts applies to the main database (#10428), with the same
|
||||
// opt-in. Not cached, so a DATA_DIR set later in the process still enables auditing.
|
||||
if (
|
||||
!process.env.DATA_DIR &&
|
||||
isTestContext() &&
|
||||
process.env.OMNIROUTE_ALLOW_DEFAULT_DATA_DIR !== "1"
|
||||
) {
|
||||
console.error(
|
||||
"[MCP Audit] No DATA_DIR in a test context — audit logging disabled instead of using ~/.omniroute"
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
const dbPath = process.env.DATA_DIR
|
||||
? join(process.env.DATA_DIR, "storage.sqlite")
|
||||
: join(homedir(), ".omniroute", "storage.sqlite");
|
||||
|
||||
@@ -52,6 +52,8 @@ export function classifyFile(file, { claimFiles, instructionFiles }) {
|
||||
/**
|
||||
* Each managed value the head changed goes back to the base value of the same claim (same check,
|
||||
* same position). A check whose claim count differs between base and head is left to a human.
|
||||
* The replacement is the base's own text for that claim, not the parsed value: a mirror that
|
||||
* writes "۲۳۳" must get "۲۳۳" back, or the line still differs from the base and keeps conflicting.
|
||||
*/
|
||||
export function restoreClaims(base, head, checks) {
|
||||
const edits = [];
|
||||
@@ -72,6 +74,7 @@ export function restoreClaims(base, head, checks) {
|
||||
label: check.label,
|
||||
from: hv[i],
|
||||
to: bv[i],
|
||||
text: b[i].start >= 0 ? (base ?? "").slice(b[i].start, b[i].end) : bv[i],
|
||||
start: claim.start,
|
||||
end: claim.end,
|
||||
});
|
||||
@@ -84,7 +87,7 @@ export function restoreClaims(base, head, checks) {
|
||||
const key = `${e.start}:${e.end}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
content = content.slice(0, e.start) + e.to + content.slice(e.end);
|
||||
content = content.slice(0, e.start) + e.text + content.slice(e.end);
|
||||
}
|
||||
for (const check of checks) {
|
||||
if (!edits.some((e) => e.label === check.label)) continue;
|
||||
@@ -279,6 +282,24 @@ function tryGit(args, opts) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A file's exact bytes at `ref` (null when the path does not exist there). Never trimmed: the
|
||||
* restore must reproduce the merge-base blob byte for byte — a trimmed read drops the final
|
||||
* newline, leaves the file "modified" and the PR still conflicting.
|
||||
*/
|
||||
export function readBlob(ref, file, { cwd = ROOT } = {}) {
|
||||
try {
|
||||
return execFileSync("git", ["show", `${ref}:${file}`], {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
maxBuffer: 256 << 20,
|
||||
});
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const RESTORE_MESSAGE = [
|
||||
"chore: drop derived values the release refresh now owns",
|
||||
"",
|
||||
@@ -363,8 +384,8 @@ async function main() {
|
||||
const changed = git(["diff", "--name-only", mb, ref]).split("\n").filter(Boolean);
|
||||
const plan = planPrRestore({
|
||||
changed,
|
||||
readBase: (f) => tryGit(["show", `${mb}:${f}`]),
|
||||
readHead: (f) => tryGit(["show", `${ref}:${f}`]),
|
||||
readBase: (f) => readBlob(mb, f),
|
||||
readHead: (f) => readBlob(ref, f),
|
||||
ctx,
|
||||
});
|
||||
const entry = {
|
||||
|
||||
@@ -117,21 +117,41 @@ function resolveEffectiveRunParams(
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the best model ID for a provider connection.
|
||||
* Applies provider overrides from global chaos config if present.
|
||||
* Resolve the model ID for a provider connection: an enabled override's `modelId`
|
||||
* wins, then the connection's `defaultModel`. Returns `null` when neither is set —
|
||||
* a provider id is not a model name, so it is never used as a fallback.
|
||||
*/
|
||||
function resolveModelId(
|
||||
conn: { provider?: string; id?: string; defaultModel?: string | null },
|
||||
overrides: ChaosConfig["providerOverrides"]
|
||||
): string {
|
||||
): string | null {
|
||||
const override = overrides.find(
|
||||
(o) =>
|
||||
o.enabled &&
|
||||
(o.providerId.toLowerCase() === (conn.provider || "").toLowerCase() ||
|
||||
o.providerId.toLowerCase() === (conn.id || "").toLowerCase())
|
||||
);
|
||||
if (override?.modelId) return override.modelId;
|
||||
return conn.defaultModel || conn.provider || conn.id || "unknown";
|
||||
const overrideModel = override?.modelId?.trim();
|
||||
if (overrideModel) return overrideModel;
|
||||
return conn.defaultModel?.trim() || null;
|
||||
}
|
||||
|
||||
export const NO_MODEL_PARTICIPANT_ERROR =
|
||||
"No model set for this provider. Set a model for it on the Chaos Mode page or a default model on the connection.";
|
||||
export const NO_MODEL_RUN_ERROR =
|
||||
"No provider has a model set. Set a model for each provider on the Chaos Mode page or a default model on its connection.";
|
||||
|
||||
/** A participant that could not be dispatched because it has no model; nothing is sent upstream. */
|
||||
function buildNoModelResult(provider: ProviderInfo): ModelResult {
|
||||
return {
|
||||
providerId: provider.id,
|
||||
providerName: provider.name,
|
||||
modelId: "",
|
||||
status: "error",
|
||||
content: null,
|
||||
error: NO_MODEL_PARTICIPANT_ERROR,
|
||||
durationMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Internal helpers — provider selection ───────────────────────────────────
|
||||
@@ -293,8 +313,7 @@ async function dispatchToModel(
|
||||
const ctx = { providerId, providerName, modelId };
|
||||
|
||||
try {
|
||||
const model = modelId || providerId;
|
||||
const request = buildDispatchRequest(model, messages, maxTokens, timeoutMs, apiKey);
|
||||
const request = buildDispatchRequest(modelId, messages, maxTokens, timeoutMs, apiKey);
|
||||
const res = await chatDispatch.postChatCompletion(request);
|
||||
return await parseDispatchResponse(res, ctx, start);
|
||||
} catch (err: unknown) {
|
||||
@@ -344,8 +363,9 @@ async function dispatchParallel(
|
||||
overrides: ChaosConfig["providerOverrides"],
|
||||
params: EffectiveRunParams
|
||||
): Promise<ModelResult[]> {
|
||||
const tasks = providers.map((p) => () => {
|
||||
const tasks = providers.map((p) => async () => {
|
||||
const modelId = resolveModelId(p, overrides);
|
||||
if (!modelId) return buildNoModelResult(p);
|
||||
const messages = buildMessages(params.effectiveSystemPrompt, input.task);
|
||||
return dispatchToModel(
|
||||
p.id,
|
||||
@@ -381,6 +401,10 @@ async function dispatchCollaborative(
|
||||
|
||||
for (const p of providers) {
|
||||
const modelId = resolveModelId(p, overrides);
|
||||
if (!modelId) {
|
||||
results.push(buildNoModelResult(p));
|
||||
continue;
|
||||
}
|
||||
const messages = buildMessages(params.effectiveSystemPrompt, context);
|
||||
|
||||
const result = await dispatchToModel(
|
||||
@@ -424,6 +448,10 @@ export async function executeChaosRun(input: ChaosRunInput): Promise<ChaosRunRes
|
||||
const params = resolveEffectiveRunParams(input, globalConfig);
|
||||
const { providers, enabledOverrides } = await selectChaosProviders(input, globalConfig);
|
||||
|
||||
if (!providers.some((p) => resolveModelId(p, enabledOverrides))) {
|
||||
throw new Error(NO_MODEL_RUN_ERROR);
|
||||
}
|
||||
|
||||
const startedAt = new Date().toISOString();
|
||||
const results =
|
||||
params.mode === "parallel"
|
||||
|
||||
46
tests/_setup/vitestDataDir.ts
Normal file
46
tests/_setup/vitestDataDir.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
// Vitest: a throwaway DATA_DIR for the whole run.
|
||||
//
|
||||
// The node:test runner gets one from tests/_setup/isolateDataDir.ts, but Vitest never loaded an
|
||||
// equivalent. Code that resolves its own database path (the MCP audit did: DATA_DIR, else
|
||||
// ~/.omniroute/storage.sqlite) then wrote into the developer's real database on every
|
||||
// `npm run test:vitest` (202 audit rows of one test fixture had piled up in a developer's
|
||||
// database).
|
||||
//
|
||||
// Wiring, in both Vitest configs:
|
||||
// - `test.env: isolatedDataDirEnv()` hands the variables to every worker (a change to
|
||||
// process.env inside a global setup does not reach the worker threads);
|
||||
// - `test.globalSetup: ["./tests/_setup/vitestDataDir.ts"]` removes the directory at the end.
|
||||
// A DATA_DIR chosen by the caller always wins, and is never removed.
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
// Set only when this file created the directory, so teardown never deletes a caller's DATA_DIR.
|
||||
const CREATED_KEY = "OMNIROUTE_VITEST_DATA_DIR";
|
||||
|
||||
export function isolatedDataDirEnv(): Record<string, string> {
|
||||
if (!process.env[CREATED_KEY]) {
|
||||
if (process.env.DATA_DIR?.trim()) return {};
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-vitest-"));
|
||||
process.env[CREATED_KEY] = dir;
|
||||
process.env.DATA_DIR = dir;
|
||||
}
|
||||
const dir = process.env[CREATED_KEY] as string;
|
||||
return {
|
||||
DATA_DIR: dir,
|
||||
// Same guard as isolateDataDir.ts: the plugin scanner would otherwise read ~/.omniroute/plugins.
|
||||
OMNIROUTE_PLUGINS_DIR: process.env.OMNIROUTE_PLUGINS_DIR?.trim() || path.join(dir, "plugins"),
|
||||
};
|
||||
}
|
||||
|
||||
export default function setup(): () => void {
|
||||
return () => {
|
||||
const dir = process.env[CREATED_KEY];
|
||||
if (!dir) return;
|
||||
try {
|
||||
fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
} catch {
|
||||
// The OS reaps its temp dir eventually.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -223,6 +223,36 @@ test("POST /api/chaos/run — 200 happy path dispatches with internal Authorizat
|
||||
assert.equal(capturedAuth?.startsWith("Bearer "), true);
|
||||
});
|
||||
|
||||
test("POST /api/chaos/run — reports a clear error without any upstream request when no provider has a model", async () => {
|
||||
await chaosConfig.setChaosConfig({
|
||||
enabled: true,
|
||||
defaultMode: "parallel",
|
||||
providerOverrides: [],
|
||||
timeoutMs: 120_000,
|
||||
maxTokens: 4096,
|
||||
});
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "openai",
|
||||
authType: "apikey",
|
||||
name: "No model route provider",
|
||||
apiKey: "sk-run-route-no-model",
|
||||
});
|
||||
|
||||
let calls = 0;
|
||||
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async () => {
|
||||
calls++;
|
||||
return jsonResponse({ choices: [{ message: { content: "should not run" } }] });
|
||||
});
|
||||
|
||||
const res = await runRoute.POST(
|
||||
makeRequest("POST", "http://localhost/api/chaos/run", { task: "Summarize" })
|
||||
);
|
||||
assert.equal(res.status, 500);
|
||||
const body = (await res.json()) as { error: { message: string } };
|
||||
assert.match(body.error.message, /No provider has a model set/);
|
||||
assert.equal(calls, 0);
|
||||
});
|
||||
|
||||
// ═════════════════════════════════════════════════════════════════════════════
|
||||
// POST /api/skills/collect/chaos
|
||||
// ═════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
@@ -28,6 +28,7 @@ const chaosExecutor = await import("../../src/lib/chaos/chaosExecutor.ts");
|
||||
|
||||
async function resetStorage() {
|
||||
core.resetDbInstance();
|
||||
chaosConfig.invalidateChaosConfigCache();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
@@ -241,3 +242,209 @@ test("executeChaosRun applies global config provider overrides for model resolut
|
||||
|
||||
assert.equal(capturedModel, "gpt-4-turbo");
|
||||
});
|
||||
|
||||
const NO_MODEL_MESSAGE =
|
||||
"No model set for this provider. Set a model for it on the Chaos Mode page or a default model on the connection.";
|
||||
|
||||
async function setOverrides(
|
||||
providerOverrides: { providerId: string; modelId?: string; enabled: boolean }[]
|
||||
) {
|
||||
await chaosConfig.setChaosConfig({
|
||||
enabled: true,
|
||||
defaultMode: "parallel",
|
||||
providerOverrides,
|
||||
timeoutMs: 120_000,
|
||||
maxTokens: 4096,
|
||||
});
|
||||
}
|
||||
|
||||
test("executeChaosRun skips a provider without a model: no upstream request, clear error, others still run", async () => {
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "openai",
|
||||
authType: "apikey",
|
||||
name: "Has default model",
|
||||
apiKey: "sk-test-has-model",
|
||||
defaultModel: "gpt-4o-mini",
|
||||
});
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "anthropic",
|
||||
authType: "apikey",
|
||||
name: "No model",
|
||||
apiKey: "sk-test-no-model",
|
||||
});
|
||||
|
||||
const dispatchedModels: string[] = [];
|
||||
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => {
|
||||
const body = (await req.json()) as { model: string };
|
||||
dispatchedModels.push(body.model);
|
||||
return jsonResponse({ choices: [{ message: { content: "ok" } }] });
|
||||
});
|
||||
|
||||
const result = await chaosExecutor.executeChaosRun({ task: "task" });
|
||||
|
||||
assert.deepEqual(dispatchedModels, ["gpt-4o-mini"]);
|
||||
assert.equal(result.totalProviders, 2);
|
||||
assert.equal(result.models.length, 2);
|
||||
|
||||
const ok = result.models.find((m) => m.providerName === "Has default model");
|
||||
const skipped = result.models.find((m) => m.providerName === "No model");
|
||||
assert.equal(ok?.status, "success");
|
||||
assert.equal(skipped?.status, "error");
|
||||
assert.equal(skipped?.content, null);
|
||||
assert.equal(skipped?.error, NO_MODEL_MESSAGE);
|
||||
assert.equal(skipped?.modelId, "");
|
||||
});
|
||||
|
||||
test("executeChaosRun skips a provider without a model in collaborative mode and keeps the chain going", async () => {
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "anthropic",
|
||||
authType: "apikey",
|
||||
name: "No model",
|
||||
apiKey: "sk-test-collab-no-model",
|
||||
});
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "openai",
|
||||
authType: "apikey",
|
||||
name: "With model",
|
||||
apiKey: "sk-test-collab-model",
|
||||
defaultModel: "gpt-4o-mini",
|
||||
});
|
||||
|
||||
let calls = 0;
|
||||
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async () => {
|
||||
calls++;
|
||||
return jsonResponse({ choices: [{ message: { content: "chain-reply" } }] });
|
||||
});
|
||||
|
||||
const result = await chaosExecutor.executeChaosRun({ task: "task", mode: "collaborative" });
|
||||
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(result.models.length, 2);
|
||||
const skipped = result.models.find((m) => m.providerName === "No model");
|
||||
const ran = result.models.find((m) => m.providerName === "With model");
|
||||
assert.equal(skipped?.status, "error");
|
||||
assert.equal(skipped?.error, NO_MODEL_MESSAGE);
|
||||
assert.equal(ran?.status, "success");
|
||||
assert.equal(result.summary, "chain-reply");
|
||||
});
|
||||
|
||||
test("executeChaosRun: an enabled override modelId supplies the model for a connection without a default", async () => {
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "openai",
|
||||
authType: "apikey",
|
||||
name: "Override only",
|
||||
apiKey: "sk-test-override-only",
|
||||
});
|
||||
await setOverrides([{ providerId: "openai", modelId: "gpt-4-turbo", enabled: true }]);
|
||||
|
||||
const dispatchedModels: string[] = [];
|
||||
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => {
|
||||
dispatchedModels.push(((await req.json()) as { model: string }).model);
|
||||
return jsonResponse({ choices: [{ message: { content: "ok" } }] });
|
||||
});
|
||||
|
||||
const result = await chaosExecutor.executeChaosRun({ task: "task" });
|
||||
|
||||
assert.deepEqual(dispatchedModels, ["gpt-4-turbo"]);
|
||||
assert.equal(result.models[0].status, "success");
|
||||
});
|
||||
|
||||
test("executeChaosRun: a blank override modelId does not count as a model", async () => {
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "openai",
|
||||
authType: "apikey",
|
||||
name: "Blank override",
|
||||
apiKey: "sk-test-blank-override",
|
||||
});
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "anthropic",
|
||||
authType: "apikey",
|
||||
name: "Has default",
|
||||
apiKey: "sk-test-blank-override-2",
|
||||
defaultModel: "claude-3-5-sonnet",
|
||||
});
|
||||
await setOverrides([
|
||||
{ providerId: "openai", modelId: " ", enabled: true },
|
||||
{ providerId: "anthropic", enabled: true },
|
||||
]);
|
||||
|
||||
const dispatchedModels: string[] = [];
|
||||
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => {
|
||||
dispatchedModels.push(((await req.json()) as { model: string }).model);
|
||||
return jsonResponse({ choices: [{ message: { content: "ok" } }] });
|
||||
});
|
||||
|
||||
const result = await chaosExecutor.executeChaosRun({ task: "task" });
|
||||
|
||||
assert.deepEqual(dispatchedModels, ["claude-3-5-sonnet"]);
|
||||
const blank = result.models.find((m) => m.providerName === "Blank override");
|
||||
assert.equal(blank?.status, "error");
|
||||
assert.equal(blank?.error, NO_MODEL_MESSAGE);
|
||||
});
|
||||
|
||||
test("executeChaosRun: a disabled override modelId is ignored in favor of the connection default", async () => {
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "anthropic",
|
||||
authType: "apikey",
|
||||
name: "Disabled override",
|
||||
apiKey: "sk-test-disabled-override",
|
||||
defaultModel: "claude-3-5-sonnet",
|
||||
});
|
||||
await setOverrides([{ providerId: "anthropic", modelId: "ignored-model", enabled: false }]);
|
||||
|
||||
const dispatchedModels: string[] = [];
|
||||
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => {
|
||||
dispatchedModels.push(((await req.json()) as { model: string }).model);
|
||||
return jsonResponse({ choices: [{ message: { content: "ok" } }] });
|
||||
});
|
||||
|
||||
await chaosExecutor.executeChaosRun({ task: "task" });
|
||||
|
||||
assert.deepEqual(dispatchedModels, ["claude-3-5-sonnet"]);
|
||||
});
|
||||
|
||||
test("executeChaosRun: an override modelId wins over the connection default model", async () => {
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "openai",
|
||||
authType: "apikey",
|
||||
name: "Both",
|
||||
apiKey: "sk-test-both",
|
||||
defaultModel: "gpt-4o-mini",
|
||||
});
|
||||
await setOverrides([{ providerId: "openai", modelId: "gpt-4-turbo", enabled: true }]);
|
||||
|
||||
const dispatchedModels: string[] = [];
|
||||
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => {
|
||||
dispatchedModels.push(((await req.json()) as { model: string }).model);
|
||||
return jsonResponse({ choices: [{ message: { content: "ok" } }] });
|
||||
});
|
||||
|
||||
await chaosExecutor.executeChaosRun({ task: "task" });
|
||||
|
||||
assert.deepEqual(dispatchedModels, ["gpt-4-turbo"]);
|
||||
});
|
||||
|
||||
test("executeChaosRun fails fast without any upstream request when no provider has a model", async () => {
|
||||
for (const provider of ["openai", "anthropic", "gemini"]) {
|
||||
await providersDb.createProviderConnection({
|
||||
provider,
|
||||
authType: "apikey",
|
||||
name: `No model ${provider}`,
|
||||
apiKey: `sk-test-nomodel-${provider}`,
|
||||
});
|
||||
}
|
||||
|
||||
let calls = 0;
|
||||
mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async () => {
|
||||
calls++;
|
||||
return jsonResponse({ choices: [{ message: { content: "ok" } }] });
|
||||
});
|
||||
|
||||
for (const mode of ["parallel", "collaborative"] as const) {
|
||||
await assert.rejects(
|
||||
() => chaosExecutor.executeChaosRun({ task: "task", mode }),
|
||||
/No provider has a model set\. Set a model for each provider on the Chaos Mode page or a default model on its connection\./
|
||||
);
|
||||
}
|
||||
assert.equal(calls, 0);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import { afterEach, describe, it } from "node:test";
|
||||
import { after, afterEach, describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
buildChecks,
|
||||
runCountsGate,
|
||||
@@ -86,10 +89,18 @@ describe("readCodeFacts failure is reported as a strict failure", () => {
|
||||
// The gate itself no longer reports that failure as documentation drift: facts that cannot be
|
||||
// collected are an INFRASTRUCTURE failure (exit 2), so a host too loaded to run the tsx probe
|
||||
// can never be mistaken for docs that need editing.
|
||||
//
|
||||
// The gate runs against an empty root: a real STRICT drift still wins over the infrastructure
|
||||
// verdict (exit 1), and a feature branch that adds a migration or a provider carries such a
|
||||
// drift by design (derived values are refreshed at release). Against the working tree, these
|
||||
// assertions would depend on the branch they run on.
|
||||
const emptyRoot = fs.mkdtempSync(path.join(os.tmpdir(), "docs-counts-err-seam-"));
|
||||
after(() => fs.rmSync(emptyRoot, { recursive: true, force: true }));
|
||||
|
||||
describe("the release gate classifies a facts failure as infrastructure", () => {
|
||||
it("exits 2 with an infrastructure verdict, not a STRICT drift", () => {
|
||||
__setSpawnSyncForTest(failingProbe);
|
||||
const r = runCountsGate({ mode: "release" }) as GateResult;
|
||||
const r = runCountsGate({ mode: "release", root: emptyRoot }) as GateResult;
|
||||
assert.equal(r.mode, "release");
|
||||
assert.equal(r.exitCode, 2, `expected exit 2 (infrastructure), got ${r.exitCode}`);
|
||||
assert.ok(r.infra, "the result must carry the infrastructure failure");
|
||||
@@ -111,7 +122,7 @@ describe("the release gate classifies a facts failure as infrastructure", () =>
|
||||
signal: "SIGKILL",
|
||||
}) as never
|
||||
);
|
||||
const r = runCountsGate({ mode: "release" }) as GateResult;
|
||||
const r = runCountsGate({ mode: "release", root: emptyRoot }) as GateResult;
|
||||
assert.equal(r.exitCode, 2);
|
||||
assert.match(JSON.stringify(r.infra), /timeout/i);
|
||||
});
|
||||
|
||||
@@ -114,7 +114,7 @@ test("NVIDIA aliases every OpenAI tool-name surface and can restore the provider
|
||||
assert.notEqual(collisions.tools[0].function.name, collisions.tools[1].function.name);
|
||||
});
|
||||
|
||||
test("NVIDIA executor applies its registry-scoped tool-name limit without changing OpenAI", () => {
|
||||
test("NVIDIA executor applies its registry-scoped tool-name limit without changing a provider that has no limit", () => {
|
||||
const original = "mcp__plugin_chrome-devtools-mcp_chrome-devtools__get.console/message";
|
||||
const request = {
|
||||
tools: [{ type: "function", function: { name: original } }],
|
||||
@@ -134,7 +134,7 @@ test("NVIDIA executor applies its registry-scoped tool-name limit without changi
|
||||
false,
|
||||
null
|
||||
);
|
||||
const openai = new DefaultExecutor("openai").transformRequest(
|
||||
const openai = new DefaultExecutor("groq").transformRequest(
|
||||
"gpt-5.4",
|
||||
structuredClone(request),
|
||||
false,
|
||||
|
||||
196
tests/unit/openai-format-tool-name-max-length.test.ts
Normal file
196
tests/unit/openai-format-tool-name-max-length.test.ts
Normal file
@@ -0,0 +1,196 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
// Isolated DATA_DIR under the worktree's _artifacts/ (never the real ~/.omniroute).
|
||||
const ARTIFACTS_DIR = path.resolve(import.meta.dirname, "../../_artifacts");
|
||||
fs.mkdirSync(ARTIFACTS_DIR, { recursive: true });
|
||||
process.env.DATA_DIR = fs.mkdtempSync(path.join(ARTIFACTS_DIR, "openai-tool-name-limit-"));
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const { getRegistryEntry } = await import("../../open-sse/config/providerRegistry.ts");
|
||||
const { DefaultExecutor } = await import("../../open-sse/executors/default.ts");
|
||||
const { AzureOpenAIExecutor } = await import("../../open-sse/executors/azure-openai.ts");
|
||||
const { AzureAiExecutor } = await import("../../open-sse/executors/azure-ai.ts");
|
||||
const { resolveToolNameMaxLength } =
|
||||
await import("../../open-sse/executors/toolNameLengthAlias.ts");
|
||||
const { restoreOpenAIToolNames } =
|
||||
await import("../../open-sse/translator/helpers/toolCallHelper.ts");
|
||||
const { createSSEStream } = await import("../../open-sse/utils/stream.ts");
|
||||
const { FORMATS } = await import("../../open-sse/translator/formats.ts");
|
||||
|
||||
// OpenAI Chat Completions (and Azure OpenAI, which implements the same schema) reject a
|
||||
// function name over 64 chars. MCP tool names from VS Code / Claude Code routinely exceed
|
||||
// it (cheaper-inference/OmniCopilot#27).
|
||||
const LONG = "mcp__example-observability-prod__trace_top_time_consuming_operations_by_service"; // 78
|
||||
const SHORT = "mcp__github__get_issue";
|
||||
const EXACT_64 = "a".repeat(64);
|
||||
|
||||
type ToolCall = { id: string; type: string; function: { name: string; arguments: string } };
|
||||
type ChatBody = {
|
||||
model: string;
|
||||
tools: Array<{ type: string; function: { name: string; parameters: object } }>;
|
||||
tool_choice?: { type: string; function: { name: string } };
|
||||
messages: Array<{
|
||||
role: string;
|
||||
content: string | null;
|
||||
tool_calls?: ToolCall[];
|
||||
tool_call_id?: string;
|
||||
name?: string;
|
||||
}>;
|
||||
_toolNameMap?: Map<string, string>;
|
||||
};
|
||||
|
||||
function chatBody(toolName: string): ChatBody {
|
||||
return {
|
||||
model: "m",
|
||||
tools: [
|
||||
{ type: "function", function: { name: toolName, parameters: { type: "object" } } },
|
||||
{ type: "function", function: { name: SHORT, parameters: { type: "object" } } },
|
||||
],
|
||||
tool_choice: { type: "function", function: { name: toolName } },
|
||||
messages: [
|
||||
{ role: "user", content: "hi" },
|
||||
{
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [{ id: "c1", type: "function", function: { name: toolName, arguments: "{}" } }],
|
||||
},
|
||||
{ role: "tool", tool_call_id: "c1", name: toolName, content: "ok" },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const EXECUTORS: Record<string, () => InstanceType<typeof DefaultExecutor>> = {
|
||||
openai: () => new DefaultExecutor("openai"),
|
||||
"azure-openai": () => new AzureOpenAIExecutor(),
|
||||
"azure-ai": () => new AzureAiExecutor(),
|
||||
};
|
||||
|
||||
function transform(
|
||||
provider: string,
|
||||
toolName: string,
|
||||
providerSpecificData?: Record<string, unknown>
|
||||
): ChatBody {
|
||||
return EXECUTORS[provider]().transformRequest("m", chatBody(toolName), false, {
|
||||
apiKey: "k",
|
||||
providerSpecificData,
|
||||
}) as ChatBody;
|
||||
}
|
||||
|
||||
for (const provider of Object.keys(EXECUTORS)) {
|
||||
test(`${provider}: a ${LONG.length}-char tool name goes upstream at most 64 chars everywhere`, () => {
|
||||
assert.ok(LONG.length > 64);
|
||||
const out = transform(provider, LONG);
|
||||
const sent = out.tools[0].function.name;
|
||||
|
||||
assert.ok(sent.length <= 64, `tools[] name is ${sent.length} chars`);
|
||||
assert.match(sent, /^[A-Za-z0-9_-]{1,64}$/);
|
||||
assert.equal(out.tools[1].function.name, SHORT);
|
||||
assert.equal(out.tool_choice?.function.name, sent);
|
||||
assert.equal(out.messages[1].tool_calls?.[0].function.name, sent);
|
||||
assert.equal(out.messages[2].name, sent);
|
||||
assert.equal(out._toolNameMap?.get(sent), LONG);
|
||||
assert.equal(JSON.stringify(out).includes("_toolNameMap"), false);
|
||||
});
|
||||
|
||||
test(`${provider}: a non-streaming tool call comes back under its original name`, () => {
|
||||
const out = transform(provider, LONG);
|
||||
const sent = out.tools[0].function.name;
|
||||
const response = {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [{ id: "c2", type: "function", function: { name: sent, arguments: "{}" } }],
|
||||
},
|
||||
finish_reason: "tool_calls",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
assert.equal(restoreOpenAIToolNames(response, out._toolNameMap), true);
|
||||
assert.equal(response.choices[0].message.tool_calls[0].function.name, LONG);
|
||||
});
|
||||
|
||||
test(`${provider}: a streamed tool call comes back under its original name`, async () => {
|
||||
const out = transform(provider, LONG);
|
||||
const sent = out.tools[0].function.name;
|
||||
const chunk = {
|
||||
id: "chatcmpl-1",
|
||||
object: "chat.completion.chunk",
|
||||
created: 1,
|
||||
model: "m",
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
delta: {
|
||||
tool_calls: [
|
||||
{ index: 0, id: "c3", type: "function", function: { name: sent, arguments: "{}" } },
|
||||
],
|
||||
},
|
||||
finish_reason: null,
|
||||
},
|
||||
],
|
||||
};
|
||||
const source = new ReadableStream({
|
||||
start(controller) {
|
||||
const encoder = new TextEncoder();
|
||||
controller.enqueue(encoder.encode(`data: ${JSON.stringify(chunk)}\n\n`));
|
||||
controller.enqueue(encoder.encode("data: [DONE]\n\n"));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
const text = await new Response(
|
||||
source.pipeThrough(
|
||||
createSSEStream({
|
||||
mode: "passthrough",
|
||||
sourceFormat: FORMATS.OPENAI,
|
||||
provider,
|
||||
model: "m",
|
||||
toolNameMap: out._toolNameMap,
|
||||
body: { messages: [{ role: "user", content: "hi" }] },
|
||||
})
|
||||
)
|
||||
).text();
|
||||
|
||||
assert.ok(text.includes(JSON.stringify(LONG)), "original name missing from the stream");
|
||||
assert.equal(text.includes(JSON.stringify(sent)), false, "alias leaked to the client");
|
||||
});
|
||||
|
||||
test(`${provider}: names of 64 chars or fewer are untouched`, () => {
|
||||
for (const name of [EXACT_64, SHORT]) {
|
||||
const out = transform(provider, name);
|
||||
assert.equal(out.tools[0].function.name, name);
|
||||
assert.equal(out.messages[1].tool_calls?.[0].function.name, name);
|
||||
assert.equal(out._toolNameMap, undefined);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test("openai declares toolNameMaxLength 64 in the registry", () => {
|
||||
assert.equal(getRegistryEntry("openai")?.toolNameMaxLength, 64);
|
||||
});
|
||||
|
||||
test("azure providers have no registry entry, so their 64 is a built-in limit", () => {
|
||||
assert.equal(getRegistryEntry("azure-openai"), null);
|
||||
assert.equal(getRegistryEntry("azure-ai"), null);
|
||||
assert.equal(resolveToolNameMaxLength("azure-openai", undefined), 64);
|
||||
assert.equal(resolveToolNameMaxLength("azure-ai", undefined), 64);
|
||||
});
|
||||
|
||||
test("a connection toolNameMaxLength still applies where the registry declares none", () => {
|
||||
// Azure: a stricter gateway in front of the deployment lowers the built-in 64.
|
||||
const azure = transform("azure-openai", "x".repeat(50), { toolNameMaxLength: 40 });
|
||||
assert.ok(azure.tools[0].function.name.length <= 40);
|
||||
// A provider with neither a registry nor a built-in limit keeps honoring the connection.
|
||||
assert.equal(resolveToolNameMaxLength("openai-compatible-gw", { toolNameMaxLength: 48 }), 48);
|
||||
assert.equal(resolveToolNameMaxLength("openai-compatible-gw", undefined), undefined);
|
||||
});
|
||||
|
||||
test.after(() => {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(process.env.DATA_DIR as string, { recursive: true, force: true });
|
||||
});
|
||||
@@ -210,7 +210,7 @@ test("applyToolNameLengthAliases accepts a custom normalizer and ignores invalid
|
||||
|
||||
test("applyRegistryToolNameLengthAliases is a no-op for providers without a limit", () => {
|
||||
const body: Record<string, unknown> = chatBody([LONG_A]);
|
||||
assert.equal(applyRegistryToolNameLengthAliases("openai", body).size, 0);
|
||||
assert.equal(applyRegistryToolNameLengthAliases("groq", body).size, 0);
|
||||
assert.equal((body as unknown as ChatBody).tools[0].function.name, LONG_A);
|
||||
assert.equal(body._toolNameMap, undefined);
|
||||
assert.equal(applyRegistryToolNameLengthAliases("nvidia", body).size, 1);
|
||||
|
||||
@@ -30,6 +30,7 @@ const strip = (await import("../../scripts/ad-hoc/strip-derived-from-pr.mjs")) a
|
||||
ctx: Ctx;
|
||||
}) => { files: PlannedFile[]; manual: string[] };
|
||||
loadContext: () => Promise<Ctx>;
|
||||
readBlob: (ref: string, file: string, opts?: { cwd?: string }) => string | null;
|
||||
buildRestoreCommit: (args: {
|
||||
cwd: string;
|
||||
head: string;
|
||||
@@ -83,6 +84,28 @@ test("claims: each value the PR changed goes back to the base value; prose edits
|
||||
assert.deepEqual(r.manual, []);
|
||||
});
|
||||
|
||||
test("claims: the base's own digits come back, not a re-rendered number", () => {
|
||||
// A translated mirror writes the count in Persian digits; the head rewrote it in ASCII.
|
||||
const toAscii = (t: string) => t.replace(/[۰-۹]/g, (d) => String("۰۱۲۳۴۵۶۷۸۹".indexOf(d)));
|
||||
const nativeCheck: Check = {
|
||||
label: "DB migrations count (README mirrors)",
|
||||
extract: (content) =>
|
||||
[...content.matchAll(/([0-9۰-۹]+) مهاجرت/dg)].map((m) => ({
|
||||
value: Number(toAscii(m[1])),
|
||||
start: (m as unknown as { indices: [number, number][] }).indices[1][0],
|
||||
end: (m as unknown as { indices: [number, number][] }).indices[1][1],
|
||||
})),
|
||||
};
|
||||
const base = "<td>۱۴۷ ماژول، ۲۳۳ مهاجرت</td>\n";
|
||||
const head = "<td>۱۴۷ ماژول، 234 مهاجرت</td>\n";
|
||||
const r = strip.restoreClaims(base, head, [nativeCheck]);
|
||||
assert.equal(r.content, base, "byte-identical to the base line");
|
||||
assert.deepEqual(r.edits, [
|
||||
{ label: "DB migrations count (README mirrors)", from: "234", to: "233" },
|
||||
]);
|
||||
assert.deepEqual(r.manual, []);
|
||||
});
|
||||
|
||||
test("claims: a PR that adds or removes a claim is left to a human", () => {
|
||||
const r = strip.restoreClaims("387 providers.\n", "388 providers. Also 388 providers.\n", [
|
||||
providerCheck,
|
||||
@@ -210,3 +233,53 @@ test("buildRestoreCommit: one commit on top of the head, only the planned files
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("restore from real git blobs is byte-exact: the final newline survives", () => {
|
||||
// Regression: contents read through a trimming helper lost the last "\n", so the "restored"
|
||||
// file still differed from the merge base and the PR stayed conflicting on it.
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "strip-bytes-"));
|
||||
const git = (...args: string[]) =>
|
||||
execFileSync("git", args, { cwd: dir, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
|
||||
const write = (file: string, content: string) => fs.writeFileSync(path.join(dir, file), content);
|
||||
try {
|
||||
git("init", "-q");
|
||||
git("config", "user.name", "test");
|
||||
git("config", "user.email", "test@example.invalid");
|
||||
write("stryker.conf.json", '{\n "testFiles": ["a"]\n}\n');
|
||||
write("AGENTS.md", "# Agents\nOne endpoint, 387 providers.\n");
|
||||
git("add", "-A");
|
||||
git("commit", "-q", "-m", "merge base");
|
||||
const base = git("rev-parse", "HEAD").trim();
|
||||
write("stryker.conf.json", '{\n "testFiles": ["a", "b"]\n}\n');
|
||||
write("AGENTS.md", "# Agents\nOne endpoint, 388 providers.\nNew rule.\n");
|
||||
git("commit", "-q", "-am", "PR head");
|
||||
const head = git("rev-parse", "HEAD").trim();
|
||||
|
||||
assert.equal(
|
||||
strip.readBlob(base, "AGENTS.md", { cwd: dir }),
|
||||
"# Agents\nOne endpoint, 387 providers.\n"
|
||||
);
|
||||
assert.equal(strip.readBlob(base, "missing.txt", { cwd: dir }), null);
|
||||
|
||||
const plan = strip.planPrRestore({
|
||||
changed: ["stryker.conf.json", "AGENTS.md"],
|
||||
readBase: (f) => strip.readBlob(base, f, { cwd: dir }),
|
||||
readHead: (f) => strip.readBlob(head, f, { cwd: dir }),
|
||||
ctx,
|
||||
});
|
||||
const commit = strip.buildRestoreCommit({ cwd: dir, head, files: plan.files }).trim();
|
||||
assert.equal(
|
||||
git("rev-parse", `${commit}:stryker.conf.json`),
|
||||
git("rev-parse", `${base}:stryker.conf.json`),
|
||||
"the restored blob is the merge-base blob, byte for byte"
|
||||
);
|
||||
assert.equal(git("diff", "--name-only", base, commit, "--", "stryker.conf.json"), "");
|
||||
assert.equal(
|
||||
git("show", `${commit}:AGENTS.md`),
|
||||
"# Agents\nOne endpoint, 387 providers.\nNew rule.\n",
|
||||
"digit-only revert keeps the PR's prose and the final newline"
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
import react from "@vitejs/plugin-react";
|
||||
import path from "path";
|
||||
import { isolatedDataDirEnv } from "./tests/_setup/vitestDataDir";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
@@ -10,6 +11,9 @@ export default defineConfig({
|
||||
testTimeout: 20000,
|
||||
globals: true,
|
||||
setupFiles: ["./tests/_setup/vitestUiPolyfills.ts"],
|
||||
// A throwaway DATA_DIR for the run, so no suite can reach the developer's ~/.omniroute.
|
||||
env: isolatedDataDirEnv(),
|
||||
globalSetup: ["./tests/_setup/vitestDataDir.ts"],
|
||||
pool: "threads",
|
||||
maxWorkers: 20,
|
||||
fileParallelism: true,
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
import path from "path";
|
||||
import { isolatedDataDirEnv } from "./tests/_setup/vitestDataDir";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
environment: "node",
|
||||
globals: true,
|
||||
// A throwaway DATA_DIR for the run, so no suite can reach the developer's ~/.omniroute.
|
||||
env: isolatedDataDirEnv(),
|
||||
globalSetup: ["./tests/_setup/vitestDataDir.ts"],
|
||||
pool: "threads",
|
||||
// MCP suites initialize isolated SQLite databases; Vitest 5 at 20 workers
|
||||
// makes native migration/close operations contend and flakes audit tests.
|
||||
|
||||
Reference in New Issue
Block a user