mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-16 17:09:09 +00:00
Release v3.8.52 — 1,442 documented changes (135 features, 1,112 fixes, 195 maintenance) from 1,386 cycle commits and 306 external contributors; full changelog in CHANGELOG.md → [3.8.52]. Local pre-flight on c754707d: check:release-green --quick --full-ci RELEASE-GREEN (0 HARD / 0 drift), webpack production build + build:cli OK; the release-PR ci.yml run was never dispatched (Actions FIFO backlog), merged with --admin by owner decision at STOP #1. ⚠️ base-red inherited: #15306 (i18n translation drift pinned by 3 unit tests + Docs Sync strict — translation backlog, not code).
461 lines
18 KiB
TypeScript
461 lines
18 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
|
|
const TEST_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-copilot-auto-"));
|
|
const TEST_DATA_DIR = path.join(TEST_ROOT, "data");
|
|
const TEST_PLUGINS_DIR = path.join(TEST_ROOT, "plugins");
|
|
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
|
|
const ORIGINAL_PLUGINS_DIR = process.env.OMNIROUTE_PLUGINS_DIR;
|
|
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
|
fs.mkdirSync(TEST_PLUGINS_DIR, { recursive: true });
|
|
process.env.DATA_DIR = TEST_DATA_DIR;
|
|
process.env.OMNIROUTE_PLUGINS_DIR = TEST_PLUGINS_DIR;
|
|
|
|
const core = await import("../../src/lib/db/core.ts");
|
|
const { getCallLogs, getCallLogById, waitForCallLogSaves } =
|
|
await import("../../src/lib/usage/callLogs.ts");
|
|
const { clearPendingRequests } = await import("../../src/lib/usage/usageHistory.ts");
|
|
const { handleChatCore } = await import("../../open-sse/handlers/chatCore.ts");
|
|
const { GithubExecutor } = await import("../../open-sse/executors/github.ts");
|
|
const { getModelsByProviderId } = await import("../../open-sse/config/providerModels.ts");
|
|
const { GITHUB_COPILOT_STATIC_FALLBACK_MODELS } =
|
|
await import("../../open-sse/services/githubCopilotModels.ts");
|
|
const { resolveCopilotAuto } = await import("../../open-sse/handlers/chatCore/copilotAutoModel.ts");
|
|
const {
|
|
COPILOT_AUTO_API_VERSION,
|
|
COPILOT_AUTO_REFRESH_WINDOW_MS,
|
|
CopilotAutoSessionCache,
|
|
copilotAutoSessions,
|
|
extractAutoPrompt,
|
|
hasAutoImageInput,
|
|
parseAutoDecision,
|
|
} = await import("../../open-sse/executors/github/autoModel.ts");
|
|
|
|
type Captured = {
|
|
url: string;
|
|
method: string;
|
|
headers: Record<string, string>;
|
|
body: Record<string, unknown> | null;
|
|
};
|
|
|
|
const originalFetch = globalThis.fetch;
|
|
const AUTO_URL = "https://api.githubcopilot.com/auto";
|
|
const IMAGE_PART = { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw0KGgo=" } };
|
|
|
|
function freshCredentials(overrides: Record<string, unknown> = {}) {
|
|
return {
|
|
accessToken: "gho-access",
|
|
copilotToken: "copilot-live-token",
|
|
copilotTokenExpiresAt: Math.floor(Date.now() / 1000) + 3600,
|
|
connectionId: "conn-auto-1",
|
|
providerSpecificData: {},
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function autoDecision(model: string, vision = false, sessionToken = `session-${model}`) {
|
|
return new Response(
|
|
JSON.stringify({
|
|
selected_model: { id: model, capabilities: { supports: { vision } } },
|
|
session_token: sessionToken,
|
|
expires_at: Math.floor(Date.now() / 1000) + 3600,
|
|
}),
|
|
{ status: 200, headers: { "Content-Type": "application/json" } }
|
|
);
|
|
}
|
|
|
|
function toPlainHeaders(headers: HeadersInit | undefined): Record<string, string> {
|
|
if (!headers) return {};
|
|
if (headers instanceof Headers) return Object.fromEntries(headers.entries());
|
|
return Object.fromEntries(
|
|
Object.entries(headers as Record<string, unknown>).map(([k, v]) => [k, String(v)])
|
|
);
|
|
}
|
|
|
|
function mockFetch(respond: (call: Captured, calls: Captured[]) => Response) {
|
|
const calls: Captured[] = [];
|
|
globalThis.fetch = (async (url: string | URL, init: RequestInit = {}) => {
|
|
const call: Captured = {
|
|
url: String(url),
|
|
method: init.method || "GET",
|
|
headers: toPlainHeaders(init.headers),
|
|
body: init.body ? JSON.parse(String(init.body)) : null,
|
|
};
|
|
calls.push(call);
|
|
return respond(call, calls);
|
|
}) as typeof fetch;
|
|
return calls;
|
|
}
|
|
|
|
function headerValue(headers: Record<string, string>, name: string): string | undefined {
|
|
const key = Object.keys(headers).find((k) => k.toLowerCase() === name.toLowerCase());
|
|
return key ? headers[key] : undefined;
|
|
}
|
|
|
|
const noopLog = { debug() {}, info() {}, warn() {}, error() {} };
|
|
|
|
test.afterEach(() => {
|
|
globalThis.fetch = originalFetch;
|
|
copilotAutoSessions.clear();
|
|
clearPendingRequests();
|
|
});
|
|
|
|
test.after(() => {
|
|
globalThis.fetch = originalFetch;
|
|
core.resetDbInstance();
|
|
if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
|
|
else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
|
|
if (ORIGINAL_PLUGINS_DIR === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR;
|
|
else process.env.OMNIROUTE_PLUGINS_DIR = ORIGINAL_PLUGINS_DIR;
|
|
fs.rmSync(TEST_ROOT, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
});
|
|
|
|
test("github registry and static fallback expose the Copilot auto model", () => {
|
|
const auto = getModelsByProviderId("github").find((model) => model.id === "auto");
|
|
assert.ok(auto, "github/auto must be listed");
|
|
assert.equal(auto.supportsVision, undefined, "vision is resolved per request, never assumed");
|
|
assert.ok(GITHUB_COPILOT_STATIC_FALLBACK_MODELS.includes("auto"));
|
|
});
|
|
|
|
test("extractAutoPrompt reads the latest user-typed text across client formats", () => {
|
|
assert.equal(
|
|
extractAutoPrompt({
|
|
messages: [
|
|
{ role: "user", content: "first" },
|
|
{ role: "assistant", content: "ok" },
|
|
{ role: "user", content: " latest question " },
|
|
],
|
|
}),
|
|
"latest question"
|
|
);
|
|
assert.equal(
|
|
extractAutoPrompt({
|
|
messages: [{ role: "user", content: [{ type: "text", text: "a" }, IMAGE_PART] }],
|
|
}),
|
|
"a"
|
|
);
|
|
// Claude agentic loop: the last user turn only carries a tool_result.
|
|
assert.equal(
|
|
extractAutoPrompt({
|
|
messages: [
|
|
{ role: "user", content: [{ type: "text", text: "refactor the parser" }] },
|
|
{ role: "assistant", content: [{ type: "tool_use", id: "t1", name: "Read", input: {} }] },
|
|
{ role: "user", content: [{ type: "tool_result", tool_use_id: "t1", content: "..." }] },
|
|
],
|
|
}),
|
|
"refactor the parser"
|
|
);
|
|
assert.equal(extractAutoPrompt({ input: "responses prompt" }), "responses prompt");
|
|
assert.equal(
|
|
extractAutoPrompt({
|
|
input: [{ role: "user", content: [{ type: "input_text", text: "from items" }] }],
|
|
}),
|
|
"from items"
|
|
);
|
|
assert.equal(extractAutoPrompt({ messages: [] }), "");
|
|
});
|
|
|
|
test("hasAutoImageInput detects image parts in messages and Responses input", () => {
|
|
assert.equal(hasAutoImageInput({ messages: [{ role: "user", content: "hi" }] }), false);
|
|
assert.equal(hasAutoImageInput({ messages: [{ role: "user", content: [IMAGE_PART] }] }), true);
|
|
});
|
|
|
|
test("parseAutoDecision requires a model id and session token and normalizes expiry", () => {
|
|
const now = 1_800_000_000_000;
|
|
assert.deepEqual(
|
|
parseAutoDecision(
|
|
{
|
|
selected_model: { id: "gpt-5-mini", capabilities: { supports: { vision: true } } },
|
|
session_token: "tok",
|
|
expires_at: 1_800_003_600,
|
|
},
|
|
now
|
|
),
|
|
{
|
|
model: "gpt-5-mini",
|
|
sessionToken: "tok",
|
|
expiresAtMs: 1_800_003_600_000,
|
|
supportsVision: true,
|
|
}
|
|
);
|
|
assert.equal(parseAutoDecision({ selected_model: { id: "x" } }, now), null);
|
|
assert.equal(parseAutoDecision({ session_token: "tok" }, now), null);
|
|
assert.equal(parseAutoDecision(null, now), null);
|
|
});
|
|
|
|
test("CopilotAutoSessionCache is bounded and honours the refresh window", () => {
|
|
const now = Date.now();
|
|
const cache = new CopilotAutoSessionCache(2);
|
|
const decision = (model: string, ttlMs = 3_600_000) => ({
|
|
model,
|
|
sessionToken: model,
|
|
expiresAtMs: now + ttlMs,
|
|
supportsVision: false,
|
|
});
|
|
cache.set("a", decision("a"), now);
|
|
cache.set("b", decision("b"), now);
|
|
cache.set("c", decision("c"), now);
|
|
assert.equal(cache.size, 2, "oldest entry evicted past the bound");
|
|
assert.equal(cache.getUsable("a", false, now), null);
|
|
assert.equal(cache.getUsable("c", false, now)?.model, "c");
|
|
assert.equal(cache.getUsable("c", true, now), null, "image turn re-resolves a non-vision pick");
|
|
cache.set("near", decision("near", COPILOT_AUTO_REFRESH_WINDOW_MS - 1), now);
|
|
assert.equal(cache.getUsable("near", false, now), null, "inside refresh window → re-resolve");
|
|
});
|
|
|
|
test("resolveAutoModel POSTs the prompt to /auto and caches the decision per session", async () => {
|
|
const calls = mockFetch((call) =>
|
|
call.url === AUTO_URL ? autoDecision("gpt-5-mini") : new Response("unexpected", { status: 500 })
|
|
);
|
|
const executor = new GithubExecutor();
|
|
const body = { messages: [{ role: "user", content: "pick a model for me" }] };
|
|
|
|
const first = await executor.resolveAutoModel({
|
|
body,
|
|
credentials: freshCredentials(),
|
|
sessionKey: "conversation-1",
|
|
});
|
|
assert.equal(first.ok, true);
|
|
assert.equal(first.ok && first.decision.model, "gpt-5-mini");
|
|
assert.equal(first.ok && first.decision.sessionToken, "session-gpt-5-mini");
|
|
assert.equal(calls.length, 1);
|
|
assert.equal(calls[0].url, AUTO_URL);
|
|
assert.equal(calls[0].method, "POST");
|
|
assert.deepEqual(calls[0].body, { prompt: "pick a model for me" });
|
|
assert.equal(headerValue(calls[0].headers, "x-github-api-version"), COPILOT_AUTO_API_VERSION);
|
|
assert.equal(headerValue(calls[0].headers, "authorization"), "Bearer copilot-live-token");
|
|
|
|
const second = await executor.resolveAutoModel({
|
|
body,
|
|
credentials: freshCredentials(),
|
|
sessionKey: "conversation-1",
|
|
});
|
|
assert.equal(second.ok && second.cached, true);
|
|
assert.equal(calls.length, 1, "same account + session reuses the cached decision");
|
|
|
|
await executor.resolveAutoModel({
|
|
body,
|
|
credentials: freshCredentials(),
|
|
sessionKey: "conversation-2",
|
|
});
|
|
assert.equal(calls.length, 2, "another conversation resolves on its own");
|
|
});
|
|
|
|
test("resolveAutoModel re-resolves with has_image when the cached pick lacks vision", async () => {
|
|
let n = 0;
|
|
const calls = mockFetch(() =>
|
|
n++ === 0 ? autoDecision("o3-mini") : autoDecision("gpt-5", true)
|
|
);
|
|
const executor = new GithubExecutor();
|
|
const credentials = freshCredentials();
|
|
await executor.resolveAutoModel({
|
|
body: { messages: [{ role: "user", content: "text only" }] },
|
|
credentials,
|
|
sessionKey: "s",
|
|
});
|
|
const withImage = await executor.resolveAutoModel({
|
|
body: {
|
|
messages: [{ role: "user", content: [{ type: "text", text: "describe" }, IMAGE_PART] }],
|
|
},
|
|
credentials,
|
|
sessionKey: "s",
|
|
});
|
|
assert.equal(calls.length, 2);
|
|
assert.deepEqual(calls[1].body, { prompt: "describe", has_image: true });
|
|
assert.equal(withImage.ok && withImage.decision.model, "gpt-5");
|
|
});
|
|
|
|
test("buildHeaders sends Copilot-Session-Token on the auto API version only when present", () => {
|
|
const executor = new GithubExecutor();
|
|
const plain = executor.buildHeaders(freshCredentials(), true, null, "gpt-5-mini");
|
|
assert.equal(plain["Copilot-Session-Token"], undefined);
|
|
|
|
const auto = executor.buildHeaders(
|
|
freshCredentials({ copilotSessionToken: "sess-123" }),
|
|
true,
|
|
null,
|
|
"gpt-5-mini"
|
|
);
|
|
assert.equal(auto["Copilot-Session-Token"], "sess-123");
|
|
assert.equal(auto["x-github-api-version"], COPILOT_AUTO_API_VERSION);
|
|
});
|
|
|
|
test("resolveCopilotAuto leaves non-auto and non-github requests untouched", async () => {
|
|
const calls = mockFetch(() => new Response("{}", { status: 200 }));
|
|
const credentials = freshCredentials();
|
|
const viaGithub = await resolveCopilotAuto("github", "gpt-5-mini", {}, credentials);
|
|
assert.equal(viaGithub.model, "gpt-5-mini");
|
|
assert.equal(viaGithub.credentials, credentials);
|
|
const ghe = await resolveCopilotAuto("ghe-copilot", "auto", {}, credentials);
|
|
assert.equal(ghe.model, "auto", "ghe-copilot does not opt into auto");
|
|
assert.equal(calls.length, 0);
|
|
});
|
|
|
|
test("resolveCopilotAuto swaps model, body and a credentials copy without mutating input", async () => {
|
|
mockFetch(() => autoDecision("claude-sonnet-5"));
|
|
const credentials = freshCredentials();
|
|
const body: Record<string, unknown> = {
|
|
model: "github/auto",
|
|
messages: [{ role: "user", content: "hello" }],
|
|
};
|
|
const outcome = await resolveCopilotAuto("github", "auto", body, credentials, "sess");
|
|
assert.equal(outcome.errorResult, undefined);
|
|
assert.equal(outcome.model, "claude-sonnet-5");
|
|
assert.equal(body.model, "claude-sonnet-5");
|
|
assert.equal(outcome.credentials.copilotSessionToken, "session-claude-sonnet-5");
|
|
assert.equal(
|
|
(credentials as Record<string, unknown>).copilotSessionToken,
|
|
undefined,
|
|
"the caller's credentials object must not carry the per-request token"
|
|
);
|
|
});
|
|
|
|
test("resolveCopilotAuto surfaces an /auto 4xx as a sanitized error result", async () => {
|
|
mockFetch(
|
|
() =>
|
|
new Response(JSON.stringify({ error: { message: "auto is not enabled" } }), { status: 403 })
|
|
);
|
|
const outcome = await resolveCopilotAuto(
|
|
"github",
|
|
"auto",
|
|
{ messages: [{ role: "user", content: "x" }] },
|
|
freshCredentials()
|
|
);
|
|
assert.ok(outcome.errorResult);
|
|
assert.equal(outcome.errorResult.status, 403);
|
|
assert.equal(outcome.model, "auto");
|
|
const payload = await outcome.errorResult.response.json();
|
|
assert.match(payload.error.message, /auto model selection failed \(403\)/);
|
|
assert.ok(!payload.error.message.includes("at /"), "no stack trace in the error body");
|
|
});
|
|
|
|
test("resolveCopilotAuto mints a fresh Copilot token for /auto when the stored one is stale", async () => {
|
|
const calls = mockFetch((call) => {
|
|
if (call.url.startsWith("https://api.github.com/copilot_internal/v2/token")) {
|
|
return new Response(
|
|
JSON.stringify({
|
|
token: "copilot-minted",
|
|
expires_at: Math.floor(Date.now() / 1000) + 3600,
|
|
}),
|
|
{ status: 200 }
|
|
);
|
|
}
|
|
return autoDecision("gpt-5-mini");
|
|
});
|
|
const stale = freshCredentials({ copilotToken: undefined, copilotTokenExpiresAt: undefined });
|
|
const outcome = await resolveCopilotAuto(
|
|
"github",
|
|
"auto",
|
|
{ messages: [{ role: "user", content: "x" }] },
|
|
stale
|
|
);
|
|
assert.equal(outcome.model, "gpt-5-mini");
|
|
// The minted token is only used for /auto: the request credentials stay stale so
|
|
// BaseExecutor.execute() runs its own refresh and persists the token as before.
|
|
assert.equal(outcome.credentials.copilotToken, undefined);
|
|
assert.equal(outcome.credentials.copilotSessionToken, "session-gpt-5-mini");
|
|
const autoCall = calls.find((call) => call.url === AUTO_URL);
|
|
assert.equal(headerValue(autoCall?.headers || {}, "authorization"), "Bearer copilot-minted");
|
|
});
|
|
|
|
async function runChatCore(body: Record<string, unknown>, respond: (call: Captured) => Response) {
|
|
const calls = mockFetch(respond);
|
|
const result = await handleChatCore({
|
|
body: structuredClone(body),
|
|
modelInfo: { provider: "github", model: "auto", extendedContext: false },
|
|
credentials: freshCredentials(),
|
|
log: noopLog,
|
|
clientRawRequest: {
|
|
endpoint: "/v1/chat/completions",
|
|
body: structuredClone(body),
|
|
headers: new Headers({ accept: "application/json" }),
|
|
},
|
|
connectionId: "conn-auto-1",
|
|
userAgent: "unit-test",
|
|
sessionAffinityKey: "chatcore-session",
|
|
} as unknown as Parameters<typeof handleChatCore>[0]);
|
|
for (let i = 0; i < 5; i++) await new Promise((resolve) => setImmediate(resolve));
|
|
return { calls, result };
|
|
}
|
|
|
|
test("chatCore routes github/auto to the selected model and records usage under it", async () => {
|
|
const { calls, result } = await runChatCore(
|
|
{
|
|
model: "github/auto",
|
|
stream: false,
|
|
messages: [{ role: "user", content: "which model am I talking to?" }],
|
|
},
|
|
(call) => {
|
|
if (call.url === AUTO_URL) return autoDecision("gpt-5-mini");
|
|
return new Response(
|
|
JSON.stringify({
|
|
id: "chatcmpl-auto",
|
|
object: "chat.completion",
|
|
model: "gpt-5-mini",
|
|
choices: [
|
|
{ index: 0, message: { role: "assistant", content: "hi" }, finish_reason: "stop" },
|
|
],
|
|
usage: { prompt_tokens: 5, completion_tokens: 1, total_tokens: 6 },
|
|
}),
|
|
{ status: 200, headers: { "Content-Type": "application/json" } }
|
|
);
|
|
}
|
|
);
|
|
assert.equal(result.success, true);
|
|
assert.equal(calls[0].url, AUTO_URL);
|
|
const chat = calls.find((call) => call.url.endsWith("/chat/completions"));
|
|
assert.ok(chat, "chat request must follow the /auto decision");
|
|
assert.equal(chat.body?.model, "gpt-5-mini");
|
|
assert.equal(headerValue(chat.headers, "copilot-session-token"), "session-gpt-5-mini");
|
|
|
|
// Call-log writes are fire-and-forget; poll (30s ceiling) so a loaded CI worker
|
|
// does not race the assertion. Green runs return on the first saved row.
|
|
let log: Awaited<ReturnType<typeof getCallLogById>> | null = null;
|
|
for (const startedAt = Date.now(); !log && Date.now() - startedAt < 30_000;) {
|
|
await waitForCallLogSaves(5000);
|
|
const rows = await getCallLogs({ limit: 5 });
|
|
log = rows.length > 0 ? await getCallLogById(rows[0].id) : null;
|
|
if (!log) await new Promise((resolve) => setTimeout(resolve, 100));
|
|
}
|
|
assert.ok(log, "the request must be persisted in call logs");
|
|
assert.equal(log.model, "gpt-5-mini", "usage is attributed to the model that served it");
|
|
assert.match(String(log.requestedModel), /auto/, "the client's requested model stays auto");
|
|
});
|
|
|
|
test("chatCore sends a Claude auto pick through Copilot's /v1/messages shim in Anthropic shape", async () => {
|
|
const { calls, result } = await runChatCore(
|
|
{
|
|
model: "github/auto",
|
|
stream: false,
|
|
messages: [{ role: "user", content: "write a haiku" }],
|
|
},
|
|
(call) => {
|
|
if (call.url === AUTO_URL) return autoDecision("claude-sonnet-5", true);
|
|
return new Response(
|
|
JSON.stringify({
|
|
id: "msg_auto",
|
|
type: "message",
|
|
role: "assistant",
|
|
model: "claude-sonnet-5",
|
|
content: [{ type: "text", text: "ok" }],
|
|
stop_reason: "end_turn",
|
|
usage: { input_tokens: 5, output_tokens: 1 },
|
|
}),
|
|
{ status: 200, headers: { "Content-Type": "application/json" } }
|
|
);
|
|
}
|
|
);
|
|
assert.equal(result.success, true);
|
|
const upstream = calls.find((call) => call.url !== AUTO_URL);
|
|
assert.equal(upstream?.url, "https://api.githubcopilot.com/v1/messages");
|
|
assert.equal(upstream?.body?.model, "claude-sonnet-5");
|
|
assert.ok(Array.isArray(upstream?.body?.messages));
|
|
assert.equal(
|
|
headerValue(upstream?.headers || {}, "copilot-session-token"),
|
|
"session-claude-sonnet-5"
|
|
);
|
|
});
|