Files
OmniRoute/scripts/cli/smoke-cli-integrations.mjs
Diego Rodrigues de Sa e Souza bf2b39c626 test(cli): isolate real-binary smoke contracts (#14382)
Merge-batch 2026-09-23 (PRs do mantenedor, 99 triadas por ⭐; esta entrou pela Trilha A).

Validação única na árvore combinada das 49 PRs da trilha sobre `release/v3.8.51` @18bbb101:
- `typecheck:core`: só o erro herdado `cliproxyAccountHealth.ts:157` (TS2322, não desta leva)
- `check:open-sse-typecheck`: 0 erros (o auggie.ts da base é corrigido pela #14511 do mesmo lote)
- eslint nos arquivos alterados: limpo
- 89 arquivos de teste node focados (710 testes) + 11 suítes vitest de UI: todo vermelho restante reproduz também no tip puro (base-red #14547/#14496) e a árvore combinada fica com um subconjunto estrito dos vermelhos do tip, ou seja, nenhum introduzido pelo lote
- teto de file-size acertado na PR de acompanhamento da leva
2026-09-23 21:50:38 -03:00

493 lines
15 KiB
JavaScript

#!/usr/bin/env node
/**
* Exercise real third-party CLI binaries against a disposable localhost server.
*
* No provider account is contacted and the credential is a fixed non-secret
* sentinel. A PASS proves that the CLI started, reached the expected protocol
* endpoint, parsed a minimal response, printed the marker, and exited cleanly.
*/
import { randomUUID } from "node:crypto";
import {
buildSmokeEnvironment,
classifySmokeResult,
runProcess,
smokeModel,
} from "./smoke-safety.mjs";
export { buildSmokeEnvironment, classifySmokeResult, runProcess } from "./smoke-safety.mjs";
import { createServer } from "node:http";
import { accessSync, constants, mkdtempSync, rmSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
export const SMOKE_MARKER = "OMNIROUTE_SMOKE_OK";
export const SMOKE_TOKEN = "omniroute-smoke-sentinel";
export const DEFAULT_SMOKE_TARGETS = Object.freeze(["aider", "goose", "opencode", "qwen", "codex"]);
const SCRIPT_PATH = fileURLToPath(import.meta.url);
const REPO_ROOT = path.resolve(path.dirname(SCRIPT_PATH), "../..");
const CLI_PATH = path.join(REPO_ROOT, "bin", "omniroute.mjs");
const AIDER_EMPTY_CONFIG = path.join(REPO_ROOT, "tests", "fixtures", "cli", "empty-aider.yml");
function jsonResponse(response, status, body) {
response.writeHead(status, { "content-type": "application/json" });
response.end(JSON.stringify(body));
}
function sseWrite(response, event) {
response.write(`event: ${event.type}\n`);
response.write(`data: ${JSON.stringify(event)}\n\n`);
}
export function openAiChatBody(model = "smoke/model", marker = SMOKE_MARKER) {
return {
id: "chatcmpl_omniroute_smoke",
object: "chat.completion",
created: 1,
model,
choices: [
{
index: 0,
message: { role: "assistant", content: marker },
finish_reason: "stop",
},
],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
};
}
function streamOpenAiChat(response, model, marker) {
response.writeHead(200, {
"content-type": "text/event-stream",
"cache-control": "no-cache",
connection: "keep-alive",
});
const base = {
id: "chatcmpl_omniroute_smoke",
object: "chat.completion.chunk",
created: 1,
model,
};
response.write(
`data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: "assistant", content: marker }, finish_reason: null }] })}\n\n`
);
response.write(
`data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\n`
);
response.end("data: [DONE]\n\n");
}
export function completedResponsesBody(model = "smoke/model", marker = SMOKE_MARKER) {
return {
id: "resp_omniroute_smoke",
object: "response",
created_at: 1,
status: "completed",
error: null,
incomplete_details: null,
instructions: null,
max_output_tokens: null,
model,
output: [
{
id: "msg_omniroute_smoke",
type: "message",
status: "completed",
role: "assistant",
content: [{ type: "output_text", text: marker, annotations: [], logprobs: [] }],
},
],
parallel_tool_calls: true,
previous_response_id: null,
reasoning: { effort: null, summary: null },
store: false,
temperature: 1,
text: { format: { type: "text" } },
tool_choice: "auto",
tools: [],
top_p: 1,
truncation: "disabled",
usage: {
input_tokens: 1,
input_tokens_details: { cached_tokens: 0 },
output_tokens: 1,
output_tokens_details: { reasoning_tokens: 0 },
total_tokens: 2,
},
metadata: {},
};
}
function streamResponses(response, model, marker) {
response.writeHead(200, {
"content-type": "text/event-stream",
"cache-control": "no-cache",
connection: "keep-alive",
});
const completed = completedResponsesBody(model, marker);
const started = { ...completed, status: "in_progress", output: [], usage: null };
const item = { ...completed.output[0], status: "in_progress", content: [] };
const part = { type: "output_text", text: "", annotations: [], logprobs: [] };
sseWrite(response, { type: "response.created", sequence_number: 0, response: started });
sseWrite(response, {
type: "response.output_item.added",
sequence_number: 1,
output_index: 0,
item,
});
sseWrite(response, {
type: "response.content_part.added",
sequence_number: 2,
item_id: item.id,
output_index: 0,
content_index: 0,
part,
});
sseWrite(response, {
type: "response.output_text.delta",
sequence_number: 3,
item_id: item.id,
output_index: 0,
content_index: 0,
delta: marker,
logprobs: [],
});
sseWrite(response, {
type: "response.output_text.done",
sequence_number: 4,
item_id: item.id,
output_index: 0,
content_index: 0,
text: marker,
logprobs: [],
});
sseWrite(response, {
type: "response.content_part.done",
sequence_number: 5,
item_id: item.id,
output_index: 0,
content_index: 0,
part: completed.output[0].content[0],
});
sseWrite(response, {
type: "response.output_item.done",
sequence_number: 6,
output_index: 0,
item: completed.output[0],
});
sseWrite(response, { type: "response.completed", sequence_number: 7, response: completed });
response.end("data: [DONE]\n\n");
}
async function readJsonBody(request) {
const chunks = [];
let size = 0;
for await (const chunk of request) {
size += chunk.length;
if (size > 2 * 1024 * 1024) throw new Error("smoke request body exceeds 2 MiB");
chunks.push(chunk);
}
if (chunks.length === 0) return {};
try {
return JSON.parse(Buffer.concat(chunks).toString("utf8"));
} catch {
return {};
}
}
export async function startSmokeServer(marker = `${SMOKE_MARKER}_${randomUUID()}`) {
const requests = [];
const state = { closed: false };
const server = createServer(async (request, response) => {
const url = new URL(request.url || "/", "http://127.0.0.1");
if (request.method === "GET" && url.pathname === "/api/monitoring/health") {
return jsonResponse(response, 200, { status: "ok" });
}
if (request.method === "GET" && url.pathname === "/v1/models") {
return jsonResponse(response, 200, {
object: "list",
data: DEFAULT_SMOKE_TARGETS.map((target) => ({ id: smokeModel(target), object: "model" })),
});
}
const authenticated = request.headers.authorization === `Bearer ${SMOKE_TOKEN}`;
const supported =
request.method === "POST" && ["/v1/chat/completions", "/v1/responses"].includes(url.pathname);
const receipt = {
method: request.method,
path: url.pathname,
model: null,
responseMode: "json",
statusCode: 404,
authenticated,
responseFinished: false,
};
requests.push(receipt);
response.once("finish", () => {
receipt.responseFinished = true;
});
if (!supported)
return jsonResponse(response, 404, { error: { message: "Unsupported smoke endpoint" } });
if (!authenticated) {
receipt.statusCode = 401;
return jsonResponse(response, 401, { error: { message: "Smoke sentinel required" } });
}
try {
const body = await readJsonBody(request);
receipt.model = typeof body.model === "string" ? body.model : null;
receipt.responseMode = body.stream ? "sse" : "json";
receipt.statusCode = 200;
if (url.pathname === "/v1/responses") {
if (body.stream) return streamResponses(response, body.model, marker);
return jsonResponse(response, 200, completedResponsesBody(body.model, marker));
}
if (body.stream) return streamOpenAiChat(response, body.model, marker);
return jsonResponse(response, 200, openAiChatBody(body.model, marker));
} catch {
receipt.statusCode = 400;
if (!response.headersSent)
jsonResponse(response, 400, { error: { message: "Invalid smoke request" } });
else response.destroy();
}
});
await new Promise((resolve, reject) => {
server.once("error", reject);
server.listen(0, "127.0.0.1", resolve);
});
const address = server.address();
if (!address || typeof address === "string") throw new Error("smoke server has no TCP address");
return {
baseUrl: `http://127.0.0.1:${address.port}`,
requests,
marker,
state,
close: () =>
new Promise((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
server.closeAllConnections?.();
}).then(() => {
state.closed = true;
}),
};
}
export function targetToolArgs(target) {
const prompt = "Say hello briefly. Do not call tools.";
switch (target) {
case "aider":
return [
"--message",
prompt,
"--yes-always",
"--no-git",
"--no-stream",
"--analytics-disable",
"--no-check-update",
"--no-show-release-notes",
"--no-fancy-input",
"--disable-playwright",
"--config",
AIDER_EMPTY_CONFIG,
"--env-file",
"/dev/null",
];
case "goose":
return [
"run",
"--text",
prompt,
"--no-session",
"--no-profile",
"--quiet",
"--max-turns",
"1",
];
case "opencode":
return ["run", prompt, "--format", "json", "--pure"];
case "qwen":
return [
"--prompt",
prompt,
"--output-format",
"json",
"--safe-mode",
"--max-session-turns",
"1",
"--max-wall-time",
"30s",
"--max-tool-calls",
"0",
];
case "codex":
return [
"exec",
"--skip-git-repo-check",
"--ephemeral",
"--ignore-user-config",
"--ignore-rules",
"--sandbox",
"read-only",
"--color",
"never",
prompt,
];
default:
throw new Error(`No real smoke recipe for '${target}'`);
}
}
export function buildSmokeInvocation(target, baseUrl, { inheritIsolatedEnv = false } = {}) {
// Aider performs an expensive browser-backed warning path for unknown model
// metadata before it ever calls the endpoint. A known OpenAI model still
// exercises the same OmniRoute transport without that unrelated cold path.
const model = smokeModel(target);
return [
CLI_PATH,
"run",
target,
"--base-url",
baseUrl,
"--model",
model,
"--api-key-env",
"OMNIROUTE_SMOKE_KEY",
...(inheritIsolatedEnv ? ["--inherit-env"] : []),
"--",
...targetToolArgs(target),
];
}
export function parseSmokeArgs(argv) {
const options = {
targets: [...DEFAULT_SMOKE_TARGETS],
binDir: "",
json: false,
help: false,
inheritIsolatedEnv: false,
};
for (let index = 0; index < argv.length; index++) {
if (argv[index] === "--help" || argv[index] === "-h") {
return { ...options, targets: [], help: true };
} else if (argv[index] === "--targets") {
if (!argv[index + 1] || argv[index + 1].startsWith("--"))
throw new Error("--targets requires a value");
options.targets = String(argv[++index] || "")
.split(",")
.map((value) => value.trim())
.filter(Boolean);
} else if (argv[index] === "--bin-dir") {
if (!argv[index + 1] || argv[index + 1].startsWith("--"))
throw new Error("--bin-dir requires a value");
options.binDir = path.resolve(String(argv[++index] || ""));
} else if (argv[index] === "--json") {
options.json = true;
} else if (argv[index] === "--inherit-isolated-env") {
options.inheritIsolatedEnv = true;
} else {
throw new Error(`Unknown option '${argv[index]}'`);
}
}
if (!options.targets.length) throw new Error("At least one smoke target is required");
for (const target of options.targets) targetToolArgs(target);
return options;
}
function binaryAvailable(target, env) {
return (env.PATH || "").split(path.delimiter).some((directory) => {
try {
accessSync(path.join(directory, target), constants.X_OK);
return true;
} catch {
return false;
}
});
}
async function runTarget(target, options) {
const temporary = mkdtempSync(path.join(os.tmpdir(), "omniroute-real-cli-smoke-"));
let server;
const result = { target, status: "FAIL", requests: [] };
try {
if (process.platform === "win32") {
result.status = "HOLD-process-isolation";
return result;
}
const env = buildSmokeEnvironment(temporary, SMOKE_TOKEN, options.binDir);
if (!binaryAvailable(target, env)) {
result.status = "HOLD-missing-binary";
return result;
}
server = await startSmokeServer();
const execution = await runProcess(buildSmokeInvocation(target, server.baseUrl, options), {
env,
cwd: temporary,
timeoutMs: options.timeoutMs,
});
Object.assign(result, {
status: classifySmokeResult(execution, server.requests, server.marker, target),
exitCode: execution.code,
timedOut: execution.timedOut,
terminationSignal: execution.terminationSignal,
markerObserved: execution.stdout.includes(server.marker),
requests: server.requests,
});
// Only this isolated child output is exposed; inherited secrets never enter its environment.
if (result.status !== "PASS") {
result.diagnostic = `${execution.stdout}\n${execution.stderr}`
.replaceAll(SMOKE_TOKEN, "[redacted]")
.slice(-4_000);
}
return result;
} finally {
try {
if (server) await server.close();
} finally {
rmSync(temporary, { recursive: true, force: true });
}
result.cleanup = {
serverClosed: server ? server.state.closed : true,
temporaryDirectoryRemoved: true,
processGroupIsolation: process.platform !== "win32",
};
}
}
export async function runRealCliSmokes(options) {
for (const target of options.targets) targetToolArgs(target);
const results = [];
for (const target of options.targets) results.push(await runTarget(target, options));
return results;
}
async function main() {
const options = parseSmokeArgs(process.argv.slice(2));
if (options.help) {
console.log(`Usage: node scripts/cli/smoke-cli-integrations.mjs [options]
Options:
--targets <ids> Comma-separated targets (default: aider,goose,opencode,qwen,codex)
--bin-dir <path> Prepend a directory containing CLI binaries to PATH
--inherit-isolated-env Pass only the harness's fresh allowlisted environment to the CLI
--json Print structured results
-h, --help Show this help`);
return;
}
const results = await runRealCliSmokes(options);
if (options.json) console.log(JSON.stringify(results, null, 2));
else {
for (const result of results) {
console.log(
`${result.target}: ${result.status} (exit=${result.exitCode}, requests=${result.requests.length}, marker=${result.markerObserved})`
);
if (result.diagnostic) console.log(result.diagnostic);
}
}
if (results.some((result) => result.status === "FAIL")) process.exitCode = 1;
else if (results.some((result) => result.status.startsWith("HOLD"))) process.exitCode = 2;
}
if (path.resolve(process.argv[1] || "") === SCRIPT_PATH) {
await main();
}