Files
OmniRoute/tests/unit/executor-replicate.test.ts
diegosouzapw 696d6818ee test(providers): match the Replicate stream host on the parsed URL
CodeQL (js/incomplete-url-substring-sanitization, 3 high) flagged the
three startsWith("https://stream.replicate.com") checks in the Replicate
executor test: a raw prefix also accepts stream.replicate.com.evil.test.
The test now compares the parsed protocol and hostname through a small
isReplicateStreamUrl() helper. Assertions are unchanged.

Co-authored-by: jowimila <127446097+jowimila@users.noreply.github.com>
2026-11-13 09:57:20 -03:00

688 lines
22 KiB
TypeScript

import test from "node:test";
import assert from "node:assert/strict";
import { getExecutor, hasSpecializedExecutor } from "../../open-sse/executors/index.ts";
import { ReplicateExecutor } from "../../open-sse/executors/replicate.ts";
import { CerebriumExecutor } from "../../open-sse/executors/cerebrium.ts";
import { APIKEY_PROVIDERS } from "../../src/shared/constants/providers.ts";
const MODEL = "meta/meta-llama-3-70b-instruct";
const credentials = { apiKey: "r8_test" };
// Match the stream origin on the parsed URL, not by string prefix: a
// prefix check on the raw string would also accept a host such as stream.replicate.com.evil.test.
function isReplicateStreamUrl(url: string): boolean {
const parsed = new URL(url);
return parsed.protocol === "https:" && parsed.hostname === "stream.replicate.com";
}
function jsonResponse(body: unknown, status = 200) {
return new Response(JSON.stringify(body), {
status,
headers: { "Content-Type": "application/json" },
});
}
async function withFetch<T>(
handler: (url: string, init: RequestInit) => Response | Promise<Response>,
run: () => Promise<T>
): Promise<T> {
const original = globalThis.fetch;
globalThis.fetch = (async (url: unknown, init: RequestInit = {}) =>
handler(String(url), init)) as typeof fetch;
try {
return await run();
} finally {
globalThis.fetch = original;
}
}
test("ReplicateExecutor is registered in the executor index", async () => {
assert.equal(hasSpecializedExecutor("replicate"), true);
assert.ok((await getExecutor("replicate")) instanceof ReplicateExecutor);
});
test("ReplicateExecutor refuses a metadata base URL before anything reaches the network (GHSA-4f49)", async () => {
const executor = new ReplicateExecutor();
const calls: string[] = [];
const result = await withFetch(
(url) => {
calls.push(url);
return jsonResponse({ status: "succeeded", output: "leaked" });
},
() =>
executor.execute({
model: MODEL,
body: { messages: [{ role: "user", content: "hi" }] },
stream: false,
credentials: {
...credentials,
providerSpecificData: { baseUrl: "http://169.254.169.254/latest" },
},
})
);
assert.deepEqual(calls, []);
assert.equal(result.response.status, 502);
});
test("ReplicateExecutor does not auto-follow a redirect with the Bearer token", async () => {
const executor = new ReplicateExecutor();
const seen: Array<{ url: string; authorization: string | null }> = [];
await withFetch(
(url, init) => {
seen.push({ url, authorization: new Headers(init.headers).get("Authorization") });
if (seen.length === 1) {
return new Response(null, {
status: 307,
headers: { Location: "https://collector.example.com/steal" },
});
}
return jsonResponse({ status: "succeeded", output: "ok" });
},
() =>
executor.execute({
model: MODEL,
body: { messages: [{ role: "user", content: "hi" }] },
stream: false,
credentials,
})
);
assert.equal(seen[0].authorization, "Bearer r8_test");
assert.equal(seen[1].url, "https://collector.example.com/steal");
assert.equal(seen[1].authorization, null);
});
test("ReplicateExecutor posts to the official-model predictions URL and wraps array output", async () => {
const executor = new ReplicateExecutor();
const calls: Array<{
url: string;
body: Record<string, unknown>;
headers: Record<string, string>;
}> = [];
const result = await withFetch(
(url, init) => {
calls.push({
url,
body: JSON.parse(String(init.body)),
headers: init.headers as Record<string, string>,
});
return jsonResponse({ status: "succeeded", output: ["Hello", ", ", "world"] });
},
() =>
executor.execute({
model: MODEL,
body: {
messages: [
{ role: "system", content: "Be brief." },
{ role: "user", content: "Hi" },
],
max_tokens: 64,
temperature: 0.5,
},
stream: false,
credentials,
} as never)
);
assert.equal(calls.length, 1);
assert.equal(
calls[0].url,
"https://api.replicate.com/v1/models/meta/meta-llama-3-70b-instruct/predictions"
);
assert.equal(calls[0].headers.Authorization, "Bearer r8_test");
assert.match(calls[0].headers.Prefer, /^wait=/);
assert.deepEqual(calls[0].body, {
input: { prompt: "Hi", system_prompt: "Be brief.", max_tokens: 64, temperature: 0.5 },
});
const json = await result.response.json();
assert.equal(json.object, "chat.completion");
assert.equal(json.choices[0].message.content, "Hello, world");
});
test("polling does not leave an abort listener behind per tick", async () => {
class QuickPollExecutor extends ReplicateExecutor {
protected pollIntervalMs = 1;
}
const controller = new AbortController();
const { signal } = controller;
const live = new Set<unknown>();
const add = signal.addEventListener.bind(signal);
const remove = signal.removeEventListener.bind(signal);
signal.addEventListener = ((type: string, listener: EventListener, options?: unknown) => {
live.add(listener);
add(type, listener, options as AddEventListenerOptions);
}) as typeof signal.addEventListener;
signal.removeEventListener = ((type: string, listener: EventListener) => {
live.delete(listener);
remove(type, listener);
}) as typeof signal.removeEventListener;
let polls = 0;
await withFetch(
(url) => {
if (!url.endsWith("/predictions/p1")) {
return jsonResponse({
status: "starting",
urls: { get: "https://api.replicate.com/v1/predictions/p1" },
});
}
polls += 1;
return jsonResponse({ status: polls < 3 ? "processing" : "succeeded", output: "ok" });
},
() =>
new QuickPollExecutor().execute({
model: MODEL,
body: { messages: [{ role: "user", content: "hi" }] },
stream: false,
credentials,
signal,
})
);
assert.equal(polls, 3);
assert.equal(live.size, 0);
});
test("ReplicateExecutor polls urls.get until the prediction finishes", async () => {
const executor = new ReplicateExecutor();
const seen: string[] = [];
const result = await withFetch(
(url, init) => {
seen.push(`${init.method} ${url}`);
if (init.method === "POST") {
return jsonResponse({
status: "processing",
urls: { get: "https://api.replicate.com/v1/predictions/abc" },
});
}
assert.equal((init.headers as Record<string, string>).Prefer, undefined);
return jsonResponse({ status: "succeeded", output: ["done"] });
},
() =>
executor.execute({
model: MODEL,
body: { messages: [{ role: "user", content: "Hi" }] },
stream: false,
credentials,
} as never)
);
assert.deepEqual(seen, [
`POST https://api.replicate.com/v1/models/${MODEL}/predictions`,
"GET https://api.replicate.com/v1/predictions/abc",
]);
assert.equal((await result.response.json()).choices[0].message.content, "done");
});
test("ReplicateExecutor synthesizes an SSE stream for stream requests", async () => {
const executor = new ReplicateExecutor();
const result = await withFetch(
() => jsonResponse({ status: "succeeded", output: ["streamed text"] }),
() =>
executor.execute({
model: MODEL,
body: { messages: [{ role: "user", content: "Hi" }] },
stream: true,
credentials,
} as never)
);
const text = await result.response.text();
assert.match(text, /"delta":\{"role":"assistant"\}/);
assert.match(text, /"content":"streamed text"/);
assert.match(text, /"finish_reason":"stop"/);
assert.ok(text.trimEnd().endsWith("data: [DONE]"));
});
test("ReplicateExecutor maps failed predictions and HTTP errors without leaking paths", async () => {
const executor = new ReplicateExecutor();
const failed = await withFetch(
() => jsonResponse({ status: "failed", error: "model crashed" }),
() =>
executor.execute({
model: MODEL,
body: { messages: [{ role: "user", content: "Hi" }] },
stream: false,
credentials,
} as never)
);
assert.equal(failed.response.status, 502);
assert.match((await failed.response.json()).error.message, /model crashed/);
const unauthorized = await withFetch(
() => jsonResponse({ detail: "Invalid token at /srv/app/secret.js" }, 401),
() =>
executor.execute({
model: MODEL,
body: { messages: [{ role: "user", content: "Hi" }] },
stream: false,
credentials,
} as never)
);
assert.equal(unauthorized.response.status, 401);
const body = await unauthorized.response.json();
assert.ok(!body.error.message.includes("/srv/app"));
assert.ok(!body.error.message.includes("at /"));
});
test("ReplicateExecutor rejects requests with no user message", async () => {
const executor = new ReplicateExecutor();
const result = await executor.execute({
model: MODEL,
body: { messages: [{ role: "system", content: "only system" }] },
stream: false,
credentials,
} as never);
assert.equal(result.response.status, 400);
});
class FastReplicateExecutor extends ReplicateExecutor {
protected pollIntervalMs = 1;
protected pollDeadlineMs = 30;
}
const userBody = { messages: [{ role: "user", content: "Hi" }] };
test("Replicate never sends the API key to a polling URL on another origin", async () => {
const executor = new ReplicateExecutor();
const hosts: string[] = [];
const result = await withFetch(
(url) => {
hosts.push(new URL(url).host);
return jsonResponse({
status: "processing",
urls: { get: "https://attacker.example/steal" },
});
},
() => executor.execute({ model: MODEL, body: userBody, stream: false, credentials } as never)
);
assert.deepEqual(hosts, ["api.replicate.com"]);
assert.equal(result.response.status, 502);
});
test("Replicate cancels the prediction when the poll deadline passes", async () => {
const executor = new FastReplicateExecutor();
const seen: string[] = [];
const result = await withFetch(
(url, init) => {
seen.push(`${init.method} ${url}`);
if (init.method === "POST" && url.endsWith("/predictions")) {
return jsonResponse({
status: "processing",
urls: {
get: "https://api.replicate.com/v1/predictions/slow",
cancel: "https://api.replicate.com/v1/predictions/slow/cancel",
},
});
}
return jsonResponse({
status: "processing",
urls: { cancel: "https://api.replicate.com/v1/predictions/slow/cancel" },
});
},
() => executor.execute({ model: MODEL, body: userBody, stream: false, credentials } as never)
);
assert.equal(result.response.status, 504);
assert.ok(seen.includes("POST https://api.replicate.com/v1/predictions/slow/cancel"));
});
test("Replicate does not cancel through a foreign-origin cancel URL", async () => {
const executor = new FastReplicateExecutor();
const hosts = new Set<string>();
await withFetch(
(url, _init) => {
hosts.add(new URL(url).host);
return jsonResponse({
status: "processing",
urls: {
get: "https://api.replicate.com/v1/predictions/slow",
cancel: "https://attacker.example/cancel",
},
});
},
() => executor.execute({ model: MODEL, body: userBody, stream: false, credentials } as never)
);
assert.deepEqual([...hosts], ["api.replicate.com"]);
});
test("Replicate keeps the upstream status and Retry-After when a poll is rate limited", async () => {
const executor = new FastReplicateExecutor();
const result = await withFetch(
(_url, init) => {
if (init.method === "POST") {
return jsonResponse({
status: "processing",
urls: { get: "https://api.replicate.com/v1/predictions/abc" },
});
}
return new Response(JSON.stringify({ detail: "Too many requests" }), {
status: 429,
headers: { "Content-Type": "application/json", "Retry-After": "7" },
});
},
() => executor.execute({ model: MODEL, body: userBody, stream: false, credentials } as never)
);
assert.equal(result.response.status, 429);
assert.equal(result.response.headers.get("Retry-After"), "7");
});
test("Replicate does not echo raw internal errors or long upstream detail", async () => {
const executor = new ReplicateExecutor();
const network = await withFetch(
() => {
throw new Error("connect ECONNREFUSED 10.0.0.5:443 at /srv/app/node_modules/x.js");
},
() => executor.execute({ model: MODEL, body: userBody, stream: false, credentials } as never)
);
assert.equal(network.response.status, 502);
const networkMessage = (await network.response.json()).error.message;
assert.ok(!networkMessage.includes("10.0.0.5"));
assert.ok(!networkMessage.includes("/srv/app"));
const verbose = await withFetch(
() => jsonResponse({ detail: "x".repeat(5000) }, 400),
() => executor.execute({ model: MODEL, body: userBody, stream: false, credentials } as never)
);
const verboseMessage = (await verbose.response.json()).error.message;
assert.ok(verboseMessage.length < 400);
});
test("Replicate is not advertised as a free provider", () => {
assert.equal(APIKEY_PROVIDERS.replicate.hasFree, false);
});
test("Cerebrium uses the connection Base URL verbatim (no /chat/completions suffix)", () => {
const executor = new CerebriumExecutor();
const url = "https://api.cerebrium.ai/v4/p-abc123/my-app/run";
assert.equal(
executor.buildUrl("meta-llama/Meta-Llama-3.1-8B-Instruct", false, 0, {
apiKey: "jwt",
providerSpecificData: { baseUrl: `${url}/` },
} as never),
url
);
});
test("Cerebrium without a Base URL fails fast with 400 and makes no upstream call", async () => {
const executor = new CerebriumExecutor();
let called = false;
const result = await withFetch(
() => {
called = true;
return jsonResponse({});
},
() =>
executor.execute({
model: "meta-llama/Meta-Llama-3.1-8B-Instruct",
body: userBody,
stream: false,
credentials: { apiKey: "jwt" },
} as never)
);
assert.equal(called, false);
assert.equal(result.response.status, 400);
assert.match((await result.response.json()).error.message, /Base URL/);
});
const encoder = new TextEncoder();
function sseBody(events: string[], holdOpen = false) {
return new Response(
new ReadableStream({
start(controller) {
for (const event of events) controller.enqueue(encoder.encode(event));
if (!holdOpen) controller.close();
},
}),
{ status: 200, headers: { "Content-Type": "text/event-stream" } }
);
}
const streamingPrediction = {
status: "starting",
urls: {
get: "https://api.replicate.com/v1/predictions/live",
cancel: "https://api.replicate.com/v1/predictions/live/cancel",
stream: "https://stream.replicate.com/v1/streams/abc",
},
};
// Consumes the response while the fetch mock is still installed (streams are lazy).
async function streamText(
handler: (url: string, init: RequestInit) => Response | Promise<Response>
): Promise<string> {
const executor = new ReplicateExecutor();
return withFetch(handler, async () => {
const out = await executor.execute({
model: MODEL,
body: userBody,
stream: true,
credentials,
} as never);
return out.response.text();
});
}
test("Replicate relays the native SSE stream token by token and reports real usage", async () => {
const seen: Array<{ url: string; headers: Record<string, string> }> = [];
const text = await streamText((url, init) => {
seen.push({ url, headers: init.headers as Record<string, string> });
if (url === "https://stream.replicate.com/v1/streams/abc") {
return sseBody([
"event: output\ndata: Hel\n\n",
"event: output\ndata: lo\n\n",
"event: done\ndata: {}\n\n",
]);
}
if (url.endsWith("/predictions/live")) {
return jsonResponse({
status: "succeeded",
metrics: { input_token_count: 11, output_token_count: 2 },
});
}
return jsonResponse(streamingPrediction);
});
const deltas = [...text.matchAll(/"delta":\{"content":"([^"]*)"\}/g)].map((m) => m[1]);
assert.deepEqual(deltas, ["Hel", " lo"]);
assert.match(text, /"usage":\{"prompt_tokens":11,"completion_tokens":2,"total_tokens":13\}/);
assert.match(text, /"finish_reason":"stop"/);
assert.ok(text.trimEnd().endsWith("data: [DONE]"));
assert.equal(seen[0].headers.Prefer, undefined);
const streamCall = seen.find((c) => isReplicateStreamUrl(c.url));
assert.ok(streamCall);
assert.equal(streamCall.headers.Authorization, undefined);
});
test("Replicate stream surfaces an upstream error event as a sanitized error frame", async () => {
const text = await streamText((url) =>
isReplicateStreamUrl(url)
? sseBody(['event: error\ndata: {"detail":"boom at /srv/app/x.js"}\n\n'])
: jsonResponse(streamingPrediction)
);
assert.match(text, /Replicate prediction failed/);
assert.ok(!text.includes("/srv/app"));
assert.ok(text.trimEnd().endsWith("data: [DONE]"));
});
test("Replicate cancels the running prediction when the client drops the stream", async () => {
const executor = new ReplicateExecutor();
const posts: string[] = [];
const result = await withFetch(
(url, init) => {
if (init.method === "POST" && url.endsWith("/cancel")) posts.push(url);
if (isReplicateStreamUrl(url)) {
return sseBody(["event: output\ndata: partial\n\n"], true);
}
return jsonResponse(streamingPrediction);
},
async () => {
const out = await executor.execute({
model: MODEL,
body: userBody,
stream: true,
credentials,
} as never);
const reader = out.response.body!.getReader();
await reader.read();
await reader.cancel();
await new Promise((resolve) => setTimeout(resolve, 20));
return out;
}
);
assert.ok(result.response);
assert.deepEqual(posts, ["https://api.replicate.com/v1/predictions/live/cancel"]);
});
test("Replicate ignores a stream URL on an untrusted host and polls instead", async () => {
const executor = new FastReplicateExecutor();
const hosts = new Set<string>();
const result = await withFetch(
(url, init) => {
hosts.add(new URL(url).host);
if (init.method === "POST") {
return jsonResponse({
status: "starting",
urls: {
get: "https://api.replicate.com/v1/predictions/p1",
stream: "https://attacker.example/stream",
},
});
}
return jsonResponse({ status: "succeeded", output: ["ok"] });
},
() => executor.execute({ model: MODEL, body: userBody, stream: true, credentials } as never)
);
assert.ok(!hosts.has("attacker.example"));
assert.match(await result.response.text(), /"content":"ok"/);
});
test("Replicate reports Replicate's token metrics and length truncation", async () => {
const executor = new ReplicateExecutor();
const result = await withFetch(
() =>
jsonResponse({
status: "succeeded",
output: ["cut off"],
metrics: { input_token_count: 5, output_token_count: 16 },
}),
() =>
executor.execute({
model: MODEL,
body: { messages: userBody.messages, max_tokens: 16 },
stream: false,
credentials,
} as never)
);
const json = await result.response.json();
assert.deepEqual(json.usage, { prompt_tokens: 5, completion_tokens: 16, total_tokens: 21 });
assert.equal(json.choices[0].finish_reason, "length");
});
test("Replicate sends versioned models to /predictions with a version field", async () => {
const executor = new ReplicateExecutor();
const calls: Array<{ url: string; body: Record<string, unknown> }> = [];
await withFetch(
(url, init) => {
calls.push({ url, body: JSON.parse(String(init.body)) });
return jsonResponse({ status: "succeeded", output: ["v"] });
},
() =>
executor.execute({
model: "owner/custom-model:abc123def456",
body: userBody,
stream: false,
credentials,
} as never)
);
assert.equal(calls[0].url, "https://api.replicate.com/v1/predictions");
assert.equal(calls[0].body.version, "abc123def456");
});
test("Replicate rejects malformed model IDs before any upstream call", async () => {
const executor = new ReplicateExecutor();
let called = false;
const result = await withFetch(
() => {
called = true;
return jsonResponse({});
},
() =>
executor.execute({
model: "../../etc/passwd",
body: userBody,
stream: false,
credentials,
} as never)
);
assert.equal(called, false);
assert.equal(result.response.status, 400);
});
test("Replicate keeps tool results in the transcript and forwards the first image", async () => {
const executor = new ReplicateExecutor();
let sent: { input: Record<string, string> } = { input: {} };
await withFetch(
(_url, init) => {
sent = JSON.parse(String(init.body));
return jsonResponse({ status: "succeeded", output: ["ok"] });
},
() =>
executor.execute({
model: MODEL,
body: {
messages: [
{
role: "user",
content: [
{ type: "text", text: "What is this?" },
{ type: "image_url", image_url: { url: "https://example.com/a.png" } },
],
},
{
role: "assistant",
content: "",
tool_calls: [{ function: { name: "lookup", arguments: '{"q":1}' } }],
},
{ role: "tool", content: "result-42" },
],
},
stream: false,
credentials,
} as never)
);
assert.equal(sent.input.image, "https://example.com/a.png");
assert.match(sent.input.prompt, /\[tool call lookup\(\{"q":1\}\)\]/);
assert.match(sent.input.prompt, /Tool result: result-42/);
assert.ok(sent.input.prompt.trimEnd().endsWith("Assistant:"));
});