mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-16 17:09:09 +00:00
#7847 reports a 3.05 MiB request (729 messages / 86 tools) reaching ~12,282 MiB of V8 heap, and asks for "a regression benchmark that records peak heap for representative 500-800-message, tool-rich requests" before any fix lands. There is currently no memory baseline in the repo at all (bench:compression is the only benchmark), so a clone-reduction change could neither be justified nor regression-guarded. npm run bench:heap-body attributes retained heap to each copy the chat path makes: | mechanism | call site | retained | x wire | | cloneLogPayload (unbounded) | chat.ts buildClientRawRequest | 3.18 MiB | 1.04x | | cloneBoundedForLog (bounded) | requestLogger.logClientRawRequest | 0.04 MiB | 0.01x | | structuredClone x3 (combo targets) | combo.ts attemptBody | 9.53 MiB | 3.12x | | JSON.stringify (token estimate) | combo.ts estimateTokens | 3.06 MiB | 1.00x | | per request (sum) | |15.81 MiB | 5.17x | It measures the real production helpers rather than reimplementations, so a change to the log bounds or the clone strategy is reflected directly. Design notes: - Deterministic: fixed-seed LCG, no Math.random(). Verified byte-identical across three consecutive runs — without that, a before/after delta measures noise, not the change. - Corpus lives in its own side-effect-free module so the unit test can import it without booting SQLite (requestLogger transitively opens the DB at import time). - Hermetic: DATA_DIR is redirected to a temp dir before importing, so the benchmark never touches the operator's real ~/.omniroute store. - Node, not bun: --expose-gc and V8 heap accounting are the measurement; another engine's heap number would not describe the production runtime. - --max-retained-mib exits non-zero, so this can become a CI gate once a target is agreed. Reports only; wires nothing into CI and changes no production code.
This commit is contained in:
1
changelog.d/maintenance/heap-benchmark-request-body.md
Normal file
1
changelog.d/maintenance/heap-benchmark-request-body.md
Normal file
@@ -0,0 +1 @@
|
||||
- chore(perf): add `npm run bench:heap-body` — a deterministic benchmark that attributes retained V8 heap to each request-body copy on the chat path (entry log clone, combo per-target clone, token-estimation string), reproducing the #7847 incident shape (3.05 MiB / 729 messages / 86 tools) so the clone-amplification work can be justified and regression-guarded with numbers instead of intuition
|
||||
@@ -79,6 +79,7 @@
|
||||
"prebuild:docs": "node scripts/docs/gen-openapi-module.mjs",
|
||||
"gen:provider-reference": "bun scripts/docs/gen-provider-reference.ts",
|
||||
"bench:compression": "bun scripts/compression/benchmark.ts",
|
||||
"bench:heap-body": "node --expose-gc --import tsx/esm scripts/perf/request-body-heap.ts",
|
||||
"eval:compression": "node --import tsx scripts/compression-eval/index.ts",
|
||||
"eval:router": "node --import tsx scripts/router-eval/index.ts",
|
||||
"eval:router:compare": "node --import tsx scripts/router-eval/compare.ts",
|
||||
|
||||
83
scripts/perf/agentPayloadCorpus.ts
Normal file
83
scripts/perf/agentPayloadCorpus.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* Deterministic coding-agent request corpus for the #7847 heap benchmark.
|
||||
*
|
||||
* Kept separate from `request-body-heap.ts` so it can be imported with zero side effects:
|
||||
* the benchmark redirects DATA_DIR and transitively boots SQLite at import time, which a unit
|
||||
* test must not do just to assert corpus stability.
|
||||
*
|
||||
* Determinism is the point. A benchmark corpus built with Math.random() would make a
|
||||
* before/after comparison measure noise instead of the change under test, so the generator uses
|
||||
* a fixed-seed LCG and is covered by tests/unit/heap-benchmark-corpus.test.ts.
|
||||
*/
|
||||
|
||||
/** Defaults reproduce the #7847 production incident: 3.05 MiB, 729 messages, 86 tools. */
|
||||
export const INCIDENT_SHAPE = {
|
||||
messages: 729,
|
||||
tools: 86,
|
||||
/** Calibrated so the defaults land on the incident's wire size. */
|
||||
contentWords: 527,
|
||||
} as const;
|
||||
|
||||
function lcg(seed: number): () => number {
|
||||
let s = seed >>> 0;
|
||||
return () => ((s = (s * 1664525 + 1013904223) >>> 0) / 0x100000000);
|
||||
}
|
||||
|
||||
const WORDS = [
|
||||
"refactor",
|
||||
"handler",
|
||||
"provider",
|
||||
"combo",
|
||||
"stream",
|
||||
"token",
|
||||
"context",
|
||||
"payload",
|
||||
"upstream",
|
||||
"fallback",
|
||||
"quota",
|
||||
"retry",
|
||||
"executor",
|
||||
"translate",
|
||||
"compression",
|
||||
"cache",
|
||||
];
|
||||
|
||||
function sentence(rnd: () => number, words: number): string {
|
||||
const out: string[] = [];
|
||||
for (let i = 0; i < words; i++) out.push(WORDS[Math.floor(rnd() * WORDS.length)]);
|
||||
return out.join(" ");
|
||||
}
|
||||
|
||||
/** A coding-agent shaped request: long alternating history plus a large tool catalog. */
|
||||
export function buildAgentPayload(
|
||||
messages: number = INCIDENT_SHAPE.messages,
|
||||
tools: number = INCIDENT_SHAPE.tools,
|
||||
contentWords: number = INCIDENT_SHAPE.contentWords
|
||||
): Record<string, unknown> {
|
||||
const rnd = lcg(0x5eed);
|
||||
return {
|
||||
model: "claude-opus-5",
|
||||
stream: true,
|
||||
messages: Array.from({ length: messages }, (_, i) => ({
|
||||
role: i % 2 === 0 ? "user" : "assistant",
|
||||
content: sentence(rnd, contentWords),
|
||||
})),
|
||||
tools: Array.from({ length: tools }, (_, i) => ({
|
||||
type: "function",
|
||||
function: {
|
||||
name: `tool_${i}`,
|
||||
description: sentence(rnd, 25),
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: Object.fromEntries(
|
||||
Array.from({ length: 8 }, (_, p) => [
|
||||
`param_${p}`,
|
||||
{ type: "string", description: sentence(rnd, 10) },
|
||||
])
|
||||
),
|
||||
required: ["param_0"],
|
||||
},
|
||||
},
|
||||
})),
|
||||
};
|
||||
}
|
||||
200
scripts/perf/request-body-heap.ts
Normal file
200
scripts/perf/request-body-heap.ts
Normal file
@@ -0,0 +1,200 @@
|
||||
/**
|
||||
* Request-body heap amplification benchmark (#7847).
|
||||
*
|
||||
* #7847 reports a 3.05 MiB request with 729 messages and 86 tools reaching ~12,282 MiB of V8
|
||||
* heap. The wire size does not explain the peak: the same logical body is retained several times
|
||||
* over (entry-point log clone, per-combo-target clone, token-estimation string, pending-request
|
||||
* state). This benchmark attributes retained heap to each of those mechanisms individually, so a
|
||||
* fix can be justified by numbers instead of intuition — and so a regression can be caught later.
|
||||
*
|
||||
* It measures the real production helpers (no reimplementation): `cloneLogPayload` is what
|
||||
* `buildClientRawRequest` calls per request, and `cloneBoundedForLog` is what the request logger
|
||||
* actually retains downstream.
|
||||
*
|
||||
* Deterministic and API-free (no network, no upstream credentials). The DATA_DIR is redirected to
|
||||
* a temp dir before importing, because the request-logger module opens the SQLite database on
|
||||
* import — the benchmark must never touch the operator's real ~/.omniroute store.
|
||||
*
|
||||
* Node only — NOT bun. We need `--expose-gc` and V8 heap accounting; measuring "V8 heap" under a
|
||||
* different engine would report a number that has nothing to do with the production runtime.
|
||||
*
|
||||
* Usage:
|
||||
* npm run bench:heap-body # the #7847 incident shape
|
||||
* npm run bench:heap-body -- --messages 800 --tools 120
|
||||
* npm run bench:heap-body -- --concurrency 16 # simulate overlapping requests
|
||||
* npm run bench:heap-body -- --json # machine-readable
|
||||
* npm run bench:heap-body -- --max-retained-mib 64 # non-zero exit if exceeded (regression gate)
|
||||
*/
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
|
||||
// Must happen BEFORE the dynamic imports below: open-sse/utils/requestLogger.ts transitively
|
||||
// opens the SQLite store at import time, and the benchmark must stay hermetic.
|
||||
const TMP_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-heapbench-"));
|
||||
process.env.DATA_DIR = TMP_DATA_DIR;
|
||||
|
||||
// open-sse/utils/requestLogger.ts imports @/lib/usage/usageHistory, so merely importing the
|
||||
// (pure) log-shaping helper boots SQLite and runs every migration against the temp DATA_DIR.
|
||||
// That noise would bury the report, so console output is parked for the duration of the import.
|
||||
// The measurements are unaffected: every baseline is taken after imports have settled.
|
||||
const { buildAgentPayload, INCIDENT_SHAPE } = await import("./agentPayloadCorpus.ts");
|
||||
|
||||
const realLog = console.log;
|
||||
console.log = () => {};
|
||||
const { cloneLogPayload } = await import("../../src/lib/logPayloads.ts");
|
||||
const { cloneBoundedForLog } = await import("../../open-sse/utils/requestLogger.ts");
|
||||
console.log = realLog;
|
||||
|
||||
// ── CLI ──────────────────────────────────────────────────────────────────────
|
||||
function numArg(flag: string, fallback: number): number {
|
||||
const i = process.argv.indexOf(flag);
|
||||
if (i === -1) return fallback;
|
||||
const v = Number(process.argv[i + 1]);
|
||||
return Number.isFinite(v) ? v : fallback;
|
||||
}
|
||||
const HAS = (flag: string) => process.argv.includes(flag);
|
||||
|
||||
// Defaults mirror the #7847 production incident.
|
||||
const MESSAGES = numArg("--messages", INCIDENT_SHAPE.messages);
|
||||
const TOOLS = numArg("--tools", INCIDENT_SHAPE.tools);
|
||||
// Calibrated so the defaults land on the incident's 3.05 MiB wire size.
|
||||
const CONTENT_WORDS = numArg("--content-words", INCIDENT_SHAPE.contentWords);
|
||||
const TARGETS = numArg("--targets", 3); // combo targets -> attemptBody clones
|
||||
const CONCURRENCY = numArg("--concurrency", 8);
|
||||
const MAX_RETAINED = numArg("--max-retained-mib", 0); // 0 = report only
|
||||
const AS_JSON = HAS("--json");
|
||||
|
||||
const MIB = 1024 * 1024;
|
||||
const fmt = (bytes: number) => (bytes / MIB).toFixed(2);
|
||||
|
||||
// ── Measurement ──────────────────────────────────────────────────────────────
|
||||
const gc = globalThis.gc as undefined | (() => void);
|
||||
|
||||
function settle(): void {
|
||||
// Several passes: one gc() does not reliably collect everything in a young generation.
|
||||
for (let i = 0; i < 4; i++) gc?.();
|
||||
}
|
||||
|
||||
/**
|
||||
* Retained heap of whatever `produce` returns, while it stays reachable.
|
||||
* This is the number that maps to the incident: many concurrent requests each holding copies.
|
||||
*/
|
||||
function measureRetained<T>(produce: () => T): { bytes: number; value: T } {
|
||||
settle();
|
||||
const before = process.memoryUsage().heapUsed;
|
||||
const value = produce();
|
||||
settle();
|
||||
const after = process.memoryUsage().heapUsed;
|
||||
return { bytes: Math.max(0, after - before), value };
|
||||
}
|
||||
|
||||
type Row = { mechanism: string; site: string; bytes: number };
|
||||
|
||||
async function main(): Promise<void> {
|
||||
if (typeof gc !== "function") {
|
||||
console.error(
|
||||
"[heap-bench] refusing to run without --expose-gc: retained-heap numbers would be\n" +
|
||||
" dominated by uncollected garbage and are not comparable across runs.\n" +
|
||||
" Use `npm run bench:heap-body`, which passes it."
|
||||
);
|
||||
process.exitCode = 2;
|
||||
return;
|
||||
}
|
||||
|
||||
const body = buildAgentPayload(MESSAGES, TOOLS, CONTENT_WORDS);
|
||||
const wireBytes = Buffer.byteLength(JSON.stringify(body), "utf8");
|
||||
|
||||
const rows: Row[] = [];
|
||||
const hold: unknown[] = []; // keep results reachable so "retained" means retained
|
||||
|
||||
// 1. The entry-point clone every chat request pays (src/sse/handlers/chat.ts buildClientRawRequest).
|
||||
{
|
||||
const r = measureRetained(() => cloneLogPayload(body));
|
||||
hold.push(r.value);
|
||||
rows.push({ mechanism: "cloneLogPayload (unbounded)", site: "chat.ts buildClientRawRequest", bytes: r.bytes });
|
||||
}
|
||||
|
||||
// 2. What the request logger actually keeps (open-sse/utils/requestLogger.ts).
|
||||
{
|
||||
const r = measureRetained(() => cloneBoundedForLog(body));
|
||||
hold.push(r.value);
|
||||
rows.push({ mechanism: "cloneBoundedForLog (bounded)", site: "requestLogger.logClientRawRequest", bytes: r.bytes });
|
||||
}
|
||||
|
||||
// 3. Combo per-target attempt clones (open-sse/services/combo.ts attemptBody).
|
||||
{
|
||||
const r = measureRetained(() => Array.from({ length: TARGETS }, () => structuredClone(body)));
|
||||
hold.push(r.value);
|
||||
rows.push({ mechanism: `structuredClone x${TARGETS} (combo targets)`, site: "combo.ts attemptBody", bytes: r.bytes });
|
||||
}
|
||||
|
||||
// 4. Whole-body serialization for token estimation (combo.ts estimateTokens(JSON.stringify(...))).
|
||||
{
|
||||
const r = measureRetained(() => JSON.stringify(body));
|
||||
hold.push(r.value);
|
||||
rows.push({ mechanism: "JSON.stringify (token estimate)", site: "combo.ts estimateTokens", bytes: r.bytes });
|
||||
}
|
||||
|
||||
// 5. Overlap: what C concurrent in-flight requests retain via the entry clone alone.
|
||||
const concurrent = measureRetained(() =>
|
||||
Array.from({ length: CONCURRENCY }, () => cloneLogPayload(body))
|
||||
);
|
||||
hold.push(concurrent.value);
|
||||
|
||||
const perRequest = rows.reduce((a, r) => a + r.bytes, 0);
|
||||
|
||||
if (AS_JSON) {
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
shape: { messages: MESSAGES, tools: TOOLS, targets: TARGETS, concurrency: CONCURRENCY },
|
||||
wireBytes,
|
||||
mechanisms: rows,
|
||||
perRequestBytes: perRequest,
|
||||
concurrentEntryCloneBytes: concurrent.bytes,
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
} else {
|
||||
console.log(`# Request-body heap amplification (#7847)\n`);
|
||||
console.log(
|
||||
`Shape: ${MESSAGES} messages · ${TOOLS} tools · wire size **${fmt(wireBytes)} MiB**` +
|
||||
` · ${TARGETS} combo targets\n`
|
||||
);
|
||||
console.log("| mechanism | call site | retained | x wire |");
|
||||
console.log("| --- | --- | ---: | ---: |");
|
||||
for (const r of rows) {
|
||||
console.log(
|
||||
`| ${r.mechanism} | \`${r.site}\` | ${fmt(r.bytes)} MiB | ${(r.bytes / wireBytes).toFixed(2)}x |`
|
||||
);
|
||||
}
|
||||
console.log(
|
||||
`| **per request (sum)** | | **${fmt(perRequest)} MiB** | **${(perRequest / wireBytes).toFixed(2)}x** |`
|
||||
);
|
||||
console.log("");
|
||||
console.log(
|
||||
`${CONCURRENCY} concurrent requests retain **${fmt(concurrent.bytes)} MiB**` +
|
||||
` via the entry clone alone (${(concurrent.bytes / wireBytes).toFixed(2)}x wire).`
|
||||
);
|
||||
console.log("");
|
||||
}
|
||||
|
||||
if (MAX_RETAINED > 0 && perRequest / MIB > MAX_RETAINED) {
|
||||
console.error(
|
||||
`[heap-bench] FAIL — per-request retained ${fmt(perRequest)} MiB exceeds --max-retained-mib ${MAX_RETAINED}`
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
// Referenced after all measurements so V8 cannot collect the holds early and flatter the numbers.
|
||||
if (hold.length === 0) console.log("unreachable");
|
||||
}
|
||||
|
||||
try {
|
||||
await main();
|
||||
} finally {
|
||||
fs.rmSync(TMP_DATA_DIR, { recursive: true, force: true });
|
||||
}
|
||||
69
tests/unit/heap-benchmark-corpus.test.ts
Normal file
69
tests/unit/heap-benchmark-corpus.test.ts
Normal file
@@ -0,0 +1,69 @@
|
||||
// Guards the #7847 heap-benchmark corpus (scripts/perf/agentPayloadCorpus.ts).
|
||||
//
|
||||
// The benchmark's whole value is before/after comparability: if the corpus drifts between runs,
|
||||
// a "10 MiB improvement" could just be a smaller payload. These tests lock the two properties
|
||||
// that comparability depends on — byte-stability across runs, and the incident wire size.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const { buildAgentPayload, INCIDENT_SHAPE } = await import("../../scripts/perf/agentPayloadCorpus.ts");
|
||||
|
||||
const wireBytes = (v: unknown) => Buffer.byteLength(JSON.stringify(v), "utf8");
|
||||
|
||||
test("corpus is byte-identical across repeated builds (no Math.random)", () => {
|
||||
const a = buildAgentPayload(40, 6, 30);
|
||||
const b = buildAgentPayload(40, 6, 30);
|
||||
assert.equal(
|
||||
JSON.stringify(a),
|
||||
JSON.stringify(b),
|
||||
"corpus must be deterministic or before/after heap numbers are not comparable"
|
||||
);
|
||||
});
|
||||
|
||||
test("corpus is byte-identical across separate module instances", async () => {
|
||||
// A fresh import must not reseed differently (e.g. from a module-level counter).
|
||||
const fresh = await import(`../../scripts/perf/agentPayloadCorpus.ts?cachebust=${1}`);
|
||||
assert.equal(
|
||||
JSON.stringify(buildAgentPayload(20, 3, 15)),
|
||||
JSON.stringify(fresh.buildAgentPayload(20, 3, 15))
|
||||
);
|
||||
});
|
||||
|
||||
test("default shape reproduces the #7847 incident (3.05 MiB, 729 messages, 86 tools)", () => {
|
||||
assert.equal(INCIDENT_SHAPE.messages, 729);
|
||||
assert.equal(INCIDENT_SHAPE.tools, 86);
|
||||
|
||||
const body = buildAgentPayload() as { messages: unknown[]; tools: unknown[] };
|
||||
assert.equal(body.messages.length, 729);
|
||||
assert.equal(body.tools.length, 86);
|
||||
|
||||
// The incident payload was 3.05 MiB. Allow a small band so unrelated shape tweaks do not
|
||||
// fail the suite, but catch a drift large enough to invalidate the comparison.
|
||||
const mib = wireBytes(body) / (1024 * 1024);
|
||||
assert.ok(
|
||||
mib > 2.9 && mib < 3.2,
|
||||
`expected ~3.05 MiB to match the incident, got ${mib.toFixed(2)} MiB — recalibrate INCIDENT_SHAPE.contentWords`
|
||||
);
|
||||
});
|
||||
|
||||
test("payload is shaped like a coding-agent request (alternating roles, tool schemas)", () => {
|
||||
const body = buildAgentPayload(6, 2, 5) as {
|
||||
messages: { role: string; content: string }[];
|
||||
tools: { type: string; function: { name: string; parameters: unknown } }[];
|
||||
};
|
||||
assert.deepEqual(
|
||||
body.messages.map((m) => m.role),
|
||||
["user", "assistant", "user", "assistant", "user", "assistant"]
|
||||
);
|
||||
assert.ok(body.messages.every((m) => m.content.length > 0));
|
||||
assert.equal(body.tools[0].type, "function");
|
||||
assert.equal(body.tools[0].function.name, "tool_0");
|
||||
assert.ok(body.tools[0].function.parameters, "tools must carry a JSON schema — they dominate size");
|
||||
});
|
||||
|
||||
test("size scales with the knobs the benchmark exposes", () => {
|
||||
const small = wireBytes(buildAgentPayload(10, 2, 20));
|
||||
assert.ok(wireBytes(buildAgentPayload(20, 2, 20)) > small, "more messages must grow the payload");
|
||||
assert.ok(wireBytes(buildAgentPayload(10, 8, 20)) > small, "more tools must grow the payload");
|
||||
assert.ok(wireBytes(buildAgentPayload(10, 2, 80)) > small, "longer content must grow the payload");
|
||||
});
|
||||
Reference in New Issue
Block a user