chore(perf): add deterministic request-body heap benchmark (#7847) (#8549)

#7847 reports a 3.05 MiB request (729 messages / 86 tools) reaching ~12,282 MiB of V8
heap, and asks for "a regression benchmark that records peak heap for representative
500-800-message, tool-rich requests" before any fix lands. There is currently no memory
baseline in the repo at all (bench:compression is the only benchmark), so a clone-reduction
change could neither be justified nor regression-guarded.

npm run bench:heap-body attributes retained heap to each copy the chat path makes:

  | mechanism                          | call site                          | retained | x wire |
  | cloneLogPayload (unbounded)        | chat.ts buildClientRawRequest      | 3.18 MiB |  1.04x |
  | cloneBoundedForLog (bounded)       | requestLogger.logClientRawRequest  | 0.04 MiB |  0.01x |
  | structuredClone x3 (combo targets) | combo.ts attemptBody               | 9.53 MiB |  3.12x |
  | JSON.stringify (token estimate)    | combo.ts estimateTokens            | 3.06 MiB |  1.00x |
  | per request (sum)                  |                                    |15.81 MiB |  5.17x |

It measures the real production helpers rather than reimplementations, so a change to the
log bounds or the clone strategy is reflected directly.

Design notes:
- Deterministic: fixed-seed LCG, no Math.random(). Verified byte-identical across three
  consecutive runs — without that, a before/after delta measures noise, not the change.
- Corpus lives in its own side-effect-free module so the unit test can import it without
  booting SQLite (requestLogger transitively opens the DB at import time).
- Hermetic: DATA_DIR is redirected to a temp dir before importing, so the benchmark never
  touches the operator's real ~/.omniroute store.
- Node, not bun: --expose-gc and V8 heap accounting are the measurement; another engine's
  heap number would not describe the production runtime.
- --max-retained-mib exits non-zero, so this can become a CI gate once a target is agreed.

Reports only; wires nothing into CI and changes no production code.
This commit is contained in:
MumuTW
2026-07-26 14:52:42 +08:00
committed by GitHub
parent 1cfbfc044a
commit 4bf47c9b80
5 changed files with 354 additions and 0 deletions

View File

@@ -0,0 +1 @@
- chore(perf): add `npm run bench:heap-body` — a deterministic benchmark that attributes retained V8 heap to each request-body copy on the chat path (entry log clone, combo per-target clone, token-estimation string), reproducing the #7847 incident shape (3.05 MiB / 729 messages / 86 tools) so the clone-amplification work can be justified and regression-guarded with numbers instead of intuition

View File

@@ -79,6 +79,7 @@
"prebuild:docs": "node scripts/docs/gen-openapi-module.mjs",
"gen:provider-reference": "bun scripts/docs/gen-provider-reference.ts",
"bench:compression": "bun scripts/compression/benchmark.ts",
"bench:heap-body": "node --expose-gc --import tsx/esm scripts/perf/request-body-heap.ts",
"eval:compression": "node --import tsx scripts/compression-eval/index.ts",
"eval:router": "node --import tsx scripts/router-eval/index.ts",
"eval:router:compare": "node --import tsx scripts/router-eval/compare.ts",

View File

@@ -0,0 +1,83 @@
/**
* Deterministic coding-agent request corpus for the #7847 heap benchmark.
*
* Kept separate from `request-body-heap.ts` so it can be imported with zero side effects:
* the benchmark redirects DATA_DIR and transitively boots SQLite at import time, which a unit
* test must not do just to assert corpus stability.
*
* Determinism is the point. A benchmark corpus built with Math.random() would make a
* before/after comparison measure noise instead of the change under test, so the generator uses
* a fixed-seed LCG and is covered by tests/unit/heap-benchmark-corpus.test.ts.
*/
/** Defaults reproduce the #7847 production incident: 3.05 MiB, 729 messages, 86 tools. */
export const INCIDENT_SHAPE = {
messages: 729,
tools: 86,
/** Calibrated so the defaults land on the incident's wire size. */
contentWords: 527,
} as const;
function lcg(seed: number): () => number {
let s = seed >>> 0;
return () => ((s = (s * 1664525 + 1013904223) >>> 0) / 0x100000000);
}
const WORDS = [
"refactor",
"handler",
"provider",
"combo",
"stream",
"token",
"context",
"payload",
"upstream",
"fallback",
"quota",
"retry",
"executor",
"translate",
"compression",
"cache",
];
function sentence(rnd: () => number, words: number): string {
const out: string[] = [];
for (let i = 0; i < words; i++) out.push(WORDS[Math.floor(rnd() * WORDS.length)]);
return out.join(" ");
}
/** A coding-agent shaped request: long alternating history plus a large tool catalog. */
export function buildAgentPayload(
messages: number = INCIDENT_SHAPE.messages,
tools: number = INCIDENT_SHAPE.tools,
contentWords: number = INCIDENT_SHAPE.contentWords
): Record<string, unknown> {
const rnd = lcg(0x5eed);
return {
model: "claude-opus-5",
stream: true,
messages: Array.from({ length: messages }, (_, i) => ({
role: i % 2 === 0 ? "user" : "assistant",
content: sentence(rnd, contentWords),
})),
tools: Array.from({ length: tools }, (_, i) => ({
type: "function",
function: {
name: `tool_${i}`,
description: sentence(rnd, 25),
parameters: {
type: "object",
properties: Object.fromEntries(
Array.from({ length: 8 }, (_, p) => [
`param_${p}`,
{ type: "string", description: sentence(rnd, 10) },
])
),
required: ["param_0"],
},
},
})),
};
}

View File

@@ -0,0 +1,200 @@
/**
* Request-body heap amplification benchmark (#7847).
*
* #7847 reports a 3.05 MiB request with 729 messages and 86 tools reaching ~12,282 MiB of V8
* heap. The wire size does not explain the peak: the same logical body is retained several times
* over (entry-point log clone, per-combo-target clone, token-estimation string, pending-request
* state). This benchmark attributes retained heap to each of those mechanisms individually, so a
* fix can be justified by numbers instead of intuition — and so a regression can be caught later.
*
* It measures the real production helpers (no reimplementation): `cloneLogPayload` is what
* `buildClientRawRequest` calls per request, and `cloneBoundedForLog` is what the request logger
* actually retains downstream.
*
* Deterministic and API-free (no network, no upstream credentials). The DATA_DIR is redirected to
* a temp dir before importing, because the request-logger module opens the SQLite database on
* import — the benchmark must never touch the operator's real ~/.omniroute store.
*
* Node only — NOT bun. We need `--expose-gc` and V8 heap accounting; measuring "V8 heap" under a
* different engine would report a number that has nothing to do with the production runtime.
*
* Usage:
* npm run bench:heap-body # the #7847 incident shape
* npm run bench:heap-body -- --messages 800 --tools 120
* npm run bench:heap-body -- --concurrency 16 # simulate overlapping requests
* npm run bench:heap-body -- --json # machine-readable
* npm run bench:heap-body -- --max-retained-mib 64 # non-zero exit if exceeded (regression gate)
*/
import os from "node:os";
import path from "node:path";
import fs from "node:fs";
// Must happen BEFORE the dynamic imports below: open-sse/utils/requestLogger.ts transitively
// opens the SQLite store at import time, and the benchmark must stay hermetic.
const TMP_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-heapbench-"));
process.env.DATA_DIR = TMP_DATA_DIR;
// open-sse/utils/requestLogger.ts imports @/lib/usage/usageHistory, so merely importing the
// (pure) log-shaping helper boots SQLite and runs every migration against the temp DATA_DIR.
// That noise would bury the report, so console output is parked for the duration of the import.
// The measurements are unaffected: every baseline is taken after imports have settled.
const { buildAgentPayload, INCIDENT_SHAPE } = await import("./agentPayloadCorpus.ts");
const realLog = console.log;
console.log = () => {};
const { cloneLogPayload } = await import("../../src/lib/logPayloads.ts");
const { cloneBoundedForLog } = await import("../../open-sse/utils/requestLogger.ts");
console.log = realLog;
// ── CLI ──────────────────────────────────────────────────────────────────────
function numArg(flag: string, fallback: number): number {
const i = process.argv.indexOf(flag);
if (i === -1) return fallback;
const v = Number(process.argv[i + 1]);
return Number.isFinite(v) ? v : fallback;
}
const HAS = (flag: string) => process.argv.includes(flag);
// Defaults mirror the #7847 production incident.
const MESSAGES = numArg("--messages", INCIDENT_SHAPE.messages);
const TOOLS = numArg("--tools", INCIDENT_SHAPE.tools);
// Calibrated so the defaults land on the incident's 3.05 MiB wire size.
const CONTENT_WORDS = numArg("--content-words", INCIDENT_SHAPE.contentWords);
const TARGETS = numArg("--targets", 3); // combo targets -> attemptBody clones
const CONCURRENCY = numArg("--concurrency", 8);
const MAX_RETAINED = numArg("--max-retained-mib", 0); // 0 = report only
const AS_JSON = HAS("--json");
const MIB = 1024 * 1024;
const fmt = (bytes: number) => (bytes / MIB).toFixed(2);
// ── Measurement ──────────────────────────────────────────────────────────────
const gc = globalThis.gc as undefined | (() => void);
function settle(): void {
// Several passes: one gc() does not reliably collect everything in a young generation.
for (let i = 0; i < 4; i++) gc?.();
}
/**
* Retained heap of whatever `produce` returns, while it stays reachable.
* This is the number that maps to the incident: many concurrent requests each holding copies.
*/
function measureRetained<T>(produce: () => T): { bytes: number; value: T } {
settle();
const before = process.memoryUsage().heapUsed;
const value = produce();
settle();
const after = process.memoryUsage().heapUsed;
return { bytes: Math.max(0, after - before), value };
}
type Row = { mechanism: string; site: string; bytes: number };
async function main(): Promise<void> {
if (typeof gc !== "function") {
console.error(
"[heap-bench] refusing to run without --expose-gc: retained-heap numbers would be\n" +
" dominated by uncollected garbage and are not comparable across runs.\n" +
" Use `npm run bench:heap-body`, which passes it."
);
process.exitCode = 2;
return;
}
const body = buildAgentPayload(MESSAGES, TOOLS, CONTENT_WORDS);
const wireBytes = Buffer.byteLength(JSON.stringify(body), "utf8");
const rows: Row[] = [];
const hold: unknown[] = []; // keep results reachable so "retained" means retained
// 1. The entry-point clone every chat request pays (src/sse/handlers/chat.ts buildClientRawRequest).
{
const r = measureRetained(() => cloneLogPayload(body));
hold.push(r.value);
rows.push({ mechanism: "cloneLogPayload (unbounded)", site: "chat.ts buildClientRawRequest", bytes: r.bytes });
}
// 2. What the request logger actually keeps (open-sse/utils/requestLogger.ts).
{
const r = measureRetained(() => cloneBoundedForLog(body));
hold.push(r.value);
rows.push({ mechanism: "cloneBoundedForLog (bounded)", site: "requestLogger.logClientRawRequest", bytes: r.bytes });
}
// 3. Combo per-target attempt clones (open-sse/services/combo.ts attemptBody).
{
const r = measureRetained(() => Array.from({ length: TARGETS }, () => structuredClone(body)));
hold.push(r.value);
rows.push({ mechanism: `structuredClone x${TARGETS} (combo targets)`, site: "combo.ts attemptBody", bytes: r.bytes });
}
// 4. Whole-body serialization for token estimation (combo.ts estimateTokens(JSON.stringify(...))).
{
const r = measureRetained(() => JSON.stringify(body));
hold.push(r.value);
rows.push({ mechanism: "JSON.stringify (token estimate)", site: "combo.ts estimateTokens", bytes: r.bytes });
}
// 5. Overlap: what C concurrent in-flight requests retain via the entry clone alone.
const concurrent = measureRetained(() =>
Array.from({ length: CONCURRENCY }, () => cloneLogPayload(body))
);
hold.push(concurrent.value);
const perRequest = rows.reduce((a, r) => a + r.bytes, 0);
if (AS_JSON) {
console.log(
JSON.stringify(
{
shape: { messages: MESSAGES, tools: TOOLS, targets: TARGETS, concurrency: CONCURRENCY },
wireBytes,
mechanisms: rows,
perRequestBytes: perRequest,
concurrentEntryCloneBytes: concurrent.bytes,
},
null,
2
)
);
} else {
console.log(`# Request-body heap amplification (#7847)\n`);
console.log(
`Shape: ${MESSAGES} messages · ${TOOLS} tools · wire size **${fmt(wireBytes)} MiB**` +
` · ${TARGETS} combo targets\n`
);
console.log("| mechanism | call site | retained | x wire |");
console.log("| --- | --- | ---: | ---: |");
for (const r of rows) {
console.log(
`| ${r.mechanism} | \`${r.site}\` | ${fmt(r.bytes)} MiB | ${(r.bytes / wireBytes).toFixed(2)}x |`
);
}
console.log(
`| **per request (sum)** | | **${fmt(perRequest)} MiB** | **${(perRequest / wireBytes).toFixed(2)}x** |`
);
console.log("");
console.log(
`${CONCURRENCY} concurrent requests retain **${fmt(concurrent.bytes)} MiB**` +
` via the entry clone alone (${(concurrent.bytes / wireBytes).toFixed(2)}x wire).`
);
console.log("");
}
if (MAX_RETAINED > 0 && perRequest / MIB > MAX_RETAINED) {
console.error(
`[heap-bench] FAIL — per-request retained ${fmt(perRequest)} MiB exceeds --max-retained-mib ${MAX_RETAINED}`
);
process.exitCode = 1;
}
// Referenced after all measurements so V8 cannot collect the holds early and flatter the numbers.
if (hold.length === 0) console.log("unreachable");
}
try {
await main();
} finally {
fs.rmSync(TMP_DATA_DIR, { recursive: true, force: true });
}

View File

@@ -0,0 +1,69 @@
// Guards the #7847 heap-benchmark corpus (scripts/perf/agentPayloadCorpus.ts).
//
// The benchmark's whole value is before/after comparability: if the corpus drifts between runs,
// a "10 MiB improvement" could just be a smaller payload. These tests lock the two properties
// that comparability depends on — byte-stability across runs, and the incident wire size.
import { test } from "node:test";
import assert from "node:assert/strict";
const { buildAgentPayload, INCIDENT_SHAPE } = await import("../../scripts/perf/agentPayloadCorpus.ts");
const wireBytes = (v: unknown) => Buffer.byteLength(JSON.stringify(v), "utf8");
test("corpus is byte-identical across repeated builds (no Math.random)", () => {
const a = buildAgentPayload(40, 6, 30);
const b = buildAgentPayload(40, 6, 30);
assert.equal(
JSON.stringify(a),
JSON.stringify(b),
"corpus must be deterministic or before/after heap numbers are not comparable"
);
});
test("corpus is byte-identical across separate module instances", async () => {
// A fresh import must not reseed differently (e.g. from a module-level counter).
const fresh = await import(`../../scripts/perf/agentPayloadCorpus.ts?cachebust=${1}`);
assert.equal(
JSON.stringify(buildAgentPayload(20, 3, 15)),
JSON.stringify(fresh.buildAgentPayload(20, 3, 15))
);
});
test("default shape reproduces the #7847 incident (3.05 MiB, 729 messages, 86 tools)", () => {
assert.equal(INCIDENT_SHAPE.messages, 729);
assert.equal(INCIDENT_SHAPE.tools, 86);
const body = buildAgentPayload() as { messages: unknown[]; tools: unknown[] };
assert.equal(body.messages.length, 729);
assert.equal(body.tools.length, 86);
// The incident payload was 3.05 MiB. Allow a small band so unrelated shape tweaks do not
// fail the suite, but catch a drift large enough to invalidate the comparison.
const mib = wireBytes(body) / (1024 * 1024);
assert.ok(
mib > 2.9 && mib < 3.2,
`expected ~3.05 MiB to match the incident, got ${mib.toFixed(2)} MiB — recalibrate INCIDENT_SHAPE.contentWords`
);
});
test("payload is shaped like a coding-agent request (alternating roles, tool schemas)", () => {
const body = buildAgentPayload(6, 2, 5) as {
messages: { role: string; content: string }[];
tools: { type: string; function: { name: string; parameters: unknown } }[];
};
assert.deepEqual(
body.messages.map((m) => m.role),
["user", "assistant", "user", "assistant", "user", "assistant"]
);
assert.ok(body.messages.every((m) => m.content.length > 0));
assert.equal(body.tools[0].type, "function");
assert.equal(body.tools[0].function.name, "tool_0");
assert.ok(body.tools[0].function.parameters, "tools must carry a JSON schema — they dominate size");
});
test("size scales with the knobs the benchmark exposes", () => {
const small = wireBytes(buildAgentPayload(10, 2, 20));
assert.ok(wireBytes(buildAgentPayload(20, 2, 20)) > small, "more messages must grow the payload");
assert.ok(wireBytes(buildAgentPayload(10, 8, 20)) > small, "more tools must grow the payload");
assert.ok(wireBytes(buildAgentPayload(10, 2, 80)) > small, "longer content must grow the payload");
});