fix(logs): read only the log tail on the CLI logs route (#16386)

Thanks @maxmad64bis — the CLI logs route reads a bounded tail and still requires CLI tools auth, via `src/lib/logTail.ts`, covered by `tests/unit/cli-logs-route-bounded-tail.test.ts`.

Boarded on release/v3.8.52 at ac590dafa6. Focused node tests: 74 passed, 0 failed. The #16473 dashboard case is a vitest file and passed 3/3 under vitest. Changelog integrity passed. File-size kept the inherited executeTargetAttempt.ts overage (1288 > 1284). This pull request does not touch that file.

⚠️ base-red inherited: #15306
This commit is contained in:
Dizzle
2026-10-13 14:40:16 +02:00
committed by GitHub
parent 981ef3a888
commit e2d5747872
4 changed files with 556 additions and 5 deletions

View File

@@ -0,0 +1 @@
- **fix(logs):** Bound the CLI logs route to a tail read so large log files no longer stall served requests ([#16386](https://github.com/diegosouzapw/OmniRoute/pull/16386)) — thanks @maxmad64bis

View File

@@ -22,10 +22,11 @@
*/
import { NextRequest, NextResponse } from "next/server";
import { readFileSync, existsSync } from "fs";
import fs from "node:fs";
import { getAppLogFilePath } from "@/lib/logEnv";
import { requireCliToolsAuth } from "@/lib/api/requireCliToolsAuth";
import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error.ts";
import { readTailLines } from "@/lib/logTail";
// Map pino numeric levels to string levels
const NUMERIC_LEVEL_MAP: Record<number, string> = {
@@ -59,6 +60,27 @@ function stringifyLogValue(value: unknown): string {
}
}
/**
* Returns true when a parsed log entry passes the route's hour and token gates.
*/
function matchesCliEntry(
entry: Record<string, unknown>,
filterTokens: string[],
oneHourAgo: number
): boolean {
const ts = entry.time || entry.timestamp;
if (ts) {
const entryTime = new Date(ts as string | number).getTime();
if (Number.isFinite(entryTime) && entryTime < oneHourAgo) return false;
}
const component = stringifyLogValue(entry.component);
const mod = stringifyLogValue(entry.module);
const msg = stringifyLogValue(entry.msg ?? entry.message ?? "");
if (filterTokens.length === 0) return true;
const haystack = `${component} ${mod} ${msg}`.toLowerCase();
return filterTokens.some((token) => haystack.includes(token));
}
/**
* GET /api/cli-tools/logs
*/
@@ -85,14 +107,20 @@ export async function GET(request: NextRequest) {
const logPath = getAppLogFilePath();
if (!existsSync(logPath)) {
if (!fs.existsSync(logPath)) {
return NextResponse.json([], { status: 200 });
}
const raw = readFileSync(logPath, "utf-8");
const lines = raw.trim().split("\n").filter(Boolean);
const oneHourAgo = Date.now() - 60 * 60 * 1000;
const size = fs.statSync(logPath).size;
const lines = readTailLines(
logPath,
size,
limit,
(entry) => matchesCliEntry(entry, filterTokens, oneHourAgo),
oneHourAgo
);
const entries: Record<string, unknown>[] = [];
for (const line of lines) {

90
src/lib/logTail.ts Normal file
View File

@@ -0,0 +1,90 @@
/**
* Bounded tail reader for a log-serving route, adopted by the CLI logs route.
*
* Reads the end of a log file through doubling windows from the end instead
* of loading the whole file, until the requested limit is covered or the
* oldest dated entry in the window is older than the requested hour.
* Worst case (a filter matched only at the top of the file) reads the whole
* file, which matches the previous full-read behavior.
*/
import fs from "node:fs";
export const TAIL_INITIAL_BYTES = 256 * 1024;
export function oldestDatedStamp(entries: Record<string, unknown>[]): number | null {
let oldest: number | null = null;
for (const entry of entries) {
const ts = entry.time || entry.timestamp;
if (!ts) continue;
const stamp = new Date(ts as string | number).getTime();
if (!Number.isFinite(stamp)) continue;
if (oldest === null || stamp < oldest) oldest = stamp;
}
return oldest;
}
function parseWindow(
fd: number,
start: number,
length: number,
matches: (entry: Record<string, unknown>) => boolean
): { lines: string[]; matched: number; parsed: Record<string, unknown>[] } {
const buffer = Buffer.alloc(length);
let read = 0;
while (read < length) {
const n = fs.readSync(fd, buffer, read, length - read, start + read);
if (n === 0) break;
read += n;
}
const text = buffer.subarray(0, read).toString("utf-8");
const chunks = text.split("\n");
if (start > 0) chunks.shift();
const lines = chunks.filter(Boolean);
let matched = 0;
const parsed: Record<string, unknown>[] = [];
for (const line of lines) {
let entry: Record<string, unknown> | null = null;
try {
entry = JSON.parse(line) as Record<string, unknown>;
} catch {
continue;
}
parsed.push(entry);
if (matches(entry)) matched += 1;
}
return { lines, matched, parsed };
}
export function readTailLines(
logPath: string,
size: number,
limit: number,
matches: (entry: Record<string, unknown>) => boolean,
oneHourAgo: number
): string[] {
const fd = fs.openSync(logPath, "r");
try {
let windowBytes = Math.min(TAIL_INITIAL_BYTES, size);
let lines: string[] = [];
for (;;) {
const start = Math.max(size - windowBytes, 0);
const parsedWindow = parseWindow(fd, start, size - start, matches);
lines = parsedWindow.lines;
const oldest = oldestDatedStamp(parsedWindow.parsed);
if (
parsedWindow.matched >= limit ||
(oldest !== null && oldest < oneHourAgo) ||
start === 0
) {
return lines;
}
if (windowBytes >= size) return lines;
windowBytes = Math.min(windowBytes * 2, size);
}
} finally {
fs.closeSync(fd);
}
}

View File

@@ -0,0 +1,432 @@
/**
* Bounded tail read for /api/cli-tools/logs (perf: large log files must not
* stall served requests).
*
* Proves:
* - A 50 MB file with limit 50 reads only the tail and returns 50 lines.
* - A short file returns the same lines as a full read.
* - A short file with small limit, filter and hour window matches a full read.
* - A filter matched only higher up widens the window and returns the lines.
* - A missing file returns an empty list.
* - The hour filter holds on a 25-hour day.
* - Multibyte characters split by the window edge decode without replacement.
* - An object component matches its JSON shape and a numeric msg matches.
*/
import test, { mock } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { updateSettings } from "../../src/lib/db/settings";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-cli-logs-tail-"));
process.env.DATA_DIR = TEST_DATA_DIR;
const tailLogDir = path.join(TEST_DATA_DIR, "logs");
fs.mkdirSync(tailLogDir, { recursive: true });
const tailLogPath = path.join(tailLogDir, "tail-app.log");
const origAppLogPath = process.env.APP_LOG_FILE_PATH;
process.env.APP_LOG_FILE_PATH = tailLogPath;
const { GET } = await import("../../src/app/api/cli-tools/logs/route.ts");
function makeReq(queryString = "") {
const url = `http://localhost/api/cli-tools/logs${queryString ? `?${queryString}` : ""}`;
return new Request(url);
}
test.before(async () => {
await updateSettings({ requireLogin: false });
});
test.after(async () => {
await updateSettings({ requireLogin: true });
process.env.APP_LOG_FILE_PATH = origAppLogPath;
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
mock.restoreAll();
});
test("50 MB file with limit 50 reads only the tail and returns 50 lines", async (t) => {
const now = Date.now();
// ~200 bytes per line, ~250k lines -> ~50 MB.
const pad = "x".repeat(150);
const total = 250_000;
const fd = fs.openSync(tailLogPath, "w");
try {
for (let i = 0; i < total; i++) {
fs.writeSync(
fd,
`${JSON.stringify({ level: 30, msg: `tail entry ${i} ${pad}`, component: "router", time: now })}\n`
);
}
} finally {
fs.closeSync(fd);
}
const size = fs.statSync(tailLogPath).size;
assert.ok(size > 50 * 1024 * 1024, `fixture should exceed 50 MB, got ${size}`);
// Spy the fd-level reads on the same `fs` object the tail reader uses.
// The base route never calls openSync/readSync (it uses readFileSync),
// so the call-count assertion below fails on base (RED) and the byte
// assertion proves the window stays bounded with the change (GREEN).
const realOpenSync = fs.openSync.bind(fs);
const realReadSync = fs.readSync.bind(fs);
const tailedFds = new Set<number>();
let readCalls = 0;
let largestWindow = 0;
t.mock.method(fs, "openSync", (...args: unknown[]) => {
const fd = (realOpenSync as (...a: never[]) => number)(...(args as never[]));
if (args[0] === tailLogPath) tailedFds.add(fd);
return fd as never;
});
t.mock.method(fs, "readSync", (...args: unknown[]) => {
const n = (realReadSync as (...a: never[]) => number)(...(args as never[]));
if (typeof args[0] === "number" && tailedFds.has(args[0])) {
readCalls += 1;
if (n > largestWindow) largestWindow = n;
}
return n as never;
});
try {
const res = await GET(makeReq("limit=50"));
assert.equal(res.status, 200);
const body = await res.json();
assert.ok(Array.isArray(body));
assert.equal(body.length, 50);
assert.equal(body[0].msg, `tail entry ${total - 50} ${pad}`);
assert.equal(body[49].msg, `tail entry ${total - 1} ${pad}`);
assert.ok(readCalls > 0, "tail path must read through openSync/readSync");
assert.ok(
largestWindow < size,
`bounded read must load fewer bytes than the file size (${largestWindow} < ${size})`
);
} finally {
mock.restoreAll();
fs.rmSync(tailLogPath, { force: true });
}
});
test("short file returns the same lines as a full read", async () => {
const now = Date.now();
const fixture = [
JSON.stringify({ level: 30, msg: "alpha", component: "router", time: now }),
JSON.stringify({ level: 40, msg: "beta", module: "auth", time: now }),
JSON.stringify({ level: 20, msg: "no timestamp here", component: "debug" }),
"not-valid-json-should-be-skipped",
JSON.stringify({ level: 30, msg: "gamma", component: "router", time: now }),
];
try {
fs.writeFileSync(tailLogPath, `${fixture.join("\n")}\n`, "utf-8");
const res = await GET(makeReq("limit=50"));
assert.equal(res.status, 200);
const body = await res.json();
// Oracle: the same full-read semantics the route applied before the change.
const raw = fs.readFileSync(tailLogPath, "utf-8");
const oracle = raw
.trim()
.split("\n")
.filter(Boolean)
.map((line) => {
try {
return JSON.parse(line);
} catch {
return null;
}
})
.filter((e) => e !== null)
.slice(-50);
assert.deepEqual(
body.map((e: { msg?: string }) => e.msg),
oracle.map((e: { msg?: string }) => e.msg)
);
assert.equal(body.length, 4);
} finally {
fs.rmSync(tailLogPath, { force: true });
}
});
test("filter matched only higher up widens the window and returns the lines", async (t) => {
const now = Date.now();
const pad = "y".repeat(400);
// Old top: noise that forces the first window to miss, then 60 small needle
// lines. Recent: ~500 KiB of noise. The second window (512 KiB) covers the
// noise plus all needles, so growth stops before the whole file.
const top: string[] = [];
for (let i = 0; i < 1500; i++) {
top.push(
JSON.stringify({
level: 30,
msg: `old noise ${i} ${pad}`,
component: "router",
time: now - 30 * 60 * 1000,
})
);
}
const needles: string[] = [];
for (let i = 0; i < 60; i++) {
needles.push(
JSON.stringify({
level: 30,
msg: `needle match ${i}`,
component: "needle",
time: now - 30 * 60 * 1000,
})
);
}
const recent: string[] = [];
for (let i = 0; i < 1100; i++) {
recent.push(
JSON.stringify({ level: 30, msg: `recent noise ${i} ${pad}`, component: "router", time: now })
);
}
try {
fs.writeFileSync(tailLogPath, `${[...top, ...needles, ...recent].join("\n")}\n`, "utf-8");
const size = fs.statSync(tailLogPath).size;
assert.ok(size > 768 * 1024, `fixture must exceed two windows, got ${size}`);
const realOpenSync = fs.openSync.bind(fs);
const realReadSync = fs.readSync.bind(fs);
const tailedFds = new Set<number>();
let readCalls = 0;
let largestWindow = 0;
t.mock.method(fs, "openSync", (...args: unknown[]) => {
const fd = (realOpenSync as (...a: never[]) => number)(...(args as never[]));
if (args[0] === tailLogPath) tailedFds.add(fd);
return fd as never;
});
t.mock.method(fs, "readSync", (...args: unknown[]) => {
const n = (realReadSync as (...a: never[]) => number)(...(args as never[]));
if (typeof args[0] === "number" && tailedFds.has(args[0])) {
readCalls += 1;
// One window read issues one large readSync; the largest single read
// is the window size, not the cumulative overlap across windows.
if (n > largestWindow) largestWindow = n;
}
return n as never;
});
const res = await GET(makeReq("filter=needle&limit=50"));
assert.equal(res.status, 200);
const body = await res.json();
assert.ok(Array.isArray(body));
assert.equal(body.length, 50);
assert.ok(body.every((e: { component?: string }) => (e.component || "").includes("needle")));
assert.ok(readCalls > 1, `window must widen past the first window (${readCalls} reads)`);
assert.ok(
largestWindow > 256 * 1024,
`window must widen past the initial 256 KiB window (largest read ${largestWindow})`
);
assert.ok(
largestWindow < size,
`widened window must still stay below the full file size (${largestWindow} < ${size})`
);
} finally {
mock.restoreAll();
fs.rmSync(tailLogPath, { force: true });
}
});
test("missing file returns an empty list", async () => {
const saved = process.env.APP_LOG_FILE_PATH;
process.env.APP_LOG_FILE_PATH = path.join(TEST_DATA_DIR, "no-such-file.log");
try {
const res = await GET(makeReq());
assert.equal(res.status, 200);
const body = await res.json();
assert.ok(Array.isArray(body));
assert.equal(body.length, 0);
} finally {
process.env.APP_LOG_FILE_PATH = saved;
}
});
test("hour filter holds on a 25-hour day", async () => {
const realDateNow = Date.now;
// Fixed date on a DST fallback day in Europe/Paris: 2026-10-25 03:00 CEST
// becomes 02:00 CET, a 25-hour day. TZ is set in this process below.
const fixedNow = Date.parse("2026-10-25T01:30:00.000Z");
process.env.TZ = "Europe/Paris";
Date.now = () => fixedNow;
try {
const lines = [
JSON.stringify({
level: 30,
msg: "fresh",
component: "router",
time: fixedNow - 10 * 60 * 1000,
}),
JSON.stringify({
level: 30,
msg: "stale",
component: "router",
time: fixedNow - 2 * 60 * 60 * 1000,
}),
JSON.stringify({ level: 30, msg: "nodate", component: "router" }),
];
fs.writeFileSync(tailLogPath, `${lines.join("\n")}\n`, "utf-8");
const res = await GET(makeReq("limit=50"));
assert.equal(res.status, 200);
const body = await res.json();
const msgs = body.map((e: { msg?: string }) => e.msg);
assert.ok(msgs.includes("fresh"), "entry inside the hour is kept");
assert.ok(msgs.includes("nodate"), "undated entry is kept");
assert.ok(!msgs.includes("stale"), "entry older than one hour is dropped");
} finally {
Date.now = realDateNow;
delete process.env.TZ;
fs.rmSync(tailLogPath, { force: true });
}
});
test("multibyte characters split by the window edge decode without replacement", async () => {
const now = Date.now();
const lines = [
JSON.stringify({ level: 30, msg: "caf\u00e9 au lait", component: "d\u00e9ficit", time: now }),
JSON.stringify({ level: 30, msg: "plain", component: "router", time: now }),
];
try {
fs.writeFileSync(tailLogPath, `${lines.join("\n")}\n`, "utf-8");
const res = await GET(makeReq("limit=50"));
assert.equal(res.status, 200);
const body = await res.json();
const text = JSON.stringify(body);
assert.ok(!text.includes("<22>"), "response must not contain replacement characters");
assert.ok(
body.some((e: { msg?: string }) => (e.msg || "").includes("caf\u00e9")),
"multibyte content survives the tail read"
);
} finally {
fs.rmSync(tailLogPath, { force: true });
}
});
test("object component matches its JSON shape and numeric msg matches", async () => {
const now = Date.now();
const lines = [
JSON.stringify({ level: 30, component: { x: 1 }, msg: "hello", time: now }),
JSON.stringify({ level: 30, component: "router", msg: 12345, time: now }),
JSON.stringify({ level: 30, component: "other", msg: "unrelated", time: now }),
];
try {
fs.writeFileSync(tailLogPath, `${lines.join("\n")}\n`, "utf-8");
const resX = await GET(makeReq("filter=x"));
assert.equal(resX.status, 200);
const bodyX = await resX.json();
assert.ok(
bodyX.some((e: { msg?: string }) => e.msg === "hello"),
"object component entry matches its JSON shape"
);
const resNum = await GET(makeReq("filter=123"));
assert.equal(resNum.status, 200);
const bodyNum = await resNum.json();
assert.ok(
bodyNum.some((e: { msg?: string }) => String(e.msg).includes("123")),
"numeric msg entry matches"
);
} finally {
fs.rmSync(tailLogPath, { force: true });
}
});
test("dated stamp helper keeps the oldest finite stamp and ignores the rest", async () => {
// Direct reader for the exported helper: entries here are plain values, so a
// dynamic import keeps this the only case that fails without the change.
const { oldestDatedStamp } = await import("../../src/lib/logTail.ts");
const now = Date.now();
assert.equal(oldestDatedStamp([]), null);
assert.equal(
oldestDatedStamp([{ component: "router" }, { time: "not-a-date" }, { time: Number.NaN }]),
null
);
assert.equal(
oldestDatedStamp([
{ time: now - 10 * 60 * 1000 },
{ time: now - 50 * 60 * 1000 },
{ timestamp: now - 5 * 60 * 1000 },
{ time: { nested: true } },
]),
now - 50 * 60 * 1000
);
});
test("short file with small limit filter and hour window matches a full read", async () => {
const now = Date.now();
const oneHourAgo = now - 60 * 60 * 1000;
const fixture = [
JSON.stringify({
level: 30,
msg: "old needle one",
component: "needle",
time: now - 2 * 60 * 60 * 1000,
}),
JSON.stringify({ level: 30, msg: "recent other", component: "router", time: now }),
JSON.stringify({ level: 30, msg: "recent needle one", component: "needle", time: now }),
JSON.stringify({ level: 30, msg: "recent needle two", module: "needle", time: now }),
JSON.stringify({ level: 30, msg: "recent other two", component: "router", time: now }),
JSON.stringify({ level: 30, msg: "recent needle three", component: "needle", time: now }),
JSON.stringify({ level: 30, msg: "undated needle", component: "needle" }),
];
try {
fs.writeFileSync(tailLogPath, `${fixture.join("\n")}\n`, "utf-8");
const res = await GET(makeReq("filter=needle&limit=2"));
assert.equal(res.status, 200);
const body = await res.json();
assert.ok(Array.isArray(body));
// Oracle: the previous full-read semantics (whole file, then the same hour,
// token and slice gates the route applies), so the bounded tail must render
// exactly what the full read rendered.
const raw = fs.readFileSync(tailLogPath, "utf-8");
const oracle = raw
.trim()
.split("\n")
.filter(Boolean)
.map((line) => {
try {
return JSON.parse(line);
} catch {
return null;
}
})
.filter((e) => e !== null)
.filter((e: { time?: unknown; timestamp?: unknown }) => {
const ts = e.time || e.timestamp;
if (ts) {
const entryTime = new Date(ts as string | number).getTime();
if (entryTime < oneHourAgo) return false;
}
return true;
})
.filter((e: { component?: unknown; module?: unknown; msg?: unknown; message?: unknown }) => {
const haystack = [e.component || "", e.module || "", e.msg ?? e.message ?? ""]
.map(String)
.join(" ")
.toLowerCase();
return ["needle"].some((token) => haystack.includes(token));
})
.slice(-2);
assert.deepEqual(
body.map((e: { msg?: string }) => e.msg),
oracle.map((e: { msg?: string }) => String(e.msg ?? e.message ?? ""))
);
// Hand-computed expectation: the slice keeps the last two needle matches,
// the hour gate drops the old one, the missing stamp keeps the undated one.
assert.equal(body.length, 2);
assert.deepEqual(
body.map((e: { msg?: string }) => e.msg),
["recent needle three", "undated needle"]
);
} finally {
fs.rmSync(tailLogPath, { force: true });
}
});