mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-22 04:19:40 +00:00
fix(memory): skip a failing rerank model during a backoff window (#20535)
This commit is contained in:
1
changelog.d/fixes/20535-memory-rerank-failure-breaker.md
Normal file
1
changelog.d/fixes/20535-memory-rerank-failure-breaker.md
Normal file
@@ -0,0 +1 @@
|
||||
- fix(memory): skip a failing rerank model for a backoff window (60 s up to 10 min) instead of retrying it on every retrieval (#20535)
|
||||
32
src/lib/memory/rerankBreaker.ts
Normal file
32
src/lib/memory/rerankBreaker.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* Per-model failure cooldown for the memory rerank step (#20535).
|
||||
*
|
||||
* A rerank model that 404s / times out used to be retried on every retrieval. After a
|
||||
* failure we skip the rerank call until `until`, doubling the window on consecutive
|
||||
* failures (60 s -> 10 min cap). The first success clears the entry.
|
||||
*/
|
||||
const BASE_COOLDOWN_MS = 60_000;
|
||||
const MAX_COOLDOWN_MS = 10 * 60_000;
|
||||
|
||||
const failures = new Map<string, { until: number; failures: number }>();
|
||||
|
||||
export function isRerankCoolingDown(model: string, now: number = Date.now()): boolean {
|
||||
const entry = failures.get(model);
|
||||
return !!entry && now < entry.until;
|
||||
}
|
||||
|
||||
/** Records a failure and returns the cooldown applied (ms). */
|
||||
export function recordRerankFailure(model: string, now: number = Date.now()): number {
|
||||
const count = (failures.get(model)?.failures ?? 0) + 1;
|
||||
const cooldown = Math.min(BASE_COOLDOWN_MS * 2 ** (count - 1), MAX_COOLDOWN_MS);
|
||||
failures.set(model, { until: now + cooldown, failures: count });
|
||||
return cooldown;
|
||||
}
|
||||
|
||||
export function recordRerankSuccess(model: string): void {
|
||||
failures.delete(model);
|
||||
}
|
||||
|
||||
export function _resetRerankBreaker(): void {
|
||||
failures.clear();
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
sanitizeFts5Query,
|
||||
} from "./retrieval/scoring";
|
||||
import type { MemoryRow } from "./retrieval/scoring";
|
||||
import { isRerankCoolingDown, recordRerankFailure, recordRerankSuccess } from "./rerankBreaker";
|
||||
|
||||
const log = logger("MEMORY_RETRIEVAL");
|
||||
|
||||
@@ -176,6 +177,7 @@ async function applyRerank<T extends { memory: Memory; score: number }>(
|
||||
rerankProviderModel: string
|
||||
): Promise<T[]> {
|
||||
if (items.length === 0) return items;
|
||||
if (isRerankCoolingDown(rerankProviderModel)) return items;
|
||||
|
||||
try {
|
||||
const documents = items.map((item) => item.memory.content);
|
||||
@@ -199,12 +201,15 @@ async function applyRerank<T extends { memory: Memory; score: number }>(
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const cooldownMs = recordRerankFailure(rerankProviderModel);
|
||||
log.warn("memory.rerank.http_fail", {
|
||||
status: res.status,
|
||||
model: rerankProviderModel,
|
||||
cooldownMs,
|
||||
});
|
||||
return items;
|
||||
}
|
||||
recordRerankSuccess(rerankProviderModel);
|
||||
|
||||
const data = (await res.json()) as {
|
||||
results?: Array<{ index: number; relevance_score: number }>;
|
||||
@@ -233,9 +238,11 @@ async function applyRerank<T extends { memory: Memory; score: number }>(
|
||||
}
|
||||
return reordered;
|
||||
} catch (err: unknown) {
|
||||
const cooldownMs = recordRerankFailure(rerankProviderModel);
|
||||
log.warn("memory.rerank.error", {
|
||||
error: sanitizeErrorMessage(err instanceof Error ? err.message : String(err)),
|
||||
model: rerankProviderModel,
|
||||
cooldownMs,
|
||||
});
|
||||
return items;
|
||||
}
|
||||
|
||||
111
tests/unit/memory-rerank-failure-breaker-20535.test.ts
Normal file
111
tests/unit/memory-rerank-failure-breaker-20535.test.ts
Normal file
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* #20535 — a failing rerank model is retried (loopback /v1/rerank) on EVERY retrieval:
|
||||
* no negative cache / breaker / backoff in applyRerank (src/lib/memory/retrieval.ts).
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import http from "node:http";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omr-repro-20535-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.DISABLE_SQLITE_AUTO_BACKUP = "true";
|
||||
delete process.env.VECTOR_STORE_DISABLE_VEC;
|
||||
|
||||
// fake "OmniRoute" loopback whose /v1/rerank always 404s (like the reporter's NVIDIA model)
|
||||
let rerankHits = 0;
|
||||
const server = http.createServer((req, res) => {
|
||||
if (req.url?.startsWith("/v1/rerank")) rerankHits++;
|
||||
res.writeHead(404, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ error: "not found" }));
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
|
||||
const port = (server.address() as { port: number }).port;
|
||||
process.env.API_PORT = String(port);
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const { getDbInstance, resetDbInstance } = core;
|
||||
const dbSettings = await import("../../src/lib/db/settings.ts");
|
||||
const memSettings = await import("../../src/lib/memory/settings.ts");
|
||||
const embCache = await import("../../src/lib/memory/embedding/cache.ts");
|
||||
const vs = await import("../../src/lib/memory/vectorStore.ts");
|
||||
const { resolveEmbeddingSource } = await import("../../src/lib/memory/embedding/index.ts");
|
||||
|
||||
test.after(async () => {
|
||||
await new Promise((r) => server.close(r));
|
||||
resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
});
|
||||
|
||||
test("#20535: every retrieval re-calls /v1/rerank even after it 404'd (no negative cache)", async (t) => {
|
||||
const db = getDbInstance();
|
||||
vs._resetVectorStoreSingleton();
|
||||
const store = vs.getVectorStore();
|
||||
if (!store) return t.skip("sqlite-vec unavailable");
|
||||
|
||||
await dbSettings.updateSettings({
|
||||
memoryEnabled: true,
|
||||
memoryStrategy: "hybrid",
|
||||
memoryStaticEnabled: true,
|
||||
memoryEmbeddingSource: "static",
|
||||
memoryRerankEnabled: true,
|
||||
memoryRerankProviderModel: "nvidia/nvidia/nv-rerankqa-mistral-4b-v3",
|
||||
});
|
||||
memSettings.invalidateMemorySettingsCache();
|
||||
const settings = await memSettings.getMemorySettings();
|
||||
const resolution = resolveEmbeddingSource(settings);
|
||||
assert.equal(resolution.source, "static");
|
||||
|
||||
const dim = resolution.dimensions as number;
|
||||
const vec = new Float32Array(dim).fill(0.1);
|
||||
// pre-seed embedding cache so no model download / network is needed
|
||||
const query = "what is the capital of france";
|
||||
embCache.set(
|
||||
embCache.buildCacheKey("static", resolution.identity ?? resolution.model, dim, query),
|
||||
vec
|
||||
);
|
||||
|
||||
await store.ensureReady(resolution);
|
||||
for (let i = 1; i <= 3; i++) {
|
||||
db.prepare(
|
||||
`INSERT INTO memories (rowid, memory_id, id, api_key_id, session_id, type, key, content, metadata, created_at, updated_at, expires_at)
|
||||
VALUES (?, ?, ?, 'k1', '', 'factual', ?, ?, '{}', datetime('now'), datetime('now'), NULL)`
|
||||
).run(i, i, `m${i}`, `key${i}`, `Paris is the capital of France ${i}`);
|
||||
await store.upsertVector(`m${i}`, vec);
|
||||
}
|
||||
|
||||
const { retrieveMemories } = await import("../../src/lib/memory/retrieval.ts");
|
||||
const N = 5;
|
||||
for (let i = 0; i < N; i++) {
|
||||
const out = await retrieveMemories("k1", {
|
||||
retrievalStrategy: "hybrid",
|
||||
query,
|
||||
maxTokens: 2000,
|
||||
});
|
||||
assert.ok(out.length > 0, "retrieval itself still succeeds");
|
||||
}
|
||||
console.log(`rerank loopback hits after ${N} retrievals: ${rerankHits}`);
|
||||
// EXPECTED (after fix: failure breaker/cooldown): 1 attempt, then skipped. Current: N.
|
||||
assert.equal(
|
||||
rerankHits,
|
||||
1,
|
||||
`rerank must be attempted once then skipped (${rerankHits}/${N} hits)`
|
||||
);
|
||||
});
|
||||
|
||||
test("#20535: breaker backs off exponentially, expires, and clears on success", async () => {
|
||||
const b = await import("../../src/lib/memory/rerankBreaker.ts");
|
||||
b._resetRerankBreaker();
|
||||
const t0 = 1_000_000;
|
||||
assert.equal(b.isRerankCoolingDown("m", t0), false);
|
||||
assert.equal(b.recordRerankFailure("m", t0), 60_000);
|
||||
assert.equal(b.isRerankCoolingDown("m", t0 + 59_999), true);
|
||||
assert.equal(b.isRerankCoolingDown("m", t0 + 60_000), false, "retry allowed after cooldown");
|
||||
assert.equal(b.recordRerankFailure("m", t0 + 60_000), 120_000);
|
||||
for (let i = 0; i < 10; i++) b.recordRerankFailure("m", t0);
|
||||
assert.equal(b.recordRerankFailure("m", t0), 600_000, "capped at 10 min");
|
||||
b.recordRerankSuccess("m");
|
||||
assert.equal(b.isRerankCoolingDown("m", t0), false);
|
||||
});
|
||||
Reference in New Issue
Block a user