mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-18 07:58:27 +00:00
fix(usage): retry a usage write once when the database is locked (#16276)
Thanks @maxmad64bis — a locked usage write is retried once, and `src/lib/usage/usageHistory.ts` shrinks below its frozen ceiling. Covered by `tests/unit/usage/usageWriteRetry.test.ts`.
Boarded on release/v3.8.52 at e2d5747872. Focused tests: 204 passed. The one failure is in tests/unit/codex-models-catalog-refresh.test.ts and belongs to #16299, which was ejected: it grows src/app/api/providers/[id]/models/route.ts from the frozen 2420 to 2422. File-size otherwise kept only the inherited executeTargetAttempt.ts overage (1288 > 1284). This pull request does not touch those files.
⚠️ base-red inherited: #15306
This commit is contained in:
@@ -0,0 +1 @@
|
||||
- **fix(usage):** a usage write blocked by a locked database is retried once after a short delay instead of being dropped; if the server stops during the wait the row is lost as before, on a best-effort basis ([#16276](https://github.com/diegosouzapw/OmniRoute/pull/16276)) — thanks @maxmad64bis (with thanks to @W25X80 for surfacing the SQLite write failures in #16176)
|
||||
@@ -8,13 +8,8 @@
|
||||
*/
|
||||
|
||||
import { getDbInstance } from "../db/core";
|
||||
import { resolveProviderId } from "@/shared/constants/providers";
|
||||
import { normalizePayloadForLog, protectPayloadForLog } from "../logPayloads";
|
||||
import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts";
|
||||
import {
|
||||
resolveOrphanedUsageAccountIdentity,
|
||||
resolveUsageAccountIdentity,
|
||||
} from "./accountIdentity";
|
||||
import {
|
||||
accumulateLatencySample,
|
||||
asRecord,
|
||||
@@ -36,14 +31,7 @@ import {
|
||||
getCompletedDetails,
|
||||
} from "./completedRequestDetails";
|
||||
import { shouldPersistToDisk } from "./migrations";
|
||||
import { emitUsageRecorded } from "./usageEvents";
|
||||
import {
|
||||
getLoggedInputTokens,
|
||||
getLoggedOutputTokens,
|
||||
getPromptCacheCreationTokens,
|
||||
getPromptCacheReadTokens,
|
||||
getReasoningTokens,
|
||||
} from "./tokenAccounting";
|
||||
import { attemptWrite, isWriteLockError, scheduleWriteRetry } from "./usageWriteRetry";
|
||||
|
||||
export type PendingRequestMetadata = {
|
||||
clientEndpoint?: string | null;
|
||||
@@ -870,120 +858,18 @@ export interface UsageEntry {
|
||||
export async function saveRequestUsage(entry: UsageEntry) {
|
||||
if (!shouldPersistToDisk) return;
|
||||
|
||||
// Freeze the timestamp on the first attempt so the deferred retry replays
|
||||
// the same value and the dedup guard keys both attempts identically.
|
||||
const frozenEntry = { ...entry, timestamp: entry.timestamp || new Date().toISOString() };
|
||||
|
||||
try {
|
||||
const db = getDbInstance();
|
||||
const timestamp = entry.timestamp || new Date().toISOString();
|
||||
const serviceTier = normalizeServiceTier(entry.serviceTier ?? entry.service_tier);
|
||||
|
||||
const tokensInput = getLoggedInputTokens(entry.tokens);
|
||||
const tokensOutput = getLoggedOutputTokens(entry.tokens);
|
||||
const connection = entry.connectionId
|
||||
? (db.prepare("SELECT * FROM provider_connections WHERE id = ?").get(entry.connectionId) as
|
||||
Record<string, unknown> | undefined)
|
||||
: undefined;
|
||||
const accountIdentity = connection
|
||||
? resolveUsageAccountIdentity(connection)
|
||||
: resolveOrphanedUsageAccountIdentity(entry.provider, entry.connectionId);
|
||||
|
||||
// Dedup guard: skip INSERT when an identical row already exists in the same
|
||||
// second. This prevents double-counting when onRequestSuccess fires more
|
||||
// than once (e.g. combo routing calling the callback from both the
|
||||
// streaming and non-streaming paths for the same underlying request).
|
||||
// Keyed on the natural identity of a request: timestamp + provider + model
|
||||
// + connectionId + apiKeyId + token counts. If only the endpoint is missing
|
||||
// on the existing row, fill it in rather than inserting a duplicate.
|
||||
let inserted = false;
|
||||
|
||||
db.transaction(() => {
|
||||
const existing = db
|
||||
.prepare(
|
||||
`SELECT id, endpoint, cpa_auth_index FROM usage_history
|
||||
WHERE timestamp = ?
|
||||
AND COALESCE(provider, '') = COALESCE(?, '')
|
||||
AND COALESCE(model, '') = COALESCE(?, '')
|
||||
AND COALESCE(connection_id, '') = COALESCE(?, '')
|
||||
AND COALESCE(api_key_id, '') = COALESCE(?, '')
|
||||
AND tokens_input = ?
|
||||
AND tokens_output = ?
|
||||
ORDER BY id DESC LIMIT 1`
|
||||
)
|
||||
.get(
|
||||
timestamp,
|
||||
entry.provider ? resolveProviderId(entry.provider) : null,
|
||||
entry.model || null,
|
||||
entry.connectionId || null,
|
||||
entry.apiKeyId || null,
|
||||
tokensInput,
|
||||
tokensOutput
|
||||
) as { id: number; endpoint: string | null; cpa_auth_index: string | null } | undefined;
|
||||
|
||||
if (existing) {
|
||||
// Back-fill endpoint if the original row missed it.
|
||||
if (!existing.endpoint && entry.endpoint) {
|
||||
db.prepare(`UPDATE usage_history SET endpoint = ? WHERE id = ?`).run(
|
||||
entry.endpoint,
|
||||
existing.id
|
||||
);
|
||||
}
|
||||
// A later completed attempt can carry the trace the first write missed.
|
||||
if (!existing.cpa_auth_index && entry.cpaAuthIndex) {
|
||||
db.prepare(`UPDATE usage_history SET cpa_auth_index = ? WHERE id = ?`).run(
|
||||
entry.cpaAuthIndex,
|
||||
existing.id
|
||||
);
|
||||
}
|
||||
return; // duplicate — do not insert
|
||||
}
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO usage_history (provider, model, connection_id, account_key, account_label,
|
||||
account_label_priority, api_key_id, api_key_name, tokens_input, tokens_output,
|
||||
tokens_cache_read, tokens_cache_creation, tokens_reasoning, service_tier, status, success,
|
||||
latency_ms, ttft_ms, error_code, combo_strategy, endpoint, cpa_auth_index, timestamp)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
).run(
|
||||
entry.provider ? resolveProviderId(entry.provider) : null,
|
||||
entry.model || null,
|
||||
entry.connectionId || null,
|
||||
accountIdentity.accountKey,
|
||||
accountIdentity.accountLabel,
|
||||
accountIdentity.accountLabelPriority,
|
||||
entry.apiKeyId || null,
|
||||
entry.apiKeyName || null,
|
||||
tokensInput,
|
||||
tokensOutput,
|
||||
getPromptCacheReadTokens(entry.tokens),
|
||||
getPromptCacheCreationTokens(entry.tokens),
|
||||
getReasoningTokens(entry.tokens),
|
||||
serviceTier,
|
||||
entry.status || null,
|
||||
entry.success === false ? 0 : 1,
|
||||
Number.isFinite(Number(entry.latencyMs)) ? Number(entry.latencyMs) : 0,
|
||||
Number.isFinite(Number(entry.timeToFirstTokenMs))
|
||||
? Number(entry.timeToFirstTokenMs)
|
||||
: Number.isFinite(Number(entry.latencyMs))
|
||||
? Number(entry.latencyMs)
|
||||
: 0,
|
||||
entry.errorCode || null,
|
||||
entry.comboStrategy || entry.combo_strategy || null,
|
||||
entry.endpoint || null,
|
||||
entry.cpaAuthIndex || null,
|
||||
timestamp
|
||||
);
|
||||
|
||||
inserted = true;
|
||||
})();
|
||||
|
||||
// Decoupled via the event bus so usageHistory never imports providerLimits
|
||||
// (which would pull the executors/translator graph into the type-check surface).
|
||||
// Only emit when a row was actually inserted — not on dedup no-ops.
|
||||
if (inserted) {
|
||||
emitUsageRecorded(entry.provider, entry.connectionId);
|
||||
}
|
||||
attemptWrite(frozenEntry);
|
||||
} catch (error) {
|
||||
console.error("Failed to save usage stats:", error);
|
||||
if (!isWriteLockError(error)) {
|
||||
console.error("Failed to save usage stats:", error);
|
||||
return;
|
||||
}
|
||||
scheduleWriteRetry(frozenEntry);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
251
src/lib/usage/usageWriteRetry.ts
Normal file
251
src/lib/usage/usageWriteRetry.ts
Normal file
@@ -0,0 +1,251 @@
|
||||
/**
|
||||
* Deferred retry for usage writes blocked by a locked database.
|
||||
*
|
||||
* The hot-path writers here are best-effort: the database caps a contended
|
||||
* write at 2 s (`busy_timeout = 2000` in `src/lib/db/core.ts`) so a busy disk
|
||||
* cannot freeze the event loop. When a write still meets a lock, the first
|
||||
* failure is swallowed silently and the entry is attempted once more after a
|
||||
* short delay, off the request path (`setTimeout`, never awaited). If the
|
||||
* second attempt fails the existing log line is written and the row is
|
||||
* dropped. Any other error keeps the current behaviour: logged at once,
|
||||
* never retried. If the server stops during the wait the row is lost as
|
||||
* before, on a best-effort basis (the timer never keeps the loop alive).
|
||||
*
|
||||
* @module lib/usage/usageWriteRetry
|
||||
*/
|
||||
|
||||
import { getDbInstance } from "../db/core";
|
||||
import { resolveProviderId } from "@/shared/constants/providers";
|
||||
import {
|
||||
resolveOrphanedUsageAccountIdentity,
|
||||
resolveUsageAccountIdentity,
|
||||
} from "./accountIdentity";
|
||||
import { normalizeServiceTier } from "./usageHistory/helpers";
|
||||
import { emitUsageRecorded } from "./usageEvents";
|
||||
import {
|
||||
getLoggedInputTokens,
|
||||
getLoggedOutputTokens,
|
||||
getPromptCacheCreationTokens,
|
||||
getPromptCacheReadTokens,
|
||||
getReasoningTokens,
|
||||
} from "./tokenAccounting";
|
||||
import type { UsageEntry } from "./usageHistory";
|
||||
|
||||
/** Delay before the single deferred attempt. Internal, not user-facing. */
|
||||
const RETRY_DELAY_MS = 1000;
|
||||
|
||||
/**
|
||||
* True when the error is a database lock, in any shape the supported drivers
|
||||
* report it. Restricted to locks: only a lock is expected to clear within the
|
||||
* retry delay. Lock classification mirrors the lock subset of the probe
|
||||
* classifier and the busy checker (drivers report plain "database is locked"
|
||||
* while better-sqlite3 carries the name in `code` and node:sqlite the numeric
|
||||
* primary code in `errcode`, extended codes in the high bits).
|
||||
*/
|
||||
export function isWriteLockError(error: unknown): boolean {
|
||||
const details = error as { code?: unknown; errcode?: unknown; message?: unknown } | null;
|
||||
const code = details?.code ?? details?.errcode;
|
||||
if (typeof code === "string" && isLockCodeName(code)) return true;
|
||||
if (typeof code === "number" && isLockCodeNumber(code)) return true;
|
||||
const message = error instanceof Error ? error.message : String(details?.message ?? error);
|
||||
return isLockMessage(message);
|
||||
}
|
||||
|
||||
function isLockCodeName(code: string): boolean {
|
||||
return /^SQLITE_(BUSY|LOCKED)/.test(code);
|
||||
}
|
||||
|
||||
function isLockCodeNumber(code: number): boolean {
|
||||
const primary = code & 0xff;
|
||||
return primary === 5 || primary === 6;
|
||||
}
|
||||
|
||||
function isLockMessage(message: string): boolean {
|
||||
return /database(?: table| schema)? is (?:locked|busy)|SQLITE_BUSY|SQLITE_LOCKED/i.test(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Single write attempt. Throws on failure so the caller decides between a
|
||||
* deferred retry (lock) and an immediate log (anything else).
|
||||
*/
|
||||
export function attemptWrite(entry: UsageEntry): void {
|
||||
const db = getDbInstance();
|
||||
const prepared = prepareEntry(db, entry);
|
||||
let inserted = false;
|
||||
|
||||
db.transaction(() => {
|
||||
inserted = insertUnlessDuplicate(db, prepared);
|
||||
})();
|
||||
|
||||
// Decoupled via the event bus so usageHistory never imports providerLimits
|
||||
// (which would pull the executors/translator graph into the type-check surface).
|
||||
// Only emit when a row was actually inserted — not on dedup no-ops.
|
||||
if (inserted) {
|
||||
emitUsageRecorded(entry.provider, entry.connectionId);
|
||||
}
|
||||
}
|
||||
|
||||
type PreparedEntry = {
|
||||
entry: UsageEntry;
|
||||
timestamp: string;
|
||||
serviceTier: string;
|
||||
tokensInput: number;
|
||||
tokensOutput: number;
|
||||
accountIdentity: {
|
||||
accountKey: string | null;
|
||||
accountLabel: string | null;
|
||||
accountLabelPriority: number;
|
||||
};
|
||||
};
|
||||
|
||||
function prepareEntry(db: ReturnType<typeof getDbInstance>, entry: UsageEntry): PreparedEntry {
|
||||
const timestamp = entry.timestamp || new Date().toISOString();
|
||||
const connection = entry.connectionId
|
||||
? (db.prepare("SELECT * FROM provider_connections WHERE id = ?").get(entry.connectionId) as
|
||||
Record<string, unknown> | undefined)
|
||||
: undefined;
|
||||
return {
|
||||
entry,
|
||||
timestamp,
|
||||
serviceTier: normalizeServiceTier(entry.serviceTier ?? entry.service_tier),
|
||||
tokensInput: getLoggedInputTokens(entry.tokens),
|
||||
tokensOutput: getLoggedOutputTokens(entry.tokens),
|
||||
accountIdentity: connection
|
||||
? resolveUsageAccountIdentity(connection)
|
||||
: resolveOrphanedUsageAccountIdentity(entry.provider, entry.connectionId),
|
||||
};
|
||||
}
|
||||
|
||||
// Dedup guard: skip INSERT when an identical row already exists in the same
|
||||
// second. This prevents double-counting when onRequestSuccess fires more
|
||||
// than once (e.g. combo routing calling the callback from both the
|
||||
// streaming and non-streaming paths for the same underlying request).
|
||||
// Keyed on the natural identity of a request: timestamp + provider + model
|
||||
// + connectionId + apiKeyId + token counts. If only the endpoint is missing
|
||||
// on the existing row, fill it in rather than inserting a duplicate.
|
||||
function insertUnlessDuplicate(
|
||||
db: ReturnType<typeof getDbInstance>,
|
||||
prepared: PreparedEntry
|
||||
): boolean {
|
||||
const { entry, timestamp, tokensInput, tokensOutput } = prepared;
|
||||
const existing = db
|
||||
.prepare(
|
||||
`SELECT id, endpoint, cpa_auth_index FROM usage_history
|
||||
WHERE timestamp = ?
|
||||
AND COALESCE(provider, '') = COALESCE(?, '')
|
||||
AND COALESCE(model, '') = COALESCE(?, '')
|
||||
AND COALESCE(connection_id, '') = COALESCE(?, '')
|
||||
AND COALESCE(api_key_id, '') = COALESCE(?, '')
|
||||
AND tokens_input = ?
|
||||
AND tokens_output = ?
|
||||
ORDER BY id DESC LIMIT 1`
|
||||
)
|
||||
.get(
|
||||
timestamp,
|
||||
entry.provider ? resolveProviderId(entry.provider) : null,
|
||||
entry.model || null,
|
||||
entry.connectionId || null,
|
||||
entry.apiKeyId || null,
|
||||
tokensInput,
|
||||
tokensOutput
|
||||
) as { id: number; endpoint: string | null; cpa_auth_index: string | null } | undefined;
|
||||
|
||||
if (existing) {
|
||||
backfillDuplicate(db, existing, entry);
|
||||
return false;
|
||||
}
|
||||
|
||||
insertEntry(db, prepared);
|
||||
return true;
|
||||
}
|
||||
|
||||
function backfillDuplicate(
|
||||
db: ReturnType<typeof getDbInstance>,
|
||||
existing: { id: number; endpoint: string | null; cpa_auth_index: string | null },
|
||||
entry: UsageEntry
|
||||
): void {
|
||||
// Back-fill endpoint if the original row missed it.
|
||||
if (!existing.endpoint && entry.endpoint) {
|
||||
db.prepare(`UPDATE usage_history SET endpoint = ? WHERE id = ?`).run(
|
||||
entry.endpoint,
|
||||
existing.id
|
||||
);
|
||||
}
|
||||
// A later completed attempt can carry the trace the first write missed.
|
||||
if (!existing.cpa_auth_index && entry.cpaAuthIndex) {
|
||||
db.prepare(`UPDATE usage_history SET cpa_auth_index = ? WHERE id = ?`).run(
|
||||
entry.cpaAuthIndex,
|
||||
existing.id
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function insertEntry(db: ReturnType<typeof getDbInstance>, prepared: PreparedEntry): void {
|
||||
const { entry, timestamp, serviceTier, tokensInput, tokensOutput, accountIdentity } = prepared;
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO usage_history (provider, model, connection_id, account_key, account_label,
|
||||
account_label_priority, api_key_id, api_key_name, tokens_input, tokens_output,
|
||||
tokens_cache_read, tokens_cache_creation, tokens_reasoning, service_tier, status, success,
|
||||
latency_ms, ttft_ms, error_code, combo_strategy, endpoint, cpa_auth_index, timestamp)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
).run(
|
||||
entry.provider ? resolveProviderId(entry.provider) : null,
|
||||
entry.model || null,
|
||||
entry.connectionId || null,
|
||||
accountIdentity.accountKey,
|
||||
accountIdentity.accountLabel,
|
||||
accountIdentity.accountLabelPriority,
|
||||
entry.apiKeyId || null,
|
||||
entry.apiKeyName || null,
|
||||
tokensInput,
|
||||
tokensOutput,
|
||||
getPromptCacheReadTokens(entry.tokens),
|
||||
getPromptCacheCreationTokens(entry.tokens),
|
||||
getReasoningTokens(entry.tokens),
|
||||
serviceTier,
|
||||
entry.status || null,
|
||||
entry.success === false ? 0 : 1,
|
||||
toFinite(entry.latencyMs),
|
||||
toFirstToken(entry),
|
||||
entry.errorCode || null,
|
||||
entry.comboStrategy || entry.combo_strategy || null,
|
||||
entry.endpoint || null,
|
||||
entry.cpaAuthIndex || null,
|
||||
timestamp
|
||||
);
|
||||
}
|
||||
|
||||
function toFinite(value: unknown): number {
|
||||
return Number.isFinite(Number(value)) ? Number(value) : 0;
|
||||
}
|
||||
|
||||
function toFirstToken(entry: UsageEntry): number {
|
||||
if (Number.isFinite(Number(entry.timeToFirstTokenMs))) return Number(entry.timeToFirstTokenMs);
|
||||
return toFinite(entry.latencyMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule the single deferred retry, off the request path. Never chained:
|
||||
* the retry attempts the write once and, on failure, writes the existing log
|
||||
* line and stops.
|
||||
*/
|
||||
export function scheduleWriteRetry(entry: UsageEntry): void {
|
||||
const timer = setTimeout(() => {
|
||||
try {
|
||||
attemptWrite(entry);
|
||||
} catch (retryError) {
|
||||
console.error("Failed to save usage stats:", retryError);
|
||||
}
|
||||
}, RETRY_DELAY_MS);
|
||||
// Allow Node.js to exit naturally even if the timer is still pending
|
||||
// (avoids keeping the event loop alive for a stray retry).
|
||||
detachTimer(timer);
|
||||
}
|
||||
|
||||
type DetachableTimer = { unref?: () => void };
|
||||
|
||||
function detachTimer(timer: DetachableTimer): void {
|
||||
timer.unref?.();
|
||||
}
|
||||
264
tests/unit/usage/usageWriteRetry.test.ts
Normal file
264
tests/unit/usage/usageWriteRetry.test.ts
Normal file
@@ -0,0 +1,264 @@
|
||||
/**
|
||||
* Gate tests for a usage write blocked by a locked database.
|
||||
*
|
||||
* A first attempt that meets a lock is deferred once, off the request path,
|
||||
* and retried a single time. A second lock drops the row with the existing
|
||||
* log line. Any other error keeps the current behaviour: logged at once,
|
||||
* never retried. A first-attempt success never schedules a second write.
|
||||
*/
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import Database from "better-sqlite3";
|
||||
|
||||
// Isolate the DB from other tests and from the real data dir.
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-usage-write-retry-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
|
||||
// Dynamic imports so DATA_DIR is set before any module initialises the DB.
|
||||
const core = await import("../../../src/lib/db/core.ts");
|
||||
const usageHistory = await import("../../../src/lib/usage/usageHistory.ts");
|
||||
const { saveRequestUsage } = usageHistory;
|
||||
|
||||
// Cleanup: close DB handle and temp directory so the test runner doesn't hang.
|
||||
test.after(() => {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
});
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
let entrySeq = 0;
|
||||
|
||||
function makeEntry(overrides: Record<string, unknown> = {}) {
|
||||
entrySeq++;
|
||||
const timestamp = new Date(Date.now() + entrySeq).toISOString();
|
||||
|
||||
return {
|
||||
provider: "test-provider",
|
||||
model: "test-model",
|
||||
connectionId: `conn-retry-${entrySeq}`,
|
||||
apiKeyId: null,
|
||||
apiKeyName: null,
|
||||
tokens: { input_tokens: 10, output_tokens: 20 },
|
||||
status: "success",
|
||||
success: true,
|
||||
latencyMs: 100,
|
||||
timeToFirstTokenMs: 50,
|
||||
errorCode: null,
|
||||
comboStrategy: null,
|
||||
endpoint: "/v1/chat/completions",
|
||||
timestamp,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function countRows(): number {
|
||||
const db = core.getDbInstance();
|
||||
const row = db.prepare("SELECT COUNT(*) AS cnt FROM usage_history").get() as { cnt: number };
|
||||
return row.cnt;
|
||||
}
|
||||
|
||||
function countRowsFor(connectionId: string): number {
|
||||
const db = core.getDbInstance();
|
||||
const row = db
|
||||
.prepare("SELECT COUNT(*) AS cnt FROM usage_history WHERE connection_id = ?")
|
||||
.get(connectionId) as { cnt: number };
|
||||
return row.cnt;
|
||||
}
|
||||
|
||||
/** Hold a write lock on the same SQLite file until released by the caller. */
|
||||
function holdWriteLock() {
|
||||
const db = new Database(path.join(TEST_DATA_DIR, "storage.sqlite"));
|
||||
db.exec("BEGIN IMMEDIATE");
|
||||
return db;
|
||||
}
|
||||
|
||||
function releaseWriteLock(db: ReturnType<typeof holdWriteLock>) {
|
||||
try {
|
||||
db.exec("ROLLBACK");
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Wait until `check()` is true, up to `timeoutMs`, polling every 50 ms. */
|
||||
async function waitFor(check: () => boolean, timeoutMs = 5000): Promise<boolean> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
for (;;) {
|
||||
if (check()) return true;
|
||||
if (Date.now() >= deadline) return false;
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
}
|
||||
|
||||
// ── predicate ────────────────────────────────────────────────────────────────
|
||||
|
||||
test("write lock predicate accepts lock shapes and rejects other errors", async () => {
|
||||
const { isWriteLockError } = await import("../../../src/lib/usage/usageWriteRetry.ts");
|
||||
assert.equal(isWriteLockError(new Error("database is locked")), true);
|
||||
assert.equal(isWriteLockError(new Error("database table is locked")), true);
|
||||
assert.equal(isWriteLockError(new Error("database schema is locked")), true);
|
||||
assert.equal(isWriteLockError(new Error("database is busy")), true);
|
||||
assert.equal(isWriteLockError({ code: "SQLITE_BUSY" }), true);
|
||||
assert.equal(isWriteLockError({ code: "SQLITE_LOCKED" }), true);
|
||||
assert.equal(isWriteLockError({ errcode: 5 }), true);
|
||||
assert.equal(isWriteLockError({ errcode: 6 }), true);
|
||||
assert.equal(isWriteLockError({ errcode: 0x0505 }), true);
|
||||
assert.equal(isWriteLockError("database is locked"), true);
|
||||
assert.equal(isWriteLockError(new Error("UNIQUE constraint failed: usage_history.id")), false);
|
||||
assert.equal(isWriteLockError(new Error("no such table: usage_history")), false);
|
||||
});
|
||||
|
||||
// ── gate 1: locked first, free second → one row ──────────────────────────────
|
||||
|
||||
test("usage write locked on the first attempt is written once the lock clears", async () => {
|
||||
const db = core.getDbInstance();
|
||||
const before = countRows();
|
||||
const errors: unknown[] = [];
|
||||
const log = console.error;
|
||||
console.error = (...args: unknown[]) => {
|
||||
errors.push(args);
|
||||
};
|
||||
const locker = holdWriteLock();
|
||||
const entry = makeEntry();
|
||||
try {
|
||||
// Filled while the rival connection holds the lock: the row cannot land yet.
|
||||
await saveRequestUsage(entry as never);
|
||||
assert.equal(countRows(), before);
|
||||
|
||||
// Free the base before the deferred attempt runs.
|
||||
releaseWriteLock(locker);
|
||||
|
||||
assert.equal(await waitFor(() => countRows() === before + 1), true);
|
||||
assert.equal(countRows(), before + 1);
|
||||
assert.equal(errors.length, 0);
|
||||
void db;
|
||||
} finally {
|
||||
try {
|
||||
if (locker.open) releaseWriteLock(locker);
|
||||
} catch {
|
||||
// Already released above.
|
||||
}
|
||||
console.error = log;
|
||||
}
|
||||
});
|
||||
|
||||
// ── gate 2: locked twice → dropped, logged, no third attempt ────────────────
|
||||
|
||||
test("usage write locked on both attempts is dropped with one log line", async () => {
|
||||
const before = countRows();
|
||||
const errors: unknown[][] = [];
|
||||
const log = console.error;
|
||||
console.error = (...args: unknown[]) => {
|
||||
errors.push(args);
|
||||
};
|
||||
const locker = holdWriteLock();
|
||||
const entry = makeEntry();
|
||||
try {
|
||||
await saveRequestUsage(entry as never);
|
||||
|
||||
// Keep the lock held past the deferred attempt, then wait past any
|
||||
// further attempt: a third try would need another full delay window.
|
||||
assert.equal(await waitFor(() => errors.length > 0), true);
|
||||
const logged = errors.length;
|
||||
assert.equal(logged, 1);
|
||||
assert.match(String(errors[0][0]), /Failed to save usage stats:/);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
assert.equal(errors.length, logged);
|
||||
assert.equal(countRows(), before);
|
||||
} finally {
|
||||
try {
|
||||
if (locker.open) releaseWriteLock(locker);
|
||||
} catch {
|
||||
// Already released above.
|
||||
}
|
||||
console.error = log;
|
||||
}
|
||||
});
|
||||
|
||||
// ── gate 3: non-lock error → current behaviour, no second attempt ────────────
|
||||
|
||||
test("usage write failing on another error is logged at once and never retried", async () => {
|
||||
const before = countRows();
|
||||
const errors: unknown[][] = [];
|
||||
const log = console.error;
|
||||
console.error = (...args: unknown[]) => {
|
||||
errors.push(args);
|
||||
};
|
||||
try {
|
||||
// A missing table is never a lock: the current path logs and stops.
|
||||
const db = core.getDbInstance();
|
||||
db.exec("ALTER TABLE usage_history RENAME TO usage_history_backup");
|
||||
try {
|
||||
await saveRequestUsage(makeEntry() as never);
|
||||
} finally {
|
||||
db.exec("ALTER TABLE usage_history_backup RENAME TO usage_history");
|
||||
}
|
||||
|
||||
assert.equal(errors.length, 1);
|
||||
assert.match(String(errors[0][0]), /Failed to save usage stats:/);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
assert.equal(errors.length, 1);
|
||||
assert.equal(countRows(), before);
|
||||
} finally {
|
||||
console.error = log;
|
||||
}
|
||||
});
|
||||
|
||||
// ── frozen timestamp: first attempt and retry share one value ───────────────
|
||||
|
||||
test("usage write retry replays the first-attempt timestamp", async () => {
|
||||
const log = console.error;
|
||||
console.error = (..._args: unknown[]) => {};
|
||||
const locker = holdWriteLock();
|
||||
// No timestamp on the entry: the envelope freezes one at the first attempt.
|
||||
const entry = makeEntry({ connectionId: "conn-retry-frozen-ts" }) as Record<string, unknown>;
|
||||
delete entry.timestamp;
|
||||
try {
|
||||
const t0 = new Date().toISOString();
|
||||
await saveRequestUsage(entry as never);
|
||||
|
||||
// Free the base before the deferred attempt runs.
|
||||
releaseWriteLock(locker);
|
||||
|
||||
assert.equal(await waitFor(() => countRowsFor("conn-retry-frozen-ts") === 1), true);
|
||||
const t1 = new Date().toISOString();
|
||||
const row = core
|
||||
.getDbInstance()
|
||||
.prepare("SELECT timestamp AS ts FROM usage_history WHERE connection_id = ?")
|
||||
.get("conn-retry-frozen-ts") as { ts: string };
|
||||
// The retry replayed the frozen value, not one recomputed ~1 s later.
|
||||
assert.ok(row.ts >= t0 && row.ts <= t1);
|
||||
assert.ok(Date.parse(t1) - Date.parse(row.ts) >= 500);
|
||||
} finally {
|
||||
try {
|
||||
if (locker.open) releaseWriteLock(locker);
|
||||
} catch {
|
||||
// Already released above.
|
||||
}
|
||||
console.error = log;
|
||||
}
|
||||
});
|
||||
|
||||
// ── gate 4: first-attempt success → no second attempt ────────────────────────
|
||||
test("usage write succeeding on the first attempt never writes twice", async () => {
|
||||
const before = countRows();
|
||||
const errors: unknown[] = [];
|
||||
const log = console.error;
|
||||
console.error = (...args: unknown[]) => {
|
||||
errors.push(args);
|
||||
};
|
||||
try {
|
||||
await saveRequestUsage(makeEntry() as never);
|
||||
assert.equal(countRows(), before + 1);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
assert.equal(countRows(), before + 1);
|
||||
assert.equal(errors.length, 0);
|
||||
} finally {
|
||||
console.error = log;
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user