fix: in-memory isolate buffer replaces direct-commit fallback

When TELEMETRY_KV is not bound, telemetry events now accumulate in
the Worker isolate's module-level buffer (same persistence model as
the rate limiter) and flush to HF when the buffer reaches 50 events,
256 KiB, or 15 minutes — whichever comes first. This eliminates the
one-commit-per-request explosion without requiring any Cloudflare
Dashboard configuration.

KV is still preferred (durable across isolate evictions), but the
fallback now produces ~1 HF commit per 50 requests instead of 50.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
faber
2026-07-10 21:59:47 -04:00
parent 2658bf15e4
commit f0744d1f1b

View File

@@ -66,6 +66,20 @@ const FLUSH_MAX_KEYS = 50 // max keys consumed per flush cycle
const FLUSH_MAX_BYTES = 5 * 1024 * 1024 // 5 MiB combined JSONL per HF commit
const FLUSH_AGE_MS = 30 * 60 * 1000 // 30 minutes
// ── In-memory isolate buffer (fallback when KV not bound) ───────────
// Same persistence model as the rate limiter: module-level state
// survives across requests within the same Worker isolate. We
// accumulate JSONL lines and flush to HF when the buffer is large
// or old enough. Isolate eviction loses buffered data — acceptable
// for best-effort telemetry, not acceptable for the commit explosion.
const MEM_FLUSH_COUNT = 50 // events before flush
const MEM_FLUSH_BYTES = 256 * 1024 // 256 KiB before flush
const MEM_FLUSH_AGE_MS = 15 * 60 * 1000 // 15 minutes before flush
const MEM_MAX_BYTES = 2 * 1024 * 1024 // 2 MiB hard cap — drop oldest if exceeded
const memBuffer: string[] = []
let memBufferBytes = 0
let memBufferFirstTs = 0
// ── Rate Limiter (dual: session + IP, in-memory per-isolate) ────────
const RATE_LIMIT_WINDOW_MS = 60_000
const RATE_LIMIT_MAX_SESSION = 10 // per session_id
@@ -264,23 +278,40 @@ export const onRequestPost: PagesFunction<Env> = async (context) => {
}, 200)
}
// ── Direct commit fallback (KV not configured) ────────────────────
// WARNING: At scale this creates one HF commit per request.
// Bind TELEMETRY_KV in production to avoid commit explosion.
console.warn('[Telemetry] TELEMETRY_KV not bound — falling back to direct HF commit')
// ── In-memory buffer fallback (KV not configured) ──────────────────
// Accumulates JSONL in the isolate's module scope and flushes to HF
// when the buffer is large or old enough. Zero config required.
// Data loss on isolate eviction is acceptable for telemetry.
const now = Date.now()
if (memBufferFirstTs === 0) memBufferFirstTs = now
const ts = new Date().toISOString().replace(/[:.]/g, '-')
const hash = shortHash(jsonl)
const filePath = `telemetry/batch_${ts}_${hash}.jsonl`
const branch = env.HF_DATASET_BRANCH || 'main'
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, jsonl)
if (ok) {
return jsonResponse({ accepted: sanitized.length, file: filePath }, 200)
// Drop oldest lines if hard cap would be exceeded
const lineBytes = new TextEncoder().encode(jsonl).byteLength
while (memBufferBytes + lineBytes > MEM_MAX_BYTES && memBuffer.length > 0) {
const dropped = memBuffer.shift()!
memBufferBytes -= new TextEncoder().encode(dropped).byteLength
}
return jsonResponse({ error: 'Failed to publish to HuggingFace' }, 502)
memBuffer.push(jsonl)
memBufferBytes += lineBytes
const age = now - memBufferFirstTs
const shouldFlush =
memBuffer.length >= MEM_FLUSH_COUNT ||
memBufferBytes >= MEM_FLUSH_BYTES ||
age >= MEM_FLUSH_AGE_MS
if (shouldFlush) {
context.waitUntil(flushMemToHF(env))
}
return jsonResponse({
accepted: sanitized.length,
buffered: true,
pending: memBuffer.length,
flushing: shouldFlush,
storage: 'memory',
}, 200)
}
// ── KV Flush: read up to FLUSH_MAX_KEYS, cap at FLUSH_MAX_BYTES ────
@@ -343,6 +374,39 @@ async function flushKVToHF(env: Env): Promise<number> {
return 0
}
// ── In-memory flush: drain buffer → one HF commit ──────────────────
async function flushMemToHF(env: Env): Promise<number> {
if (memBuffer.length === 0) return 0
// Snapshot and clear before the async commit so new requests
// accumulate into a fresh buffer, not into the one we're flushing.
const snapshot = memBuffer.splice(0, memBuffer.length)
const snapshotBytes = memBufferBytes
memBufferBytes = 0
memBufferFirstTs = 0
const combined = snapshot.join('\n')
const digest = await sha256Hex(combined)
const filePath = `telemetry/membuf_${digest}.jsonl`
const branch = env.HF_DATASET_BRANCH || 'main'
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
if (ok) {
console.log(`[Telemetry] Memory flush: ${snapshot.length} batches (${(snapshotBytes / 1024).toFixed(0)} KiB) → ${filePath}`)
return snapshot.length
}
// Commit failed — push lines back so the next request retries.
// Prepend so ordering is roughly preserved.
memBuffer.unshift(...snapshot)
memBufferBytes += snapshotBytes
if (memBufferFirstTs === 0) memBufferFirstTs = Date.now()
console.error(`[Telemetry] Memory flush failed — ${snapshot.length} batches returned to buffer`)
return 0
}
// ── HuggingFace Hub Commit ───────────────────────────────────────────
async function commitToHF(