From f0744d1f1b5e90e9e62b7ec2c6d7aa0f768d386b Mon Sep 17 00:00:00 2001 From: faber Date: Fri, 10 Jul 2026 21:59:47 -0400 Subject: [PATCH] fix: in-memory isolate buffer replaces direct-commit fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When TELEMETRY_KV is not bound, telemetry events now accumulate in the Worker isolate's module-level buffer (same persistence model as the rate limiter) and flush to HF when the buffer reaches 50 events, 256 KiB, or 15 minutes — whichever comes first. This eliminates the one-commit-per-request explosion without requiring any Cloudflare Dashboard configuration. KV is still preferred (durable across isolate evictions), but the fallback now produces ~1 HF commit per 50 requests instead of 50. Co-Authored-By: Claude Opus 4.6 --- functions/api/telemetry.ts | 92 ++++++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 14 deletions(-) diff --git a/functions/api/telemetry.ts b/functions/api/telemetry.ts index faefd8b..0d785a0 100644 --- a/functions/api/telemetry.ts +++ b/functions/api/telemetry.ts @@ -66,6 +66,20 @@ const FLUSH_MAX_KEYS = 50 // max keys consumed per flush cycle const FLUSH_MAX_BYTES = 5 * 1024 * 1024 // 5 MiB combined JSONL per HF commit const FLUSH_AGE_MS = 30 * 60 * 1000 // 30 minutes +// ── In-memory isolate buffer (fallback when KV not bound) ─────────── +// Same persistence model as the rate limiter: module-level state +// survives across requests within the same Worker isolate. We +// accumulate JSONL lines and flush to HF when the buffer is large +// or old enough. Isolate eviction loses buffered data — acceptable +// for best-effort telemetry, not acceptable for the commit explosion. +const MEM_FLUSH_COUNT = 50 // events before flush +const MEM_FLUSH_BYTES = 256 * 1024 // 256 KiB before flush +const MEM_FLUSH_AGE_MS = 15 * 60 * 1000 // 15 minutes before flush +const MEM_MAX_BYTES = 2 * 1024 * 1024 // 2 MiB hard cap — drop oldest if exceeded +const memBuffer: string[] = [] +let memBufferBytes = 0 +let memBufferFirstTs = 0 + // ── Rate Limiter (dual: session + IP, in-memory per-isolate) ──────── const RATE_LIMIT_WINDOW_MS = 60_000 const RATE_LIMIT_MAX_SESSION = 10 // per session_id @@ -264,23 +278,40 @@ export const onRequestPost: PagesFunction = async (context) => { }, 200) } - // ── Direct commit fallback (KV not configured) ──────────────────── - // WARNING: At scale this creates one HF commit per request. - // Bind TELEMETRY_KV in production to avoid commit explosion. - console.warn('[Telemetry] TELEMETRY_KV not bound — falling back to direct HF commit') + // ── In-memory buffer fallback (KV not configured) ────────────────── + // Accumulates JSONL in the isolate's module scope and flushes to HF + // when the buffer is large or old enough. Zero config required. + // Data loss on isolate eviction is acceptable for telemetry. + const now = Date.now() + if (memBufferFirstTs === 0) memBufferFirstTs = now - const ts = new Date().toISOString().replace(/[:.]/g, '-') - const hash = shortHash(jsonl) - const filePath = `telemetry/batch_${ts}_${hash}.jsonl` - - const branch = env.HF_DATASET_BRANCH || 'main' - const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, jsonl) - - if (ok) { - return jsonResponse({ accepted: sanitized.length, file: filePath }, 200) + // Drop oldest lines if hard cap would be exceeded + const lineBytes = new TextEncoder().encode(jsonl).byteLength + while (memBufferBytes + lineBytes > MEM_MAX_BYTES && memBuffer.length > 0) { + const dropped = memBuffer.shift()! + memBufferBytes -= new TextEncoder().encode(dropped).byteLength } - return jsonResponse({ error: 'Failed to publish to HuggingFace' }, 502) + memBuffer.push(jsonl) + memBufferBytes += lineBytes + + const age = now - memBufferFirstTs + const shouldFlush = + memBuffer.length >= MEM_FLUSH_COUNT || + memBufferBytes >= MEM_FLUSH_BYTES || + age >= MEM_FLUSH_AGE_MS + + if (shouldFlush) { + context.waitUntil(flushMemToHF(env)) + } + + return jsonResponse({ + accepted: sanitized.length, + buffered: true, + pending: memBuffer.length, + flushing: shouldFlush, + storage: 'memory', + }, 200) } // ── KV Flush: read up to FLUSH_MAX_KEYS, cap at FLUSH_MAX_BYTES ──── @@ -343,6 +374,39 @@ async function flushKVToHF(env: Env): Promise { return 0 } +// ── In-memory flush: drain buffer → one HF commit ────────────────── + +async function flushMemToHF(env: Env): Promise { + if (memBuffer.length === 0) return 0 + + // Snapshot and clear before the async commit so new requests + // accumulate into a fresh buffer, not into the one we're flushing. + const snapshot = memBuffer.splice(0, memBuffer.length) + const snapshotBytes = memBufferBytes + memBufferBytes = 0 + memBufferFirstTs = 0 + + const combined = snapshot.join('\n') + const digest = await sha256Hex(combined) + const filePath = `telemetry/membuf_${digest}.jsonl` + const branch = env.HF_DATASET_BRANCH || 'main' + + const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined) + + if (ok) { + console.log(`[Telemetry] Memory flush: ${snapshot.length} batches (${(snapshotBytes / 1024).toFixed(0)} KiB) → ${filePath}`) + return snapshot.length + } + + // Commit failed — push lines back so the next request retries. + // Prepend so ordering is roughly preserved. + memBuffer.unshift(...snapshot) + memBufferBytes += snapshotBytes + if (memBufferFirstTs === 0) memBufferFirstTs = Date.now() + console.error(`[Telemetry] Memory flush failed — ${snapshot.length} batches returned to buffer`) + return 0 +} + // ── HuggingFace Hub Commit ─────────────────────────────────────────── async function commitToHF(