fix: remove fake KV lock, add GET ?flush=force, add .gitignore

KV lock removed — KV is eventually consistent and can't do atomic CAS.
Concurrent flushes may produce duplicate JSONL files on HF, which is
harmless (extra telemetry rows, not data loss or corruption).

Added GET /api/telemetry?flush=force so an external cron can trigger
guaranteed periodic flushes regardless of traffic volume.

Added .gitignore (repo had none).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
faber
2026-07-10 16:24:48 -04:00
parent 2a42b57cfb
commit cebc27cf4c
2 changed files with 65 additions and 43 deletions

7
.gitignore vendored Normal file
View File

@@ -0,0 +1,7 @@
node_modules/
.next/
.env
.env.local
.wrangler/
dist/
tsconfig.tsbuildinfo

View File

@@ -108,6 +108,25 @@ export const onRequestOptions: PagesFunction<Env> = async () => {
return new Response(null, { status: 204, headers: CORS_HEADERS })
}
// GET ?flush=force — external cron trigger for guaranteed flush
// Set up a CF Worker cron or external cron to hit this every 30 min
// to ensure batches don't sit in KV forever during low-traffic periods.
export const onRequestGet: PagesFunction<Env> = async (context) => {
const { request, env } = context
const url = new URL(request.url)
if (url.searchParams.get('flush') !== 'force') {
return jsonResponse({ status: 'ok', hint: 'POST events here, or GET ?flush=force to trigger a flush' }, 200)
}
if (!env.TELEMETRY_KV || !env.HF_TOKEN || !env.HF_DATASET_REPO) {
return jsonResponse({ error: 'KV or HF not configured' }, 503)
}
const flushed = await flushKVToHF(env)
return jsonResponse({ flushed }, 200)
}
// Main handler
export const onRequestPost: PagesFunction<Env> = async (context) => {
const { request, env } = context
@@ -175,11 +194,11 @@ export const onRequestPost: PagesFunction<Env> = async (context) => {
}
if (shouldFlush) {
// Acquire a simple lock to prevent concurrent flushes
const lockValue = await env.TELEMETRY_KV.get('flush:lock')
if (!lockValue) {
context.waitUntil(flushKVToHF(env))
}
// Flush is idempotent: concurrent flushes may produce duplicate
// JSONL files on HF, which is harmless (extra telemetry rows).
// KV is eventually consistent and can't do atomic locks, so we
// accept rare duplicates rather than pretend we can prevent them.
context.waitUntil(flushKVToHF(env))
}
return jsonResponse({
@@ -207,49 +226,45 @@ export const onRequestPost: PagesFunction<Env> = async (context) => {
// ── KV Flush: combine all pending batches into one HF commit ────────
async function flushKVToHF(env: Env): Promise<void> {
async function flushKVToHF(env: Env): Promise<number> {
const kv = env.TELEMETRY_KV!
const allKeys: { name: string }[] = []
// Acquire lock (60s TTL — if flush crashes, lock auto-expires)
await kv.put('flush:lock', '1', { expirationTtl: 60 })
let cursor: string | undefined
do {
const list = await kv.list({ prefix: 'batch:', cursor })
allKeys.push(...list.keys)
cursor = list.list_complete ? undefined : list.cursor
} while (cursor)
try {
const allKeys: { name: string }[] = []
if (allKeys.length === 0) return 0
let cursor: string | undefined
do {
const list = await kv.list({ prefix: 'batch:', cursor })
allKeys.push(...list.keys)
cursor = list.list_complete ? undefined : list.cursor
} while (cursor)
if (allKeys.length === 0) return
const chunks: string[] = []
for (const key of allKeys) {
const value = await kv.get(key.name)
if (value) chunks.push(value)
}
if (chunks.length === 0) return
const combined = chunks.join('\n')
const ts = new Date().toISOString().replace(/[:.]/g, '-')
const hash = shortHash(combined)
const filePath = `telemetry/batch_${ts}_${hash}.jsonl`
const branch = env.HF_DATASET_BRANCH || 'main'
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
if (ok) {
await Promise.all(allKeys.map(key => kv.delete(key.name)))
console.log(`[Telemetry] Flushed ${allKeys.length} batches → ${filePath}`)
} else {
console.error(`[Telemetry] Flush failed — ${allKeys.length} batches remain in KV`)
}
} finally {
await kv.delete('flush:lock')
const chunks: string[] = []
for (const key of allKeys) {
const value = await kv.get(key.name)
if (value) chunks.push(value)
}
if (chunks.length === 0) return 0
const combined = chunks.join('\n')
const ts = new Date().toISOString().replace(/[:.]/g, '-')
const hash = shortHash(combined)
const filePath = `telemetry/batch_${ts}_${hash}.jsonl`
const branch = env.HF_DATASET_BRANCH || 'main'
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
if (ok) {
// Delete flushed keys. If a concurrent flush already deleted some,
// the delete is a no-op — KV delete on missing keys is safe.
await Promise.all(allKeys.map(key => kv.delete(key.name)))
console.log(`[Telemetry] Flushed ${allKeys.length} batches → ${filePath}`)
return allKeys.length
}
console.error(`[Telemetry] Flush failed — ${allKeys.length} batches remain in KV`)
return 0
}
// ── HuggingFace Hub Commit ───────────────────────────────────────────