mirror of
https://github.com/elder-plinius/G0DM0D3.git
synced 2026-11-16 07:02:25 +00:00
fix: remove fake KV lock, add GET ?flush=force, add .gitignore
KV lock removed — KV is eventually consistent and can't do atomic CAS. Concurrent flushes may produce duplicate JSONL files on HF, which is harmless (extra telemetry rows, not data loss or corruption). Added GET /api/telemetry?flush=force so an external cron can trigger guaranteed periodic flushes regardless of traffic volume. Added .gitignore (repo had none). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
7
.gitignore
vendored
Normal file
7
.gitignore
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
node_modules/
|
||||
.next/
|
||||
.env
|
||||
.env.local
|
||||
.wrangler/
|
||||
dist/
|
||||
tsconfig.tsbuildinfo
|
||||
@@ -108,6 +108,25 @@ export const onRequestOptions: PagesFunction<Env> = async () => {
|
||||
return new Response(null, { status: 204, headers: CORS_HEADERS })
|
||||
}
|
||||
|
||||
// GET ?flush=force — external cron trigger for guaranteed flush
|
||||
// Set up a CF Worker cron or external cron to hit this every 30 min
|
||||
// to ensure batches don't sit in KV forever during low-traffic periods.
|
||||
export const onRequestGet: PagesFunction<Env> = async (context) => {
|
||||
const { request, env } = context
|
||||
const url = new URL(request.url)
|
||||
|
||||
if (url.searchParams.get('flush') !== 'force') {
|
||||
return jsonResponse({ status: 'ok', hint: 'POST events here, or GET ?flush=force to trigger a flush' }, 200)
|
||||
}
|
||||
|
||||
if (!env.TELEMETRY_KV || !env.HF_TOKEN || !env.HF_DATASET_REPO) {
|
||||
return jsonResponse({ error: 'KV or HF not configured' }, 503)
|
||||
}
|
||||
|
||||
const flushed = await flushKVToHF(env)
|
||||
return jsonResponse({ flushed }, 200)
|
||||
}
|
||||
|
||||
// Main handler
|
||||
export const onRequestPost: PagesFunction<Env> = async (context) => {
|
||||
const { request, env } = context
|
||||
@@ -175,11 +194,11 @@ export const onRequestPost: PagesFunction<Env> = async (context) => {
|
||||
}
|
||||
|
||||
if (shouldFlush) {
|
||||
// Acquire a simple lock to prevent concurrent flushes
|
||||
const lockValue = await env.TELEMETRY_KV.get('flush:lock')
|
||||
if (!lockValue) {
|
||||
context.waitUntil(flushKVToHF(env))
|
||||
}
|
||||
// Flush is idempotent: concurrent flushes may produce duplicate
|
||||
// JSONL files on HF, which is harmless (extra telemetry rows).
|
||||
// KV is eventually consistent and can't do atomic locks, so we
|
||||
// accept rare duplicates rather than pretend we can prevent them.
|
||||
context.waitUntil(flushKVToHF(env))
|
||||
}
|
||||
|
||||
return jsonResponse({
|
||||
@@ -207,49 +226,45 @@ export const onRequestPost: PagesFunction<Env> = async (context) => {
|
||||
|
||||
// ── KV Flush: combine all pending batches into one HF commit ────────
|
||||
|
||||
async function flushKVToHF(env: Env): Promise<void> {
|
||||
async function flushKVToHF(env: Env): Promise<number> {
|
||||
const kv = env.TELEMETRY_KV!
|
||||
const allKeys: { name: string }[] = []
|
||||
|
||||
// Acquire lock (60s TTL — if flush crashes, lock auto-expires)
|
||||
await kv.put('flush:lock', '1', { expirationTtl: 60 })
|
||||
let cursor: string | undefined
|
||||
do {
|
||||
const list = await kv.list({ prefix: 'batch:', cursor })
|
||||
allKeys.push(...list.keys)
|
||||
cursor = list.list_complete ? undefined : list.cursor
|
||||
} while (cursor)
|
||||
|
||||
try {
|
||||
const allKeys: { name: string }[] = []
|
||||
if (allKeys.length === 0) return 0
|
||||
|
||||
let cursor: string | undefined
|
||||
do {
|
||||
const list = await kv.list({ prefix: 'batch:', cursor })
|
||||
allKeys.push(...list.keys)
|
||||
cursor = list.list_complete ? undefined : list.cursor
|
||||
} while (cursor)
|
||||
|
||||
if (allKeys.length === 0) return
|
||||
|
||||
const chunks: string[] = []
|
||||
for (const key of allKeys) {
|
||||
const value = await kv.get(key.name)
|
||||
if (value) chunks.push(value)
|
||||
}
|
||||
|
||||
if (chunks.length === 0) return
|
||||
|
||||
const combined = chunks.join('\n')
|
||||
const ts = new Date().toISOString().replace(/[:.]/g, '-')
|
||||
const hash = shortHash(combined)
|
||||
const filePath = `telemetry/batch_${ts}_${hash}.jsonl`
|
||||
const branch = env.HF_DATASET_BRANCH || 'main'
|
||||
|
||||
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
|
||||
|
||||
if (ok) {
|
||||
await Promise.all(allKeys.map(key => kv.delete(key.name)))
|
||||
console.log(`[Telemetry] Flushed ${allKeys.length} batches → ${filePath}`)
|
||||
} else {
|
||||
console.error(`[Telemetry] Flush failed — ${allKeys.length} batches remain in KV`)
|
||||
}
|
||||
} finally {
|
||||
await kv.delete('flush:lock')
|
||||
const chunks: string[] = []
|
||||
for (const key of allKeys) {
|
||||
const value = await kv.get(key.name)
|
||||
if (value) chunks.push(value)
|
||||
}
|
||||
|
||||
if (chunks.length === 0) return 0
|
||||
|
||||
const combined = chunks.join('\n')
|
||||
const ts = new Date().toISOString().replace(/[:.]/g, '-')
|
||||
const hash = shortHash(combined)
|
||||
const filePath = `telemetry/batch_${ts}_${hash}.jsonl`
|
||||
const branch = env.HF_DATASET_BRANCH || 'main'
|
||||
|
||||
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
|
||||
|
||||
if (ok) {
|
||||
// Delete flushed keys. If a concurrent flush already deleted some,
|
||||
// the delete is a no-op — KV delete on missing keys is safe.
|
||||
await Promise.all(allKeys.map(key => kv.delete(key.name)))
|
||||
console.log(`[Telemetry] Flushed ${allKeys.length} batches → ${filePath}`)
|
||||
return allKeys.length
|
||||
}
|
||||
|
||||
console.error(`[Telemetry] Flush failed — ${allKeys.length} batches remain in KV`)
|
||||
return 0
|
||||
}
|
||||
|
||||
// ── HuggingFace Hub Commit ───────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user