mirror of
https://github.com/elder-plinius/G0DM0D3.git
synced 2026-11-16 07:02:25 +00:00
fix: remove fake KV lock, add GET ?flush=force, add .gitignore
KV lock removed — KV is eventually consistent and can't do atomic CAS. Concurrent flushes may produce duplicate JSONL files on HF, which is harmless (extra telemetry rows, not data loss or corruption). Added GET /api/telemetry?flush=force so an external cron can trigger guaranteed periodic flushes regardless of traffic volume. Added .gitignore (repo had none). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
7
.gitignore
vendored
Normal file
7
.gitignore
vendored
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
node_modules/
|
||||||
|
.next/
|
||||||
|
.env
|
||||||
|
.env.local
|
||||||
|
.wrangler/
|
||||||
|
dist/
|
||||||
|
tsconfig.tsbuildinfo
|
||||||
@@ -108,6 +108,25 @@ export const onRequestOptions: PagesFunction<Env> = async () => {
|
|||||||
return new Response(null, { status: 204, headers: CORS_HEADERS })
|
return new Response(null, { status: 204, headers: CORS_HEADERS })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GET ?flush=force — external cron trigger for guaranteed flush
|
||||||
|
// Set up a CF Worker cron or external cron to hit this every 30 min
|
||||||
|
// to ensure batches don't sit in KV forever during low-traffic periods.
|
||||||
|
export const onRequestGet: PagesFunction<Env> = async (context) => {
|
||||||
|
const { request, env } = context
|
||||||
|
const url = new URL(request.url)
|
||||||
|
|
||||||
|
if (url.searchParams.get('flush') !== 'force') {
|
||||||
|
return jsonResponse({ status: 'ok', hint: 'POST events here, or GET ?flush=force to trigger a flush' }, 200)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!env.TELEMETRY_KV || !env.HF_TOKEN || !env.HF_DATASET_REPO) {
|
||||||
|
return jsonResponse({ error: 'KV or HF not configured' }, 503)
|
||||||
|
}
|
||||||
|
|
||||||
|
const flushed = await flushKVToHF(env)
|
||||||
|
return jsonResponse({ flushed }, 200)
|
||||||
|
}
|
||||||
|
|
||||||
// Main handler
|
// Main handler
|
||||||
export const onRequestPost: PagesFunction<Env> = async (context) => {
|
export const onRequestPost: PagesFunction<Env> = async (context) => {
|
||||||
const { request, env } = context
|
const { request, env } = context
|
||||||
@@ -175,11 +194,11 @@ export const onRequestPost: PagesFunction<Env> = async (context) => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (shouldFlush) {
|
if (shouldFlush) {
|
||||||
// Acquire a simple lock to prevent concurrent flushes
|
// Flush is idempotent: concurrent flushes may produce duplicate
|
||||||
const lockValue = await env.TELEMETRY_KV.get('flush:lock')
|
// JSONL files on HF, which is harmless (extra telemetry rows).
|
||||||
if (!lockValue) {
|
// KV is eventually consistent and can't do atomic locks, so we
|
||||||
context.waitUntil(flushKVToHF(env))
|
// accept rare duplicates rather than pretend we can prevent them.
|
||||||
}
|
context.waitUntil(flushKVToHF(env))
|
||||||
}
|
}
|
||||||
|
|
||||||
return jsonResponse({
|
return jsonResponse({
|
||||||
@@ -207,49 +226,45 @@ export const onRequestPost: PagesFunction<Env> = async (context) => {
|
|||||||
|
|
||||||
// ── KV Flush: combine all pending batches into one HF commit ────────
|
// ── KV Flush: combine all pending batches into one HF commit ────────
|
||||||
|
|
||||||
async function flushKVToHF(env: Env): Promise<void> {
|
async function flushKVToHF(env: Env): Promise<number> {
|
||||||
const kv = env.TELEMETRY_KV!
|
const kv = env.TELEMETRY_KV!
|
||||||
|
const allKeys: { name: string }[] = []
|
||||||
|
|
||||||
// Acquire lock (60s TTL — if flush crashes, lock auto-expires)
|
let cursor: string | undefined
|
||||||
await kv.put('flush:lock', '1', { expirationTtl: 60 })
|
do {
|
||||||
|
const list = await kv.list({ prefix: 'batch:', cursor })
|
||||||
|
allKeys.push(...list.keys)
|
||||||
|
cursor = list.list_complete ? undefined : list.cursor
|
||||||
|
} while (cursor)
|
||||||
|
|
||||||
try {
|
if (allKeys.length === 0) return 0
|
||||||
const allKeys: { name: string }[] = []
|
|
||||||
|
|
||||||
let cursor: string | undefined
|
const chunks: string[] = []
|
||||||
do {
|
for (const key of allKeys) {
|
||||||
const list = await kv.list({ prefix: 'batch:', cursor })
|
const value = await kv.get(key.name)
|
||||||
allKeys.push(...list.keys)
|
if (value) chunks.push(value)
|
||||||
cursor = list.list_complete ? undefined : list.cursor
|
|
||||||
} while (cursor)
|
|
||||||
|
|
||||||
if (allKeys.length === 0) return
|
|
||||||
|
|
||||||
const chunks: string[] = []
|
|
||||||
for (const key of allKeys) {
|
|
||||||
const value = await kv.get(key.name)
|
|
||||||
if (value) chunks.push(value)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (chunks.length === 0) return
|
|
||||||
|
|
||||||
const combined = chunks.join('\n')
|
|
||||||
const ts = new Date().toISOString().replace(/[:.]/g, '-')
|
|
||||||
const hash = shortHash(combined)
|
|
||||||
const filePath = `telemetry/batch_${ts}_${hash}.jsonl`
|
|
||||||
const branch = env.HF_DATASET_BRANCH || 'main'
|
|
||||||
|
|
||||||
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
|
|
||||||
|
|
||||||
if (ok) {
|
|
||||||
await Promise.all(allKeys.map(key => kv.delete(key.name)))
|
|
||||||
console.log(`[Telemetry] Flushed ${allKeys.length} batches → ${filePath}`)
|
|
||||||
} else {
|
|
||||||
console.error(`[Telemetry] Flush failed — ${allKeys.length} batches remain in KV`)
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
await kv.delete('flush:lock')
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (chunks.length === 0) return 0
|
||||||
|
|
||||||
|
const combined = chunks.join('\n')
|
||||||
|
const ts = new Date().toISOString().replace(/[:.]/g, '-')
|
||||||
|
const hash = shortHash(combined)
|
||||||
|
const filePath = `telemetry/batch_${ts}_${hash}.jsonl`
|
||||||
|
const branch = env.HF_DATASET_BRANCH || 'main'
|
||||||
|
|
||||||
|
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
|
||||||
|
|
||||||
|
if (ok) {
|
||||||
|
// Delete flushed keys. If a concurrent flush already deleted some,
|
||||||
|
// the delete is a no-op — KV delete on missing keys is safe.
|
||||||
|
await Promise.all(allKeys.map(key => kv.delete(key.name)))
|
||||||
|
console.log(`[Telemetry] Flushed ${allKeys.length} batches → ${filePath}`)
|
||||||
|
return allKeys.length
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error(`[Telemetry] Flush failed — ${allKeys.length} batches remain in KV`)
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── HuggingFace Hub Commit ───────────────────────────────────────────
|
// ── HuggingFace Hub Commit ───────────────────────────────────────────
|
||||||
|
|||||||
Reference in New Issue
Block a user