fix: derive flush filename from SHA-256 digest only (no timestamp)

Concurrent flushes reading identical KV keys now produce the exact same
filename (`flush_<sha256>.jsonl`), making HF overwrites truly idempotent.
The previous approach included a timestamp which broke idempotency — two
isolates hitting the threshold simultaneously would write different files
containing duplicate data.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
faber
2026-07-10 16:55:27 -04:00
parent b5992aea15
commit 132ef2e2ac

View File

@@ -260,13 +260,11 @@ async function flushKVToHF(env: Env): Promise<number> {
if (chunks.length === 0) return 0
const combined = chunks.join('\n')
// Derive filename from the key set — if two concurrent flushes read
// the same keys, they produce the same filename and HF overwrites
// (idempotent). This prevents duplicate data from concurrent flushes.
const keyFingerprint = shortHash(allKeys.map(k => k.name).sort().join(','))
const ts = new Date().toISOString().replace(/[:.]/g, '-')
const hash = keyFingerprint
const filePath = `telemetry/batch_${ts}_${hash}.jsonl`
// Filename is derived entirely from the key set — no timestamp.
// Concurrent flushes reading identical keys produce the exact same
// filename, so HF overwrites idempotently (no duplicate rows).
const keyDigest = await sha256Hex(allKeys.map(k => k.name).sort().join(','))
const filePath = `telemetry/flush_${keyDigest}.jsonl`
const branch = env.HF_DATASET_BRANCH || 'main'
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
@@ -392,6 +390,12 @@ function shortHash(str: string): string {
return Math.abs(h).toString(36).slice(0, 6)
}
async function sha256Hex(str: string): Promise<string> {
const data = new TextEncoder().encode(str)
const buf = await crypto.subtle.digest('SHA-256', data)
return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('')
}
function jsonResponse(data: unknown, status: number): Response {
return new Response(JSON.stringify(data), {
status,