mirror of
https://github.com/elder-plinius/G0DM0D3.git
synced 2026-11-16 07:02:25 +00:00
fix: derive flush filename from SHA-256 digest only (no timestamp)
Concurrent flushes reading identical KV keys now produce the exact same filename (`flush_<sha256>.jsonl`), making HF overwrites truly idempotent. The previous approach included a timestamp which broke idempotency — two isolates hitting the threshold simultaneously would write different files containing duplicate data. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -260,13 +260,11 @@ async function flushKVToHF(env: Env): Promise<number> {
|
||||
if (chunks.length === 0) return 0
|
||||
|
||||
const combined = chunks.join('\n')
|
||||
// Derive filename from the key set — if two concurrent flushes read
|
||||
// the same keys, they produce the same filename and HF overwrites
|
||||
// (idempotent). This prevents duplicate data from concurrent flushes.
|
||||
const keyFingerprint = shortHash(allKeys.map(k => k.name).sort().join(','))
|
||||
const ts = new Date().toISOString().replace(/[:.]/g, '-')
|
||||
const hash = keyFingerprint
|
||||
const filePath = `telemetry/batch_${ts}_${hash}.jsonl`
|
||||
// Filename is derived entirely from the key set — no timestamp.
|
||||
// Concurrent flushes reading identical keys produce the exact same
|
||||
// filename, so HF overwrites idempotently (no duplicate rows).
|
||||
const keyDigest = await sha256Hex(allKeys.map(k => k.name).sort().join(','))
|
||||
const filePath = `telemetry/flush_${keyDigest}.jsonl`
|
||||
const branch = env.HF_DATASET_BRANCH || 'main'
|
||||
|
||||
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
|
||||
@@ -392,6 +390,12 @@ function shortHash(str: string): string {
|
||||
return Math.abs(h).toString(36).slice(0, 6)
|
||||
}
|
||||
|
||||
async function sha256Hex(str: string): Promise<string> {
|
||||
const data = new TextEncoder().encode(str)
|
||||
const buf = await crypto.subtle.digest('SHA-256', data)
|
||||
return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('')
|
||||
}
|
||||
|
||||
function jsonResponse(data: unknown, status: number): Response {
|
||||
return new Response(JSON.stringify(data), {
|
||||
status,
|
||||
|
||||
Reference in New Issue
Block a user