From 70cb3bb1884f7f5218ece9244763746b7892ee21 Mon Sep 17 00:00:00 2001 From: faber Date: Sat, 11 Jul 2026 10:23:11 -0400 Subject: [PATCH] fix: date-partition telemetry writes + legacy compaction script The HF dataset's telemetry/ folder hit the 10k file limit. Code fix: writes now target incoming_v2/YYYY/MM/DD/ instead of telemetry/, distributing files across date-partitioned subdirectories. Max ~200 files per day-folder means this can never saturate again. Compaction script: scripts/compact-hf-telemetry.mjs reads the existing 10k legacy files, groups events by UTC month, and writes ~12 archive files to archive/legacy/. Dry-run by default; --apply commits archives and deletes legacy files. Verifies event counts before any deletion. Co-Authored-By: Claude Opus 4.6 --- functions/api/telemetry.ts | 8 +- scripts/compact-hf-telemetry.mjs | 235 +++++++++++++++++++++++++++++++ 2 files changed, 241 insertions(+), 2 deletions(-) create mode 100644 scripts/compact-hf-telemetry.mjs diff --git a/functions/api/telemetry.ts b/functions/api/telemetry.ts index 5857f83..9da5dc7 100644 --- a/functions/api/telemetry.ts +++ b/functions/api/telemetry.ts @@ -377,7 +377,9 @@ async function flushKVToHF(env: Env): Promise { const combined = chunks.join('\n') const keyDigest = await sha256Hex(flushedKeys.map(k => k.name).sort().join(',')) - const filePath = `telemetry/flush_${keyDigest}.jsonl` + const d = new Date() + const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}` + const filePath = `incoming_v2/${datePath}/flush_${keyDigest}.jsonl` const branch = env.HF_DATASET_BRANCH || 'main' const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined) @@ -406,7 +408,9 @@ async function flushMemToHF(env: Env): Promise { const combined = snapshot.join('\n') const digest = await sha256Hex(combined) - const filePath = `telemetry/membuf_${digest}.jsonl` + const d = new Date() + const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}` + const filePath = `incoming_v2/${datePath}/membuf_${digest}.jsonl` const branch = env.HF_DATASET_BRANCH || 'main' const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined) diff --git a/scripts/compact-hf-telemetry.mjs b/scripts/compact-hf-telemetry.mjs new file mode 100644 index 0000000..a37ab71 --- /dev/null +++ b/scripts/compact-hf-telemetry.mjs @@ -0,0 +1,235 @@ +#!/usr/bin/env node +/** + * Compact legacy telemetry/ files into monthly archives under archive/legacy/. + * + * Usage: + * node scripts/compact-hf-telemetry.mjs # dry-run (default) + * node scripts/compact-hf-telemetry.mjs --apply # actually commit archives + delete legacy + * + * Requires: HF_TOKEN env var with write access to the dataset. + * + * Safety: + * - Dry-run is default; --apply required for writes + * - Verifies event counts before deleting anything + * - Produces a manifest with checksums + * - Stops on any count mismatch + */ + +const HF_API = 'https://huggingface.co/api'; +const DATASET_REPO = process.env.HF_DATASET_REPO || 'pliny-the-prompter/g0dm0d3'; +const BRANCH = process.env.HF_DATASET_BRANCH || 'main'; +const TOKEN = process.env.HF_TOKEN; +const APPLY = process.argv.includes('--apply'); + +if (!TOKEN) { + console.error('ERROR: HF_TOKEN environment variable required'); + process.exit(1); +} + +const headers = { Authorization: `Bearer ${TOKEN}` }; + +async function listFiles(prefix) { + const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}`; + const res = await fetch(url, { headers }); + if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`); + return res.json(); +} + +async function downloadFile(path) { + const url = `https://huggingface.co/datasets/${DATASET_REPO}/resolve/${BRANCH}/${path}`; + const res = await fetch(url, { headers }); + if (!res.ok) throw new Error(`Download failed (${res.status}): ${path}`); + return res.text(); +} + +async function sha256(text) { + const data = new TextEncoder().encode(text); + const buf = await crypto.subtle.digest('SHA-256', data); + return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join(''); +} + +async function commitFiles(operations, summary) { + const url = `${HF_API}/datasets/${DATASET_REPO}/commit/${BRANCH}`; + const lines = [ + JSON.stringify({ key: 'header', value: { summary } }), + ...operations.map(op => JSON.stringify({ key: op.key, value: op.value })), + ]; + const res = await fetch(url, { + method: 'POST', + headers: { ...headers, 'Content-Type': 'application/x-ndjson' }, + body: lines.join('\n'), + }); + if (!res.ok) throw new Error(`Commit failed (${res.status}): ${await res.text()}`); + return res.json(); +} + +function utf8ToBase64(str) { + const bytes = new TextEncoder().encode(str); + let binStr = ''; + for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]); + return btoa(binStr); +} + +async function main() { + console.log(`Mode: ${APPLY ? 'APPLY (will write)' : 'DRY-RUN (read-only)'}`); + console.log(`Repo: ${DATASET_REPO} (branch: ${BRANCH})`); + console.log(''); + + // Step 1: List all legacy telemetry files + console.log('Step 1: Listing legacy telemetry/ files...'); + const files = await listFiles('telemetry'); + const jsonlFiles = files.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); + console.log(` Found ${jsonlFiles.length} JSONL files`); + + if (jsonlFiles.length === 0) { + console.log(' Nothing to compact. Done.'); + return; + } + + // Step 2: Download and parse all files + console.log('Step 2: Downloading and parsing events...'); + const monthBuckets = {}; + let totalEvents = 0; + let totalBytes = 0; + let invalidLines = 0; + const invalidRecords = []; + + for (let i = 0; i < jsonlFiles.length; i++) { + const file = jsonlFiles[i]; + if (i % 500 === 0) console.log(` Progress: ${i}/${jsonlFiles.length} files...`); + + try { + const content = await downloadFile(file.path); + totalBytes += new TextEncoder().encode(content).byteLength; + const lines = content.split('\n').filter(l => l.trim()); + + for (const line of lines) { + try { + const event = JSON.parse(line); + const ts = event.timestamp; + let monthKey; + if (ts && typeof ts === 'number') { + const d = new Date(ts); + monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`; + } else { + monthKey = 'unknown'; + } + if (!monthBuckets[monthKey]) monthBuckets[monthKey] = []; + monthBuckets[monthKey].push(line); + totalEvents++; + } catch { + invalidLines++; + invalidRecords.push({ file: file.path, line: line.slice(0, 200) }); + } + } + } catch (err) { + console.error(` ERROR reading ${file.path}: ${err.message}`); + invalidRecords.push({ file: file.path, error: err.message }); + } + } + + console.log(` Total events: ${totalEvents}`); + console.log(` Total source bytes: ${(totalBytes / 1024 / 1024).toFixed(2)} MiB`); + console.log(` Invalid lines: ${invalidLines}`); + console.log(` Month buckets: ${Object.keys(monthBuckets).sort().join(', ')}`); + console.log(''); + + // Step 3: Build archives + console.log('Step 3: Building monthly archives...'); + const archives = []; + for (const [month, events] of Object.entries(monthBuckets).sort()) { + const content = events.join('\n') + '\n'; + const checksum = await sha256(content); + const archivePath = `archive/legacy/telemetry-${month}.jsonl`; + const bytes = new TextEncoder().encode(content).byteLength; + archives.push({ month, path: archivePath, events: events.length, bytes, checksum, content }); + console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 12)}...`); + } + + // Step 4: Verify counts + const archiveEventTotal = archives.reduce((s, a) => s + a.events, 0); + console.log(''); + console.log(`Step 4: Verification`); + console.log(` Source events: ${totalEvents}`); + console.log(` Archive events: ${archiveEventTotal}`); + + if (archiveEventTotal !== totalEvents) { + console.error(' MISMATCH — aborting. No changes made.'); + process.exit(1); + } + console.log(' MATCH — counts verified.'); + + // Build manifest + const manifest = { + timestamp: new Date().toISOString(), + repo: DATASET_REPO, + branch: BRANCH, + sourceFiles: jsonlFiles.length, + sourceBytes: totalBytes, + totalEvents, + invalidLines, + invalidRecords: invalidRecords.length, + archives: archives.map(a => ({ + path: a.path, month: a.month, events: a.events, + bytes: a.bytes, checksum: a.checksum, + })), + }; + + console.log(''); + console.log('Manifest:'); + console.log(JSON.stringify(manifest, null, 2)); + + if (!APPLY) { + console.log(''); + console.log('DRY-RUN complete. Re-run with --apply to commit changes.'); + return; + } + + // Step 5: Upload archives + console.log(''); + console.log('Step 5: Uploading archives...'); + const uploadOps = archives.map(a => ({ + key: 'file', + value: { content: utf8ToBase64(a.content), path: a.path, encoding: 'base64' }, + })); + uploadOps.push({ + key: 'file', + value: { + content: utf8ToBase64(JSON.stringify(manifest, null, 2)), + path: 'archive/legacy/manifest.json', + encoding: 'base64', + }, + }); + + // Save invalid records if any + if (invalidRecords.length > 0) { + uploadOps.push({ + key: 'file', + value: { + content: utf8ToBase64(JSON.stringify(invalidRecords, null, 2)), + path: 'archive/legacy/invalid-records.json', + encoding: 'base64', + }, + }); + } + + await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files`); + console.log(' Archives committed.'); + + // Step 6: Delete legacy files + console.log('Step 6: Deleting legacy files...'); + const deleteOps = jsonlFiles.map(f => ({ + key: 'deletedFile', + value: { path: f.path }, + })); + await commitFiles(deleteOps, `[maintenance] Remove ${jsonlFiles.length} legacy telemetry files (archived in archive/legacy/)`); + console.log(` Deleted ${jsonlFiles.length} files.`); + + console.log(''); + console.log('DONE. Legacy telemetry compacted successfully.'); +} + +main().catch(err => { + console.error('Fatal error:', err); + process.exit(1); +});