diff --git a/functions/api/telemetry.ts b/functions/api/telemetry.ts index 5857f83..4e09eb2 100644 --- a/functions/api/telemetry.ts +++ b/functions/api/telemetry.ts @@ -377,7 +377,11 @@ async function flushKVToHF(env: Env): Promise { const combined = chunks.join('\n') const keyDigest = await sha256Hex(flushedKeys.map(k => k.name).sort().join(',')) - const filePath = `telemetry/flush_${keyDigest}.jsonl` + // Derive partition from oldest batch key timestamp (stable across retries) + const oldestTs = parseInt(flushedKeys[0].name.split(':')[1], 10) + const d = !isNaN(oldestTs) ? new Date(oldestTs) : new Date() + const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}` + const filePath = `incoming_v2/${datePath}/flush_${keyDigest}.jsonl` const branch = env.HF_DATASET_BRANCH || 'main' const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined) @@ -401,12 +405,16 @@ async function flushMemToHF(env: Env): Promise { // accumulate into a fresh buffer, not into the one we're flushing. const snapshot = memBuffer.splice(0, memBuffer.length) const snapshotBytes = memBufferBytes + const snapshotTs = memBufferFirstTs memBufferBytes = 0 memBufferFirstTs = 0 const combined = snapshot.join('\n') const digest = await sha256Hex(combined) - const filePath = `telemetry/membuf_${digest}.jsonl` + // Derive partition from buffer start time (stable across retries) + const d = snapshotTs > 0 ? new Date(snapshotTs) : new Date() + const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}` + const filePath = `incoming_v2/${datePath}/membuf_${digest}.jsonl` const branch = env.HF_DATASET_BRANCH || 'main' const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined) @@ -420,7 +428,7 @@ async function flushMemToHF(env: Env): Promise { // Prepend so ordering is roughly preserved. memBuffer.unshift(...snapshot) memBufferBytes += snapshotBytes - if (memBufferFirstTs === 0) memBufferFirstTs = Date.now() + if (memBufferFirstTs === 0) memBufferFirstTs = snapshotTs console.error(`[Telemetry] Memory flush failed — ${snapshot.length} batches returned to buffer`) return 0 } diff --git a/scripts/compact-hf-telemetry.mjs b/scripts/compact-hf-telemetry.mjs new file mode 100644 index 0000000..19de5a6 --- /dev/null +++ b/scripts/compact-hf-telemetry.mjs @@ -0,0 +1,548 @@ +#!/usr/bin/env node +/** + * Compact legacy telemetry/ files into monthly archives under archive/legacy/. + * + * Usage: + * node scripts/compact-hf-telemetry.mjs # dry-run (default) + * node scripts/compact-hf-telemetry.mjs --apply # commit archives + delete legacy + * + * Requires: HF_TOKEN env var with write access to the dataset. + * + * Safety guarantees: + * - Dry-run is default; --apply required for any writes + * - FAIL-CLOSED: aborts on ANY download failure + * - Paginates the full file listing (HF API returns max 1000/page) + * - RESUMABLE: derives a stable run ID from source file list hash; + * stores archives under archive/legacy//. On rerun after partial + * deletion, detects existing manifest and resumes deletion from the + * manifest's original source list — never rebuilds archives from a + * reduced file set. + * - Verifies ALL uploaded artifacts (archives + manifest + invalid-records) + * by re-downloading and checking checksums + * - Batches delete operations (max 50 per HF commit) + * - Preserves FULL invalid lines in a separate artifact + * - Post-deletion relist confirms all manifest sources are gone + */ + +const HF_API = 'https://huggingface.co/api'; +const DATASET_REPO = process.env.HF_DATASET_REPO || 'pliny-the-prompter/g0dm0d3'; +const BRANCH = process.env.HF_DATASET_BRANCH || 'main'; +const TOKEN = process.env.HF_TOKEN; +const APPLY = process.argv.includes('--apply'); +const DELETE_BATCH_SIZE = 50; + +if (!TOKEN) { + console.error('ERROR: HF_TOKEN environment variable required'); + process.exit(1); +} + +const headers = { Authorization: `Bearer ${TOKEN}` }; + +// ── Paginated file listing ────────────────────────────────────────── +async function listAllFiles(prefix, recursive = false) { + const allFiles = []; + let cursor = null; + + while (true) { + const params = new URLSearchParams(); + if (cursor) params.set('cursor', cursor); + if (recursive) params.set('recursive', 'true'); + const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}?${params}`; + const res = await fetch(url, { headers }); + if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`); + + const page = await res.json(); + if (!Array.isArray(page) || page.length === 0) break; + + allFiles.push(...page); + console.log(` Listed ${allFiles.length} entries so far...`); + + const linkHeader = res.headers.get('link') || ''; + const nextMatch = linkHeader.match(/<[^>]*[?&]cursor=([^&>]+)[^>]*>;\s*rel="next"/); + if (nextMatch) { + cursor = decodeURIComponent(nextMatch[1]); + } else { + break; + } + } + + return allFiles; +} + +async function downloadFile(path) { + const url = `https://huggingface.co/datasets/${DATASET_REPO}/resolve/${BRANCH}/${path}`; + const res = await fetch(url, { headers }); + if (!res.ok) throw new Error(`Download failed (${res.status}): ${path}`); + return res.text(); +} + +async function fileExists(path) { + const url = `https://huggingface.co/datasets/${DATASET_REPO}/resolve/${BRANCH}/${path}`; + const res = await fetch(url, { method: 'HEAD', headers }); + return res.ok; +} + +async function sha256(text) { + const data = new TextEncoder().encode(text); + const buf = await crypto.subtle.digest('SHA-256', data); + return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join(''); +} + +function utf8ToBase64(str) { + const bytes = new TextEncoder().encode(str); + let binStr = ''; + for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]); + return btoa(binStr); +} + +async function commitFiles(operations, summary) { + const url = `${HF_API}/datasets/${DATASET_REPO}/commit/${BRANCH}`; + const lines = [ + JSON.stringify({ key: 'header', value: { summary } }), + ...operations.map(op => JSON.stringify({ key: op.key, value: op.value })), + ]; + const res = await fetch(url, { + method: 'POST', + headers: { ...headers, 'Content-Type': 'application/x-ndjson' }, + body: lines.join('\n'), + }); + if (!res.ok) throw new Error(`Commit failed (${res.status}): ${(await res.text()).slice(0, 500)}`); + return res.json(); +} + +// ── Derive stable run ID from the sorted source file list ─────────── +async function deriveRunId(sourceFiles) { + const pathList = sourceFiles.map(f => f.path).sort().join('\n'); + const hash = await sha256(pathList); + return hash.slice(0, 12); +} + +async function main() { + console.log(`Mode: ${APPLY ? 'APPLY (will write)' : 'DRY-RUN (read-only)'}`); + console.log(`Repo: ${DATASET_REPO} (branch: ${BRANCH})`); + console.log(''); + + // ── Step 1: Paginated listing ───────────────────────────────────── + console.log('Step 1: Listing ALL legacy telemetry/ files (paginated)...'); + const allEntries = await listAllFiles('telemetry'); + const jsonlFiles = allEntries.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); + console.log(` Total entries: ${allEntries.length}`); + console.log(` JSONL files: ${jsonlFiles.length}`); + + if (jsonlFiles.length === 0) { + console.log(' Nothing to compact. Done.'); + return; + } + + // ── Step 2: Check for existing manifest (resume detection) ──────── + // Derive run ID from the FULL original source list. If a prior run + // already archived everything, its manifest contains the original + // source paths — we can resume deletion without re-downloading. + const currentRunId = await deriveRunId(jsonlFiles); + const runDir = `archive/legacy/${currentRunId}`; + const manifestPath = `${runDir}/manifest.json`; + + console.log(''); + console.log(`Step 2: Checking for existing manifest (run ID: ${currentRunId})...`); + + // Also check if a DIFFERENT run's manifest exists (partial deletion scenario) + let archiveLegacyEntries = []; + try { + archiveLegacyEntries = await listAllFiles('archive/legacy', true); + } catch (err) { + if (!/404|EntryNotFound|does not exist/i.test(String(err.message))) throw err; + } + const existingManifests = archiveLegacyEntries.filter( + f => f.type === 'file' && f.path.endsWith('/manifest.json') + ); + + let resumeManifest = null; + + if (existingManifests.length > 0) { + // Check each manifest to see if it contains source files that are now missing + // (meaning deletion already started from that run) + for (const mf of existingManifests) { + try { + const mfContent = await downloadFile(mf.path); + const mfData = JSON.parse(mfContent); + if (mfData.sourceFilePaths && Array.isArray(mfData.sourceFilePaths)) { + const currentPaths = new Set(jsonlFiles.map(f => f.path)); + const manifestPaths = new Set(mfData.sourceFilePaths); + + // If manifest has paths that no longer exist, deletion already started + const alreadyDeleted = mfData.sourceFilePaths.filter(p => !currentPaths.has(p)); + if (alreadyDeleted.length > 0) { + console.log(` Found prior manifest at ${mf.path}`); + console.log(` Original sources: ${mfData.sourceFilePaths.length}`); + console.log(` Already deleted: ${alreadyDeleted.length}`); + console.log(` Remaining: ${mfData.sourceFilePaths.length - alreadyDeleted.length}`); + console.log(' RESUMING from prior manifest (archives already verified).'); + resumeManifest = mfData; + break; + } + + // If current files exactly match or are a superset, this is a fresh run + // for the same data — use the existing archives if checksums match + if (mf.path === manifestPath) { + console.log(` Found matching manifest at ${manifestPath}`); + console.log(' Verifying existing archives...'); + let allValid = true; + for (const arch of mfData.archives) { + try { + const content = await downloadFile(arch.path); + const checksum = await sha256(content); + if (checksum !== arch.checksum) { + console.error(` CHECKSUM MISMATCH: ${arch.path}`); + allValid = false; + break; + } + } catch { + console.error(` MISSING: ${arch.path}`); + allValid = false; + break; + } + } + if (allValid) { + console.log(' All archives verified. Resuming deletion phase.'); + resumeManifest = mfData; + break; + } else { + console.log(' Archives invalid — rebuilding from scratch.'); + } + } + } + } catch (err) { + console.warn(` Could not parse manifest ${mf.path}: ${err.message}`); + } + } + } + + // ── RESUME PATH: skip download/archive, go straight to deletion ─── + if (resumeManifest) { + if (!APPLY) { + const remaining = resumeManifest.sourceFilePaths.filter( + p => jsonlFiles.some(f => f.path === p) + ); + console.log(''); + console.log('DRY-RUN: Would resume deletion of remaining files.'); + console.log(` Files still to delete: ${remaining.length}`); + console.log(` Batches: ${Math.ceil(remaining.length / DELETE_BATCH_SIZE)}`); + return; + } + + // Delete only files that still exist AND are in the manifest + const manifestPathSet = new Set(resumeManifest.sourceFilePaths); + const toDelete = jsonlFiles.filter(f => manifestPathSet.has(f.path)); + + console.log(''); + console.log(`Resuming deletion: ${toDelete.length} files remaining...`); + + const batches = []; + for (let i = 0; i < toDelete.length; i += DELETE_BATCH_SIZE) { + batches.push(toDelete.slice(i, i + DELETE_BATCH_SIZE)); + } + + for (let i = 0; i < batches.length; i++) { + const batch = batches[i]; + const deleteOps = batch.map(f => ({ + key: 'deletedFile', + value: { path: f.path }, + })); + await commitFiles(deleteOps, `[maintenance] Remove legacy telemetry batch ${i + 1}/${batches.length} (${batch.length} files)`); + console.log(` Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`); + } + + // Post-deletion verification + console.log(''); + console.log('Post-deletion verification: relisting telemetry/...'); + const remaining = await listAllFiles('telemetry'); + const remainingJsonl = remaining.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); + const leftover = remainingJsonl.filter(f => manifestPathSet.has(f.path)); + if (leftover.length > 0) { + console.error(` WARNING: ${leftover.length} manifest sources still exist after deletion!`); + console.error(' Paths:', leftover.map(f => f.path).slice(0, 10).join(', ')); + process.exit(1); + } + console.log(` Confirmed: 0 manifest sources remain. ${remainingJsonl.length} non-manifest files present.`); + + console.log(''); + console.log('DONE (resumed). Legacy telemetry compacted successfully.'); + return; + } + + // ── FRESH PATH: download, archive, verify, delete ───────────────── + + // ── Step 3: Download and parse ALL files (fail-closed) ──────────── + console.log(''); + console.log('Step 3: Downloading and parsing events (fail-closed)...'); + const monthBuckets = {}; + let totalEvents = 0; + let totalSourceBytes = 0; + let downloadFailures = 0; + let invalidLines = 0; + const invalidRecords = []; + const failedFiles = []; + + for (let i = 0; i < jsonlFiles.length; i++) { + const file = jsonlFiles[i]; + if (i % 200 === 0 || i === jsonlFiles.length - 1) { + console.log(` Progress: ${i + 1}/${jsonlFiles.length} files...`); + } + + let content; + try { + content = await downloadFile(file.path); + } catch (err) { + downloadFailures++; + failedFiles.push({ path: file.path, error: err.message }); + continue; + } + + totalSourceBytes += new TextEncoder().encode(content).byteLength; + const lines = content.split('\n').filter(l => l.trim()); + + for (const line of lines) { + try { + const event = JSON.parse(line); + const ts = event.timestamp; + let monthKey; + if (ts && typeof ts === 'number') { + const d = new Date(ts); + monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`; + } else { + monthKey = 'unknown'; + } + if (!monthBuckets[monthKey]) monthBuckets[monthKey] = []; + monthBuckets[monthKey].push(line); + totalEvents++; + } catch { + invalidLines++; + invalidRecords.push({ file: file.path, line }); + } + } + } + + console.log(` Total events parsed: ${totalEvents}`); + console.log(` Total source bytes: ${(totalSourceBytes / 1024 / 1024).toFixed(2)} MiB`); + console.log(` Download failures: ${downloadFailures}`); + console.log(` Invalid lines: ${invalidLines}`); + console.log(` Month buckets: ${Object.keys(monthBuckets).sort().join(', ')}`); + + // ── FAIL-CLOSED: abort if any downloads failed ──────────────────── + if (downloadFailures > 0) { + console.error(''); + console.error(`ABORT: ${downloadFailures} file(s) failed to download.`); + console.error('Cannot guarantee data preservation. Fix network/auth issues and retry.'); + console.error('Failed files:', failedFiles.map(f => f.path).join(', ')); + process.exit(1); + } + + // ── Step 4: Build archives ──────────────────────────────────────── + console.log(''); + console.log(`Step 4: Building monthly archives under ${runDir}/...`); + const archives = []; + for (const [month, events] of Object.entries(monthBuckets).sort()) { + const content = events.join('\n') + '\n'; + const checksum = await sha256(content); + const archivePath = `${runDir}/telemetry-${month}.jsonl`; + const bytes = new TextEncoder().encode(content).byteLength; + archives.push({ month, path: archivePath, events: events.length, bytes, checksum, content }); + console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 16)}...`); + } + + // ── Step 5: Verify counts ───────────────────────────────────────── + const archiveEventTotal = archives.reduce((s, a) => s + a.events, 0); + const sourceFileCount = jsonlFiles.length; + + console.log(''); + console.log('Step 5: Count verification'); + console.log(` Source files examined: ${sourceFileCount}`); + console.log(` Events parsed from sources: ${totalEvents}`); + console.log(` Events in archives: ${archiveEventTotal}`); + console.log(` Invalid lines preserved: ${invalidLines}`); + + if (archiveEventTotal !== totalEvents) { + console.error(' COUNT MISMATCH — aborting. No changes made.'); + process.exit(1); + } + console.log(' COUNTS MATCH.'); + + // ── Build manifest (includes full source path list for resumability) ─ + const invalidRecordsContent = invalidRecords.length > 0 + ? JSON.stringify(invalidRecords, null, 2) + : null; + const invalidRecordsChecksum = invalidRecordsContent + ? await sha256(invalidRecordsContent) + : null; + + const manifest = { + runId: currentRunId, + timestamp: new Date().toISOString(), + repo: DATASET_REPO, + branch: BRANCH, + sourceFiles: sourceFileCount, + sourceBytes: totalSourceBytes, + sourceFilePaths: jsonlFiles.map(f => f.path).sort(), + totalEvents, + invalidLines, + invalidRecordCount: invalidRecords.length, + invalidRecordsChecksum, + archives: archives.map(a => ({ + path: a.path, month: a.month, events: a.events, + bytes: a.bytes, checksum: a.checksum, + })), + }; + + const manifestContent = JSON.stringify(manifest, null, 2); + const manifestChecksum = await sha256(manifestContent); + + console.log(''); + console.log('Manifest:'); + console.log(manifestContent); + + if (!APPLY) { + console.log(''); + console.log('DRY-RUN complete. Re-run with --apply to commit changes.'); + console.log(`Would upload ${archives.length} archive files + manifest + invalid-records.`); + console.log(`Would delete ${sourceFileCount} legacy files in ${Math.ceil(sourceFileCount / DELETE_BATCH_SIZE)} batches.`); + return; + } + + // ── Step 6: Upload archives ─────────────────────────────────────── + console.log(''); + console.log('Step 6: Uploading archives...'); + + // Check if archives already exist and match (idempotent upload) + let archivesAlreadyExist = false; + if (await fileExists(manifestPath)) { + try { + const existing = await downloadFile(manifestPath); + const existingChecksum = await sha256(existing); + if (existingChecksum === manifestChecksum) { + console.log(' Archives already uploaded (manifest checksum matches). Skipping upload.'); + archivesAlreadyExist = true; + } + } catch { /* not found or parse error — upload fresh */ } + } + + if (!archivesAlreadyExist) { + const uploadOps = archives.map(a => ({ + key: 'file', + value: { content: utf8ToBase64(a.content), path: a.path, encoding: 'base64' }, + })); + uploadOps.push({ + key: 'file', + value: { + content: utf8ToBase64(manifestContent), + path: manifestPath, + encoding: 'base64', + }, + }); + if (invalidRecordsContent) { + uploadOps.push({ + key: 'file', + value: { + content: utf8ToBase64(invalidRecordsContent), + path: `${runDir}/invalid-records.json`, + encoding: 'base64', + }, + }); + } + + await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files (run ${currentRunId})`); + console.log(' Archives committed.'); + } + + // ── Step 7: Verify ALL uploaded artifacts ───────────────────────── + console.log(''); + console.log('Step 7: Verifying uploaded artifacts...'); + + // Verify manifest + const dlManifest = await downloadFile(manifestPath); + const dlManifestChecksum = await sha256(dlManifest); + if (dlManifestChecksum !== manifestChecksum) { + console.error(` MANIFEST CHECKSUM MISMATCH!`); + console.error(` Expected: ${manifestChecksum}`); + console.error(` Got: ${dlManifestChecksum}`); + console.error(' ABORTING — will not delete sources.'); + process.exit(1); + } + console.log(` ${manifestPath}: checksum verified`); + + // Verify each archive + for (const archive of archives) { + const downloaded = await downloadFile(archive.path); + const remoteChecksum = await sha256(downloaded); + if (remoteChecksum !== archive.checksum) { + console.error(` CHECKSUM MISMATCH for ${archive.path}!`); + console.error(` Expected: ${archive.checksum}`); + console.error(` Got: ${remoteChecksum}`); + console.error(' ABORTING — archives uploaded but legacy files NOT deleted.'); + process.exit(1); + } + console.log(` ${archive.path}: checksum verified`); + } + + // Verify invalid-records if present + if (invalidRecordsContent) { + const dlInvalid = await downloadFile(`${runDir}/invalid-records.json`); + const dlInvalidChecksum = await sha256(dlInvalid); + if (dlInvalidChecksum !== invalidRecordsChecksum) { + console.error(` INVALID-RECORDS CHECKSUM MISMATCH!`); + console.error(' ABORTING — will not delete sources.'); + process.exit(1); + } + console.log(` ${runDir}/invalid-records.json: checksum verified`); + } + + console.log(' All artifacts verified.'); + + // ── Step 8: Delete legacy files in batches ──────────────────────── + console.log(''); + console.log(`Step 8: Deleting ${sourceFileCount} legacy files in batches of ${DELETE_BATCH_SIZE}...`); + const batches = []; + for (let i = 0; i < jsonlFiles.length; i += DELETE_BATCH_SIZE) { + batches.push(jsonlFiles.slice(i, i + DELETE_BATCH_SIZE)); + } + + for (let i = 0; i < batches.length; i++) { + const batch = batches[i]; + const deleteOps = batch.map(f => ({ + key: 'deletedFile', + value: { path: f.path }, + })); + await commitFiles(deleteOps, `[maintenance] Remove legacy telemetry batch ${i + 1}/${batches.length} (${batch.length} files)`); + console.log(` Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`); + } + + // ── Step 9: Post-deletion verification ──────────────────────────── + console.log(''); + console.log('Step 9: Post-deletion verification...'); + const postDeleteEntries = await listAllFiles('telemetry'); + const postDeleteJsonl = postDeleteEntries.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); + const sourcePathSet = new Set(jsonlFiles.map(f => f.path)); + const leftover = postDeleteJsonl.filter(f => sourcePathSet.has(f.path)); + + if (leftover.length > 0) { + console.error(` WARNING: ${leftover.length} source files still exist after deletion!`); + console.error(' First 10:', leftover.slice(0, 10).map(f => f.path).join(', ')); + console.error(' Rerun with --apply to retry deletion of remaining files.'); + process.exit(1); + } + console.log(` Confirmed: all ${sourceFileCount} manifest sources deleted.`); + if (postDeleteJsonl.length > 0) { + console.log(` Note: ${postDeleteJsonl.length} non-manifest JSONL files remain (new incoming data).`); + } + + console.log(''); + console.log('DONE. Legacy telemetry compacted successfully.'); + console.log(` Run ID: ${currentRunId}`); + console.log(` ${archives.length} monthly archives in ${runDir}/`); + console.log(` ${sourceFileCount} legacy files removed`); + console.log(` ${totalEvents} events preserved`); +} + +main().catch(err => { + console.error('Fatal error:', err); + process.exit(1); +});