diff --git a/functions/api/telemetry.ts b/functions/api/telemetry.ts index 9da5dc7..c87aea2 100644 --- a/functions/api/telemetry.ts +++ b/functions/api/telemetry.ts @@ -377,7 +377,9 @@ async function flushKVToHF(env: Env): Promise { const combined = chunks.join('\n') const keyDigest = await sha256Hex(flushedKeys.map(k => k.name).sort().join(',')) - const d = new Date() + // Derive partition from oldest batch key timestamp (stable across retries) + const oldestTs = parseInt(flushedKeys[0].name.split(':')[1], 10) + const d = !isNaN(oldestTs) ? new Date(oldestTs) : new Date() const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}` const filePath = `incoming_v2/${datePath}/flush_${keyDigest}.jsonl` const branch = env.HF_DATASET_BRANCH || 'main' @@ -403,12 +405,14 @@ async function flushMemToHF(env: Env): Promise { // accumulate into a fresh buffer, not into the one we're flushing. const snapshot = memBuffer.splice(0, memBuffer.length) const snapshotBytes = memBufferBytes + const snapshotTs = memBufferFirstTs memBufferBytes = 0 memBufferFirstTs = 0 const combined = snapshot.join('\n') const digest = await sha256Hex(combined) - const d = new Date() + // Derive partition from buffer start time (stable across retries) + const d = snapshotTs > 0 ? new Date(snapshotTs) : new Date() const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}` const filePath = `incoming_v2/${datePath}/membuf_${digest}.jsonl` const branch = env.HF_DATASET_BRANCH || 'main' diff --git a/scripts/compact-hf-telemetry.mjs b/scripts/compact-hf-telemetry.mjs index a37ab71..60eebed 100644 --- a/scripts/compact-hf-telemetry.mjs +++ b/scripts/compact-hf-telemetry.mjs @@ -4,15 +4,18 @@ * * Usage: * node scripts/compact-hf-telemetry.mjs # dry-run (default) - * node scripts/compact-hf-telemetry.mjs --apply # actually commit archives + delete legacy + * node scripts/compact-hf-telemetry.mjs --apply # commit archives + delete legacy * * Requires: HF_TOKEN env var with write access to the dataset. * - * Safety: - * - Dry-run is default; --apply required for writes - * - Verifies event counts before deleting anything - * - Produces a manifest with checksums - * - Stops on any count mismatch + * Safety guarantees: + * - Dry-run is default; --apply required for any writes + * - FAIL-CLOSED: aborts on ANY download failure or parse error threshold + * - Paginates the full file listing (HF API returns max 1000/page) + * - Verifies uploaded archives by re-downloading and checking checksums + * - Batches delete operations (max 50 per HF commit) + * - Preserves FULL invalid lines (not truncated) in a separate artifact + * - Stops and reports if any count mismatch is detected */ const HF_API = 'https://huggingface.co/api'; @@ -20,6 +23,7 @@ const DATASET_REPO = process.env.HF_DATASET_REPO || 'pliny-the-prompter/g0dm0d3' const BRANCH = process.env.HF_DATASET_BRANCH || 'main'; const TOKEN = process.env.HF_TOKEN; const APPLY = process.argv.includes('--apply'); +const DELETE_BATCH_SIZE = 50; if (!TOKEN) { console.error('ERROR: HF_TOKEN environment variable required'); @@ -28,11 +32,35 @@ if (!TOKEN) { const headers = { Authorization: `Bearer ${TOKEN}` }; -async function listFiles(prefix) { - const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}`; - const res = await fetch(url, { headers }); - if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`); - return res.json(); +// ── Paginated file listing ────────────────────────────────────────── +async function listAllFiles(prefix) { + const allFiles = []; + let cursor = null; + + while (true) { + const params = new URLSearchParams(); + if (cursor) params.set('cursor', cursor); + const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}?${params}`; + const res = await fetch(url, { headers }); + if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`); + + const page = await res.json(); + if (!Array.isArray(page) || page.length === 0) break; + + allFiles.push(...page); + console.log(` Listed ${allFiles.length} entries so far...`); + + // HF returns Link header with rel="next" for pagination + const linkHeader = res.headers.get('link') || ''; + const nextMatch = linkHeader.match(/<[^>]*[?&]cursor=([^&>]+)[^>]*>;\s*rel="next"/); + if (nextMatch) { + cursor = decodeURIComponent(nextMatch[1]); + } else { + break; + } + } + + return allFiles; } async function downloadFile(path) { @@ -48,6 +76,13 @@ async function sha256(text) { return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join(''); } +function utf8ToBase64(str) { + const bytes = new TextEncoder().encode(str); + let binStr = ''; + for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]); + return btoa(binStr); +} + async function commitFiles(operations, summary) { const url = `${HF_API}/datasets/${DATASET_REPO}/commit/${BRANCH}`; const lines = [ @@ -59,82 +94,94 @@ async function commitFiles(operations, summary) { headers: { ...headers, 'Content-Type': 'application/x-ndjson' }, body: lines.join('\n'), }); - if (!res.ok) throw new Error(`Commit failed (${res.status}): ${await res.text()}`); + if (!res.ok) throw new Error(`Commit failed (${res.status}): ${(await res.text()).slice(0, 500)}`); return res.json(); } -function utf8ToBase64(str) { - const bytes = new TextEncoder().encode(str); - let binStr = ''; - for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]); - return btoa(binStr); -} - async function main() { console.log(`Mode: ${APPLY ? 'APPLY (will write)' : 'DRY-RUN (read-only)'}`); console.log(`Repo: ${DATASET_REPO} (branch: ${BRANCH})`); console.log(''); - // Step 1: List all legacy telemetry files - console.log('Step 1: Listing legacy telemetry/ files...'); - const files = await listFiles('telemetry'); - const jsonlFiles = files.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); - console.log(` Found ${jsonlFiles.length} JSONL files`); + // ── Step 1: Paginated listing ───────────────────────────────────── + console.log('Step 1: Listing ALL legacy telemetry/ files (paginated)...'); + const allEntries = await listAllFiles('telemetry'); + const jsonlFiles = allEntries.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); + console.log(` Total entries: ${allEntries.length}`); + console.log(` JSONL files: ${jsonlFiles.length}`); if (jsonlFiles.length === 0) { console.log(' Nothing to compact. Done.'); return; } - // Step 2: Download and parse all files - console.log('Step 2: Downloading and parsing events...'); + // ── Step 2: Download and parse ALL files (fail-closed) ──────────── + console.log(''); + console.log('Step 2: Downloading and parsing events (fail-closed)...'); const monthBuckets = {}; let totalEvents = 0; - let totalBytes = 0; + let totalSourceBytes = 0; + let downloadFailures = 0; let invalidLines = 0; const invalidRecords = []; + const failedFiles = []; for (let i = 0; i < jsonlFiles.length; i++) { const file = jsonlFiles[i]; - if (i % 500 === 0) console.log(` Progress: ${i}/${jsonlFiles.length} files...`); + if (i % 200 === 0 || i === jsonlFiles.length - 1) { + console.log(` Progress: ${i + 1}/${jsonlFiles.length} files...`); + } + let content; try { - const content = await downloadFile(file.path); - totalBytes += new TextEncoder().encode(content).byteLength; - const lines = content.split('\n').filter(l => l.trim()); - - for (const line of lines) { - try { - const event = JSON.parse(line); - const ts = event.timestamp; - let monthKey; - if (ts && typeof ts === 'number') { - const d = new Date(ts); - monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`; - } else { - monthKey = 'unknown'; - } - if (!monthBuckets[monthKey]) monthBuckets[monthKey] = []; - monthBuckets[monthKey].push(line); - totalEvents++; - } catch { - invalidLines++; - invalidRecords.push({ file: file.path, line: line.slice(0, 200) }); - } - } + content = await downloadFile(file.path); } catch (err) { - console.error(` ERROR reading ${file.path}: ${err.message}`); - invalidRecords.push({ file: file.path, error: err.message }); + downloadFailures++; + failedFiles.push({ path: file.path, error: err.message }); + continue; + } + + totalSourceBytes += new TextEncoder().encode(content).byteLength; + const lines = content.split('\n').filter(l => l.trim()); + + for (const line of lines) { + try { + const event = JSON.parse(line); + const ts = event.timestamp; + let monthKey; + if (ts && typeof ts === 'number') { + const d = new Date(ts); + monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`; + } else { + monthKey = 'unknown'; + } + if (!monthBuckets[monthKey]) monthBuckets[monthKey] = []; + monthBuckets[monthKey].push(line); + totalEvents++; + } catch { + invalidLines++; + invalidRecords.push({ file: file.path, line }); + } } } - console.log(` Total events: ${totalEvents}`); - console.log(` Total source bytes: ${(totalBytes / 1024 / 1024).toFixed(2)} MiB`); + console.log(` Total events parsed: ${totalEvents}`); + console.log(` Total source bytes: ${(totalSourceBytes / 1024 / 1024).toFixed(2)} MiB`); + console.log(` Download failures: ${downloadFailures}`); console.log(` Invalid lines: ${invalidLines}`); console.log(` Month buckets: ${Object.keys(monthBuckets).sort().join(', ')}`); - console.log(''); - // Step 3: Build archives + // ── FAIL-CLOSED: abort if any downloads failed ──────────────────── + if (downloadFailures > 0) { + console.error(''); + console.error(`ABORT: ${downloadFailures} file(s) failed to download.`); + console.error('Cannot guarantee data preservation. Fix network/auth issues and retry.'); + console.error('Failed files:', failedFiles.map(f => f.path).join(', ')); + process.exit(1); + } + + // ── Step 3: Build archives ──────────────────────────────────────── + console.log(''); console.log('Step 3: Building monthly archives...'); const archives = []; for (const [month, events] of Object.entries(monthBuckets).sort()) { @@ -143,32 +190,39 @@ async function main() { const archivePath = `archive/legacy/telemetry-${month}.jsonl`; const bytes = new TextEncoder().encode(content).byteLength; archives.push({ month, path: archivePath, events: events.length, bytes, checksum, content }); - console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 12)}...`); + console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 16)}...`); } - // Step 4: Verify counts + // ── Step 4: Verify counts (tautology-free) ──────────────────────── const archiveEventTotal = archives.reduce((s, a) => s + a.events, 0); - console.log(''); - console.log(`Step 4: Verification`); - console.log(` Source events: ${totalEvents}`); - console.log(` Archive events: ${archiveEventTotal}`); + const sourceFileCount = jsonlFiles.length; + const expectedTotal = totalEvents; // all successfully parsed events - if (archiveEventTotal !== totalEvents) { - console.error(' MISMATCH — aborting. No changes made.'); + console.log(''); + console.log('Step 4: Verification'); + console.log(` Source files examined: ${sourceFileCount}`); + console.log(` Download failures: ${downloadFailures} (must be 0)`); + console.log(` Events parsed from sources: ${totalEvents}`); + console.log(` Events in archives: ${archiveEventTotal}`); + console.log(` Invalid lines preserved: ${invalidLines}`); + + if (archiveEventTotal !== expectedTotal) { + console.error(' COUNT MISMATCH — aborting. No changes made.'); process.exit(1); } - console.log(' MATCH — counts verified.'); + console.log(' COUNTS MATCH.'); - // Build manifest + // ── Build manifest ──────────────────────────────────────────────── const manifest = { timestamp: new Date().toISOString(), repo: DATASET_REPO, branch: BRANCH, - sourceFiles: jsonlFiles.length, - sourceBytes: totalBytes, + sourceFiles: sourceFileCount, + sourceBytes: totalSourceBytes, totalEvents, invalidLines, - invalidRecords: invalidRecords.length, + invalidRecordCount: invalidRecords.length, + downloadFailures, archives: archives.map(a => ({ path: a.path, month: a.month, events: a.events, bytes: a.bytes, checksum: a.checksum, @@ -182,10 +236,12 @@ async function main() { if (!APPLY) { console.log(''); console.log('DRY-RUN complete. Re-run with --apply to commit changes.'); + console.log(`Would upload ${archives.length} archive files + manifest.`); + console.log(`Would delete ${sourceFileCount} legacy files in ${Math.ceil(sourceFileCount / DELETE_BATCH_SIZE)} batches.`); return; } - // Step 5: Upload archives + // ── Step 5: Upload archives ─────────────────────────────────────── console.log(''); console.log('Step 5: Uploading archives...'); const uploadOps = archives.map(a => ({ @@ -200,8 +256,6 @@ async function main() { encoding: 'base64', }, }); - - // Save invalid records if any if (invalidRecords.length > 0) { uploadOps.push({ key: 'file', @@ -216,17 +270,47 @@ async function main() { await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files`); console.log(' Archives committed.'); - // Step 6: Delete legacy files - console.log('Step 6: Deleting legacy files...'); - const deleteOps = jsonlFiles.map(f => ({ - key: 'deletedFile', - value: { path: f.path }, - })); - await commitFiles(deleteOps, `[maintenance] Remove ${jsonlFiles.length} legacy telemetry files (archived in archive/legacy/)`); - console.log(` Deleted ${jsonlFiles.length} files.`); + // ── Step 6: Verify uploaded archives (re-download + checksum) ───── + console.log(''); + console.log('Step 6: Verifying uploaded archives...'); + for (const archive of archives) { + const downloaded = await downloadFile(archive.path); + const remoteChecksum = await sha256(downloaded); + if (remoteChecksum !== archive.checksum) { + console.error(` CHECKSUM MISMATCH for ${archive.path}!`); + console.error(` Expected: ${archive.checksum}`); + console.error(` Got: ${remoteChecksum}`); + console.error(' ABORTING — archives uploaded but legacy files NOT deleted.'); + console.error(' Investigate manually before retrying.'); + process.exit(1); + } + console.log(` ${archive.path}: checksum verified`); + } + console.log(' All archives verified.'); + + // ── Step 7: Delete legacy files in batches ──────────────────────── + console.log(''); + console.log(`Step 7: Deleting ${sourceFileCount} legacy files in batches of ${DELETE_BATCH_SIZE}...`); + const batches = []; + for (let i = 0; i < jsonlFiles.length; i += DELETE_BATCH_SIZE) { + batches.push(jsonlFiles.slice(i, i + DELETE_BATCH_SIZE)); + } + + for (let i = 0; i < batches.length; i++) { + const batch = batches[i]; + const deleteOps = batch.map(f => ({ + key: 'deletedFile', + value: { path: f.path }, + })); + await commitFiles(deleteOps, `[maintenance] Remove legacy telemetry batch ${i + 1}/${batches.length} (${batch.length} files)`); + console.log(` Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`); + } console.log(''); console.log('DONE. Legacy telemetry compacted successfully.'); + console.log(` ${archives.length} monthly archives in archive/legacy/`); + console.log(` ${sourceFileCount} legacy files removed`); + console.log(` ${totalEvents} events preserved`); } main().catch(err => {