From 9f4f898b8e15e043a3af63d694ca578f901524d8 Mon Sep 17 00:00:00 2001 From: faber Date: Sat, 11 Jul 2026 13:03:34 -0400 Subject: [PATCH] fix: resumable compaction + restore memBufferFirstTs on flush failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compaction script (P0 restart-safe migration): - Derives stable run ID from hash of sorted source file paths - Archives stored under versioned archive/legacy// path - Manifest includes full sourceFilePaths list for resume detection - On rerun: detects existing manifest, verifies archives, resumes deletion from the original source list — never rebuilds from a partial file set - Verifies ALL artifacts (manifest + archives + invalid-records) - Post-deletion relist confirms every manifest source is gone - Idempotent upload: skips if manifest checksum already matches Telemetry writer (memBufferFirstTs bug): - On flush failure, restore snapshotTs instead of Date.now() so cross-midnight retries don't change partition paths Co-Authored-By: Claude Opus 4.6 --- functions/api/telemetry.ts | 2 +- scripts/compact-hf-telemetry.mjs | 315 ++++++++++++++++++++++++++----- 2 files changed, 270 insertions(+), 47 deletions(-) diff --git a/functions/api/telemetry.ts b/functions/api/telemetry.ts index c87aea2..4e09eb2 100644 --- a/functions/api/telemetry.ts +++ b/functions/api/telemetry.ts @@ -428,7 +428,7 @@ async function flushMemToHF(env: Env): Promise { // Prepend so ordering is roughly preserved. memBuffer.unshift(...snapshot) memBufferBytes += snapshotBytes - if (memBufferFirstTs === 0) memBufferFirstTs = Date.now() + if (memBufferFirstTs === 0) memBufferFirstTs = snapshotTs console.error(`[Telemetry] Memory flush failed — ${snapshot.length} batches returned to buffer`) return 0 } diff --git a/scripts/compact-hf-telemetry.mjs b/scripts/compact-hf-telemetry.mjs index 60eebed..d23e4ec 100644 --- a/scripts/compact-hf-telemetry.mjs +++ b/scripts/compact-hf-telemetry.mjs @@ -10,12 +10,18 @@ * * Safety guarantees: * - Dry-run is default; --apply required for any writes - * - FAIL-CLOSED: aborts on ANY download failure or parse error threshold + * - FAIL-CLOSED: aborts on ANY download failure * - Paginates the full file listing (HF API returns max 1000/page) - * - Verifies uploaded archives by re-downloading and checking checksums + * - RESUMABLE: derives a stable run ID from source file list hash; + * stores archives under archive/legacy//. On rerun after partial + * deletion, detects existing manifest and resumes deletion from the + * manifest's original source list — never rebuilds archives from a + * reduced file set. + * - Verifies ALL uploaded artifacts (archives + manifest + invalid-records) + * by re-downloading and checking checksums * - Batches delete operations (max 50 per HF commit) - * - Preserves FULL invalid lines (not truncated) in a separate artifact - * - Stops and reports if any count mismatch is detected + * - Preserves FULL invalid lines in a separate artifact + * - Post-deletion relist confirms all manifest sources are gone */ const HF_API = 'https://huggingface.co/api'; @@ -50,7 +56,6 @@ async function listAllFiles(prefix) { allFiles.push(...page); console.log(` Listed ${allFiles.length} entries so far...`); - // HF returns Link header with rel="next" for pagination const linkHeader = res.headers.get('link') || ''; const nextMatch = linkHeader.match(/<[^>]*[?&]cursor=([^&>]+)[^>]*>;\s*rel="next"/); if (nextMatch) { @@ -70,6 +75,12 @@ async function downloadFile(path) { return res.text(); } +async function fileExists(path) { + const url = `https://huggingface.co/datasets/${DATASET_REPO}/resolve/${BRANCH}/${path}`; + const res = await fetch(url, { method: 'HEAD', headers }); + return res.ok; +} + async function sha256(text) { const data = new TextEncoder().encode(text); const buf = await crypto.subtle.digest('SHA-256', data); @@ -98,6 +109,13 @@ async function commitFiles(operations, summary) { return res.json(); } +// ── Derive stable run ID from the sorted source file list ─────────── +async function deriveRunId(sourceFiles) { + const pathList = sourceFiles.map(f => f.path).sort().join('\n'); + const hash = await sha256(pathList); + return hash.slice(0, 12); +} + async function main() { console.log(`Mode: ${APPLY ? 'APPLY (will write)' : 'DRY-RUN (read-only)'}`); console.log(`Repo: ${DATASET_REPO} (branch: ${BRANCH})`); @@ -115,9 +133,142 @@ async function main() { return; } - // ── Step 2: Download and parse ALL files (fail-closed) ──────────── + // ── Step 2: Check for existing manifest (resume detection) ──────── + // Derive run ID from the FULL original source list. If a prior run + // already archived everything, its manifest contains the original + // source paths — we can resume deletion without re-downloading. + const currentRunId = await deriveRunId(jsonlFiles); + const runDir = `archive/legacy/${currentRunId}`; + const manifestPath = `${runDir}/manifest.json`; + console.log(''); - console.log('Step 2: Downloading and parsing events (fail-closed)...'); + console.log(`Step 2: Checking for existing manifest (run ID: ${currentRunId})...`); + + // Also check if a DIFFERENT run's manifest exists (partial deletion scenario) + const archiveLegacyEntries = await listAllFiles('archive/legacy'); + const existingManifests = archiveLegacyEntries.filter( + f => f.type === 'file' && f.path.endsWith('/manifest.json') + ); + + let resumeManifest = null; + + if (existingManifests.length > 0) { + // Check each manifest to see if it contains source files that are now missing + // (meaning deletion already started from that run) + for (const mf of existingManifests) { + try { + const mfContent = await downloadFile(mf.path); + const mfData = JSON.parse(mfContent); + if (mfData.sourceFilePaths && Array.isArray(mfData.sourceFilePaths)) { + const currentPaths = new Set(jsonlFiles.map(f => f.path)); + const manifestPaths = new Set(mfData.sourceFilePaths); + + // If manifest has paths that no longer exist, deletion already started + const alreadyDeleted = mfData.sourceFilePaths.filter(p => !currentPaths.has(p)); + if (alreadyDeleted.length > 0) { + console.log(` Found prior manifest at ${mf.path}`); + console.log(` Original sources: ${mfData.sourceFilePaths.length}`); + console.log(` Already deleted: ${alreadyDeleted.length}`); + console.log(` Remaining: ${mfData.sourceFilePaths.length - alreadyDeleted.length}`); + console.log(' RESUMING from prior manifest (archives already verified).'); + resumeManifest = mfData; + break; + } + + // If current files exactly match or are a superset, this is a fresh run + // for the same data — use the existing archives if checksums match + if (mf.path === manifestPath) { + console.log(` Found matching manifest at ${manifestPath}`); + console.log(' Verifying existing archives...'); + let allValid = true; + for (const arch of mfData.archives) { + try { + const content = await downloadFile(arch.path); + const checksum = await sha256(content); + if (checksum !== arch.checksum) { + console.error(` CHECKSUM MISMATCH: ${arch.path}`); + allValid = false; + break; + } + } catch { + console.error(` MISSING: ${arch.path}`); + allValid = false; + break; + } + } + if (allValid) { + console.log(' All archives verified. Resuming deletion phase.'); + resumeManifest = mfData; + break; + } else { + console.log(' Archives invalid — rebuilding from scratch.'); + } + } + } + } catch (err) { + console.warn(` Could not parse manifest ${mf.path}: ${err.message}`); + } + } + } + + // ── RESUME PATH: skip download/archive, go straight to deletion ─── + if (resumeManifest) { + if (!APPLY) { + const remaining = resumeManifest.sourceFilePaths.filter( + p => jsonlFiles.some(f => f.path === p) + ); + console.log(''); + console.log('DRY-RUN: Would resume deletion of remaining files.'); + console.log(` Files still to delete: ${remaining.length}`); + console.log(` Batches: ${Math.ceil(remaining.length / DELETE_BATCH_SIZE)}`); + return; + } + + // Delete only files that still exist AND are in the manifest + const manifestPathSet = new Set(resumeManifest.sourceFilePaths); + const toDelete = jsonlFiles.filter(f => manifestPathSet.has(f.path)); + + console.log(''); + console.log(`Resuming deletion: ${toDelete.length} files remaining...`); + + const batches = []; + for (let i = 0; i < toDelete.length; i += DELETE_BATCH_SIZE) { + batches.push(toDelete.slice(i, i + DELETE_BATCH_SIZE)); + } + + for (let i = 0; i < batches.length; i++) { + const batch = batches[i]; + const deleteOps = batch.map(f => ({ + key: 'deletedFile', + value: { path: f.path }, + })); + await commitFiles(deleteOps, `[maintenance] Remove legacy telemetry batch ${i + 1}/${batches.length} (${batch.length} files)`); + console.log(` Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`); + } + + // Post-deletion verification + console.log(''); + console.log('Post-deletion verification: relisting telemetry/...'); + const remaining = await listAllFiles('telemetry'); + const remainingJsonl = remaining.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); + const leftover = remainingJsonl.filter(f => manifestPathSet.has(f.path)); + if (leftover.length > 0) { + console.error(` WARNING: ${leftover.length} manifest sources still exist after deletion!`); + console.error(' Paths:', leftover.map(f => f.path).slice(0, 10).join(', ')); + process.exit(1); + } + console.log(` Confirmed: 0 manifest sources remain. ${remainingJsonl.length} non-manifest files present.`); + + console.log(''); + console.log('DONE (resumed). Legacy telemetry compacted successfully.'); + return; + } + + // ── FRESH PATH: download, archive, verify, delete ───────────────── + + // ── Step 3: Download and parse ALL files (fail-closed) ──────────── + console.log(''); + console.log('Step 3: Downloading and parsing events (fail-closed)...'); const monthBuckets = {}; let totalEvents = 0; let totalSourceBytes = 0; @@ -180,99 +331,139 @@ async function main() { process.exit(1); } - // ── Step 3: Build archives ──────────────────────────────────────── + // ── Step 4: Build archives ──────────────────────────────────────── console.log(''); - console.log('Step 3: Building monthly archives...'); + console.log(`Step 4: Building monthly archives under ${runDir}/...`); const archives = []; for (const [month, events] of Object.entries(monthBuckets).sort()) { const content = events.join('\n') + '\n'; const checksum = await sha256(content); - const archivePath = `archive/legacy/telemetry-${month}.jsonl`; + const archivePath = `${runDir}/telemetry-${month}.jsonl`; const bytes = new TextEncoder().encode(content).byteLength; archives.push({ month, path: archivePath, events: events.length, bytes, checksum, content }); console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 16)}...`); } - // ── Step 4: Verify counts (tautology-free) ──────────────────────── + // ── Step 5: Verify counts ───────────────────────────────────────── const archiveEventTotal = archives.reduce((s, a) => s + a.events, 0); const sourceFileCount = jsonlFiles.length; - const expectedTotal = totalEvents; // all successfully parsed events console.log(''); - console.log('Step 4: Verification'); + console.log('Step 5: Count verification'); console.log(` Source files examined: ${sourceFileCount}`); - console.log(` Download failures: ${downloadFailures} (must be 0)`); console.log(` Events parsed from sources: ${totalEvents}`); console.log(` Events in archives: ${archiveEventTotal}`); console.log(` Invalid lines preserved: ${invalidLines}`); - if (archiveEventTotal !== expectedTotal) { + if (archiveEventTotal !== totalEvents) { console.error(' COUNT MISMATCH — aborting. No changes made.'); process.exit(1); } console.log(' COUNTS MATCH.'); - // ── Build manifest ──────────────────────────────────────────────── + // ── Build manifest (includes full source path list for resumability) ─ + const invalidRecordsContent = invalidRecords.length > 0 + ? JSON.stringify(invalidRecords, null, 2) + : null; + const invalidRecordsChecksum = invalidRecordsContent + ? await sha256(invalidRecordsContent) + : null; + const manifest = { + runId: currentRunId, timestamp: new Date().toISOString(), repo: DATASET_REPO, branch: BRANCH, sourceFiles: sourceFileCount, sourceBytes: totalSourceBytes, + sourceFilePaths: jsonlFiles.map(f => f.path).sort(), totalEvents, invalidLines, invalidRecordCount: invalidRecords.length, - downloadFailures, + invalidRecordsChecksum, archives: archives.map(a => ({ path: a.path, month: a.month, events: a.events, bytes: a.bytes, checksum: a.checksum, })), }; + const manifestContent = JSON.stringify(manifest, null, 2); + const manifestChecksum = await sha256(manifestContent); + console.log(''); console.log('Manifest:'); - console.log(JSON.stringify(manifest, null, 2)); + console.log(manifestContent); if (!APPLY) { console.log(''); console.log('DRY-RUN complete. Re-run with --apply to commit changes.'); - console.log(`Would upload ${archives.length} archive files + manifest.`); + console.log(`Would upload ${archives.length} archive files + manifest + invalid-records.`); console.log(`Would delete ${sourceFileCount} legacy files in ${Math.ceil(sourceFileCount / DELETE_BATCH_SIZE)} batches.`); return; } - // ── Step 5: Upload archives ─────────────────────────────────────── + // ── Step 6: Upload archives ─────────────────────────────────────── console.log(''); - console.log('Step 5: Uploading archives...'); - const uploadOps = archives.map(a => ({ - key: 'file', - value: { content: utf8ToBase64(a.content), path: a.path, encoding: 'base64' }, - })); - uploadOps.push({ - key: 'file', - value: { - content: utf8ToBase64(JSON.stringify(manifest, null, 2)), - path: 'archive/legacy/manifest.json', - encoding: 'base64', - }, - }); - if (invalidRecords.length > 0) { + console.log('Step 6: Uploading archives...'); + + // Check if archives already exist and match (idempotent upload) + let archivesAlreadyExist = false; + if (await fileExists(manifestPath)) { + try { + const existing = await downloadFile(manifestPath); + const existingChecksum = await sha256(existing); + if (existingChecksum === manifestChecksum) { + console.log(' Archives already uploaded (manifest checksum matches). Skipping upload.'); + archivesAlreadyExist = true; + } + } catch { /* not found or parse error — upload fresh */ } + } + + if (!archivesAlreadyExist) { + const uploadOps = archives.map(a => ({ + key: 'file', + value: { content: utf8ToBase64(a.content), path: a.path, encoding: 'base64' }, + })); uploadOps.push({ key: 'file', value: { - content: utf8ToBase64(JSON.stringify(invalidRecords, null, 2)), - path: 'archive/legacy/invalid-records.json', + content: utf8ToBase64(manifestContent), + path: manifestPath, encoding: 'base64', }, }); + if (invalidRecordsContent) { + uploadOps.push({ + key: 'file', + value: { + content: utf8ToBase64(invalidRecordsContent), + path: `${runDir}/invalid-records.json`, + encoding: 'base64', + }, + }); + } + + await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files (run ${currentRunId})`); + console.log(' Archives committed.'); } - await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files`); - console.log(' Archives committed.'); - - // ── Step 6: Verify uploaded archives (re-download + checksum) ───── + // ── Step 7: Verify ALL uploaded artifacts ───────────────────────── console.log(''); - console.log('Step 6: Verifying uploaded archives...'); + console.log('Step 7: Verifying uploaded artifacts...'); + + // Verify manifest + const dlManifest = await downloadFile(manifestPath); + const dlManifestChecksum = await sha256(dlManifest); + if (dlManifestChecksum !== manifestChecksum) { + console.error(` MANIFEST CHECKSUM MISMATCH!`); + console.error(` Expected: ${manifestChecksum}`); + console.error(` Got: ${dlManifestChecksum}`); + console.error(' ABORTING — will not delete sources.'); + process.exit(1); + } + console.log(` ${manifestPath}: checksum verified`); + + // Verify each archive for (const archive of archives) { const downloaded = await downloadFile(archive.path); const remoteChecksum = await sha256(downloaded); @@ -281,16 +472,28 @@ async function main() { console.error(` Expected: ${archive.checksum}`); console.error(` Got: ${remoteChecksum}`); console.error(' ABORTING — archives uploaded but legacy files NOT deleted.'); - console.error(' Investigate manually before retrying.'); process.exit(1); } console.log(` ${archive.path}: checksum verified`); } - console.log(' All archives verified.'); - // ── Step 7: Delete legacy files in batches ──────────────────────── + // Verify invalid-records if present + if (invalidRecordsContent) { + const dlInvalid = await downloadFile(`${runDir}/invalid-records.json`); + const dlInvalidChecksum = await sha256(dlInvalid); + if (dlInvalidChecksum !== invalidRecordsChecksum) { + console.error(` INVALID-RECORDS CHECKSUM MISMATCH!`); + console.error(' ABORTING — will not delete sources.'); + process.exit(1); + } + console.log(` ${runDir}/invalid-records.json: checksum verified`); + } + + console.log(' All artifacts verified.'); + + // ── Step 8: Delete legacy files in batches ──────────────────────── console.log(''); - console.log(`Step 7: Deleting ${sourceFileCount} legacy files in batches of ${DELETE_BATCH_SIZE}...`); + console.log(`Step 8: Deleting ${sourceFileCount} legacy files in batches of ${DELETE_BATCH_SIZE}...`); const batches = []; for (let i = 0; i < jsonlFiles.length; i += DELETE_BATCH_SIZE) { batches.push(jsonlFiles.slice(i, i + DELETE_BATCH_SIZE)); @@ -306,9 +509,29 @@ async function main() { console.log(` Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`); } + // ── Step 9: Post-deletion verification ──────────────────────────── + console.log(''); + console.log('Step 9: Post-deletion verification...'); + const postDeleteEntries = await listAllFiles('telemetry'); + const postDeleteJsonl = postDeleteEntries.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); + const sourcePathSet = new Set(jsonlFiles.map(f => f.path)); + const leftover = postDeleteJsonl.filter(f => sourcePathSet.has(f.path)); + + if (leftover.length > 0) { + console.error(` WARNING: ${leftover.length} source files still exist after deletion!`); + console.error(' First 10:', leftover.slice(0, 10).map(f => f.path).join(', ')); + console.error(' Rerun with --apply to retry deletion of remaining files.'); + process.exit(1); + } + console.log(` Confirmed: all ${sourceFileCount} manifest sources deleted.`); + if (postDeleteJsonl.length > 0) { + console.log(` Note: ${postDeleteJsonl.length} non-manifest JSONL files remain (new incoming data).`); + } + console.log(''); console.log('DONE. Legacy telemetry compacted successfully.'); - console.log(` ${archives.length} monthly archives in archive/legacy/`); + console.log(` Run ID: ${currentRunId}`); + console.log(` ${archives.length} monthly archives in ${runDir}/`); console.log(` ${sourceFileCount} legacy files removed`); console.log(` ${totalEvents} events preserved`); }