fix: resumable compaction + restore memBufferFirstTs on flush failure

Compaction script (P0 restart-safe migration):
- Derives stable run ID from hash of sorted source file paths
- Archives stored under versioned archive/legacy/<runId>/ path
- Manifest includes full sourceFilePaths list for resume detection
- On rerun: detects existing manifest, verifies archives, resumes
  deletion from the original source list — never rebuilds from
  a partial file set
- Verifies ALL artifacts (manifest + archives + invalid-records)
- Post-deletion relist confirms every manifest source is gone
- Idempotent upload: skips if manifest checksum already matches

Telemetry writer (memBufferFirstTs bug):
- On flush failure, restore snapshotTs instead of Date.now() so
  cross-midnight retries don't change partition paths

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
faber
2026-07-11 13:03:34 -04:00
parent 2896b530ac
commit 9f4f898b8e
2 changed files with 270 additions and 47 deletions

View File

@@ -428,7 +428,7 @@ async function flushMemToHF(env: Env): Promise<number> {
// Prepend so ordering is roughly preserved.
memBuffer.unshift(...snapshot)
memBufferBytes += snapshotBytes
if (memBufferFirstTs === 0) memBufferFirstTs = Date.now()
if (memBufferFirstTs === 0) memBufferFirstTs = snapshotTs
console.error(`[Telemetry] Memory flush failed — ${snapshot.length} batches returned to buffer`)
return 0
}

View File

@@ -10,12 +10,18 @@
*
* Safety guarantees:
* - Dry-run is default; --apply required for any writes
* - FAIL-CLOSED: aborts on ANY download failure or parse error threshold
* - FAIL-CLOSED: aborts on ANY download failure
* - Paginates the full file listing (HF API returns max 1000/page)
* - Verifies uploaded archives by re-downloading and checking checksums
* - RESUMABLE: derives a stable run ID from source file list hash;
* stores archives under archive/legacy/<runId>/. On rerun after partial
* deletion, detects existing manifest and resumes deletion from the
* manifest's original source list — never rebuilds archives from a
* reduced file set.
* - Verifies ALL uploaded artifacts (archives + manifest + invalid-records)
* by re-downloading and checking checksums
* - Batches delete operations (max 50 per HF commit)
* - Preserves FULL invalid lines (not truncated) in a separate artifact
* - Stops and reports if any count mismatch is detected
* - Preserves FULL invalid lines in a separate artifact
* - Post-deletion relist confirms all manifest sources are gone
*/
const HF_API = 'https://huggingface.co/api';
@@ -50,7 +56,6 @@ async function listAllFiles(prefix) {
allFiles.push(...page);
console.log(` Listed ${allFiles.length} entries so far...`);
// HF returns Link header with rel="next" for pagination
const linkHeader = res.headers.get('link') || '';
const nextMatch = linkHeader.match(/<[^>]*[?&]cursor=([^&>]+)[^>]*>;\s*rel="next"/);
if (nextMatch) {
@@ -70,6 +75,12 @@ async function downloadFile(path) {
return res.text();
}
async function fileExists(path) {
const url = `https://huggingface.co/datasets/${DATASET_REPO}/resolve/${BRANCH}/${path}`;
const res = await fetch(url, { method: 'HEAD', headers });
return res.ok;
}
async function sha256(text) {
const data = new TextEncoder().encode(text);
const buf = await crypto.subtle.digest('SHA-256', data);
@@ -98,6 +109,13 @@ async function commitFiles(operations, summary) {
return res.json();
}
// ── Derive stable run ID from the sorted source file list ───────────
async function deriveRunId(sourceFiles) {
const pathList = sourceFiles.map(f => f.path).sort().join('\n');
const hash = await sha256(pathList);
return hash.slice(0, 12);
}
async function main() {
console.log(`Mode: ${APPLY ? 'APPLY (will write)' : 'DRY-RUN (read-only)'}`);
console.log(`Repo: ${DATASET_REPO} (branch: ${BRANCH})`);
@@ -115,9 +133,142 @@ async function main() {
return;
}
// ── Step 2: Download and parse ALL files (fail-closed) ────────────
// ── Step 2: Check for existing manifest (resume detection) ────────
// Derive run ID from the FULL original source list. If a prior run
// already archived everything, its manifest contains the original
// source paths — we can resume deletion without re-downloading.
const currentRunId = await deriveRunId(jsonlFiles);
const runDir = `archive/legacy/${currentRunId}`;
const manifestPath = `${runDir}/manifest.json`;
console.log('');
console.log('Step 2: Downloading and parsing events (fail-closed)...');
console.log(`Step 2: Checking for existing manifest (run ID: ${currentRunId})...`);
// Also check if a DIFFERENT run's manifest exists (partial deletion scenario)
const archiveLegacyEntries = await listAllFiles('archive/legacy');
const existingManifests = archiveLegacyEntries.filter(
f => f.type === 'file' && f.path.endsWith('/manifest.json')
);
let resumeManifest = null;
if (existingManifests.length > 0) {
// Check each manifest to see if it contains source files that are now missing
// (meaning deletion already started from that run)
for (const mf of existingManifests) {
try {
const mfContent = await downloadFile(mf.path);
const mfData = JSON.parse(mfContent);
if (mfData.sourceFilePaths && Array.isArray(mfData.sourceFilePaths)) {
const currentPaths = new Set(jsonlFiles.map(f => f.path));
const manifestPaths = new Set(mfData.sourceFilePaths);
// If manifest has paths that no longer exist, deletion already started
const alreadyDeleted = mfData.sourceFilePaths.filter(p => !currentPaths.has(p));
if (alreadyDeleted.length > 0) {
console.log(` Found prior manifest at ${mf.path}`);
console.log(` Original sources: ${mfData.sourceFilePaths.length}`);
console.log(` Already deleted: ${alreadyDeleted.length}`);
console.log(` Remaining: ${mfData.sourceFilePaths.length - alreadyDeleted.length}`);
console.log(' RESUMING from prior manifest (archives already verified).');
resumeManifest = mfData;
break;
}
// If current files exactly match or are a superset, this is a fresh run
// for the same data — use the existing archives if checksums match
if (mf.path === manifestPath) {
console.log(` Found matching manifest at ${manifestPath}`);
console.log(' Verifying existing archives...');
let allValid = true;
for (const arch of mfData.archives) {
try {
const content = await downloadFile(arch.path);
const checksum = await sha256(content);
if (checksum !== arch.checksum) {
console.error(` CHECKSUM MISMATCH: ${arch.path}`);
allValid = false;
break;
}
} catch {
console.error(` MISSING: ${arch.path}`);
allValid = false;
break;
}
}
if (allValid) {
console.log(' All archives verified. Resuming deletion phase.');
resumeManifest = mfData;
break;
} else {
console.log(' Archives invalid — rebuilding from scratch.');
}
}
}
} catch (err) {
console.warn(` Could not parse manifest ${mf.path}: ${err.message}`);
}
}
}
// ── RESUME PATH: skip download/archive, go straight to deletion ───
if (resumeManifest) {
if (!APPLY) {
const remaining = resumeManifest.sourceFilePaths.filter(
p => jsonlFiles.some(f => f.path === p)
);
console.log('');
console.log('DRY-RUN: Would resume deletion of remaining files.');
console.log(` Files still to delete: ${remaining.length}`);
console.log(` Batches: ${Math.ceil(remaining.length / DELETE_BATCH_SIZE)}`);
return;
}
// Delete only files that still exist AND are in the manifest
const manifestPathSet = new Set(resumeManifest.sourceFilePaths);
const toDelete = jsonlFiles.filter(f => manifestPathSet.has(f.path));
console.log('');
console.log(`Resuming deletion: ${toDelete.length} files remaining...`);
const batches = [];
for (let i = 0; i < toDelete.length; i += DELETE_BATCH_SIZE) {
batches.push(toDelete.slice(i, i + DELETE_BATCH_SIZE));
}
for (let i = 0; i < batches.length; i++) {
const batch = batches[i];
const deleteOps = batch.map(f => ({
key: 'deletedFile',
value: { path: f.path },
}));
await commitFiles(deleteOps, `[maintenance] Remove legacy telemetry batch ${i + 1}/${batches.length} (${batch.length} files)`);
console.log(` Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`);
}
// Post-deletion verification
console.log('');
console.log('Post-deletion verification: relisting telemetry/...');
const remaining = await listAllFiles('telemetry');
const remainingJsonl = remaining.filter(f => f.type === 'file' && f.path.endsWith('.jsonl'));
const leftover = remainingJsonl.filter(f => manifestPathSet.has(f.path));
if (leftover.length > 0) {
console.error(` WARNING: ${leftover.length} manifest sources still exist after deletion!`);
console.error(' Paths:', leftover.map(f => f.path).slice(0, 10).join(', '));
process.exit(1);
}
console.log(` Confirmed: 0 manifest sources remain. ${remainingJsonl.length} non-manifest files present.`);
console.log('');
console.log('DONE (resumed). Legacy telemetry compacted successfully.');
return;
}
// ── FRESH PATH: download, archive, verify, delete ─────────────────
// ── Step 3: Download and parse ALL files (fail-closed) ────────────
console.log('');
console.log('Step 3: Downloading and parsing events (fail-closed)...');
const monthBuckets = {};
let totalEvents = 0;
let totalSourceBytes = 0;
@@ -180,99 +331,139 @@ async function main() {
process.exit(1);
}
// ── Step 3: Build archives ────────────────────────────────────────
// ── Step 4: Build archives ────────────────────────────────────────
console.log('');
console.log('Step 3: Building monthly archives...');
console.log(`Step 4: Building monthly archives under ${runDir}/...`);
const archives = [];
for (const [month, events] of Object.entries(monthBuckets).sort()) {
const content = events.join('\n') + '\n';
const checksum = await sha256(content);
const archivePath = `archive/legacy/telemetry-${month}.jsonl`;
const archivePath = `${runDir}/telemetry-${month}.jsonl`;
const bytes = new TextEncoder().encode(content).byteLength;
archives.push({ month, path: archivePath, events: events.length, bytes, checksum, content });
console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 16)}...`);
}
// ── Step 4: Verify counts (tautology-free) ────────────────────────
// ── Step 5: Verify counts ─────────────────────────────────────────
const archiveEventTotal = archives.reduce((s, a) => s + a.events, 0);
const sourceFileCount = jsonlFiles.length;
const expectedTotal = totalEvents; // all successfully parsed events
console.log('');
console.log('Step 4: Verification');
console.log('Step 5: Count verification');
console.log(` Source files examined: ${sourceFileCount}`);
console.log(` Download failures: ${downloadFailures} (must be 0)`);
console.log(` Events parsed from sources: ${totalEvents}`);
console.log(` Events in archives: ${archiveEventTotal}`);
console.log(` Invalid lines preserved: ${invalidLines}`);
if (archiveEventTotal !== expectedTotal) {
if (archiveEventTotal !== totalEvents) {
console.error(' COUNT MISMATCH — aborting. No changes made.');
process.exit(1);
}
console.log(' COUNTS MATCH.');
// ── Build manifest ────────────────────────────────────────────────
// ── Build manifest (includes full source path list for resumability) ─
const invalidRecordsContent = invalidRecords.length > 0
? JSON.stringify(invalidRecords, null, 2)
: null;
const invalidRecordsChecksum = invalidRecordsContent
? await sha256(invalidRecordsContent)
: null;
const manifest = {
runId: currentRunId,
timestamp: new Date().toISOString(),
repo: DATASET_REPO,
branch: BRANCH,
sourceFiles: sourceFileCount,
sourceBytes: totalSourceBytes,
sourceFilePaths: jsonlFiles.map(f => f.path).sort(),
totalEvents,
invalidLines,
invalidRecordCount: invalidRecords.length,
downloadFailures,
invalidRecordsChecksum,
archives: archives.map(a => ({
path: a.path, month: a.month, events: a.events,
bytes: a.bytes, checksum: a.checksum,
})),
};
const manifestContent = JSON.stringify(manifest, null, 2);
const manifestChecksum = await sha256(manifestContent);
console.log('');
console.log('Manifest:');
console.log(JSON.stringify(manifest, null, 2));
console.log(manifestContent);
if (!APPLY) {
console.log('');
console.log('DRY-RUN complete. Re-run with --apply to commit changes.');
console.log(`Would upload ${archives.length} archive files + manifest.`);
console.log(`Would upload ${archives.length} archive files + manifest + invalid-records.`);
console.log(`Would delete ${sourceFileCount} legacy files in ${Math.ceil(sourceFileCount / DELETE_BATCH_SIZE)} batches.`);
return;
}
// ── Step 5: Upload archives ───────────────────────────────────────
// ── Step 6: Upload archives ───────────────────────────────────────
console.log('');
console.log('Step 5: Uploading archives...');
const uploadOps = archives.map(a => ({
key: 'file',
value: { content: utf8ToBase64(a.content), path: a.path, encoding: 'base64' },
}));
uploadOps.push({
key: 'file',
value: {
content: utf8ToBase64(JSON.stringify(manifest, null, 2)),
path: 'archive/legacy/manifest.json',
encoding: 'base64',
},
});
if (invalidRecords.length > 0) {
console.log('Step 6: Uploading archives...');
// Check if archives already exist and match (idempotent upload)
let archivesAlreadyExist = false;
if (await fileExists(manifestPath)) {
try {
const existing = await downloadFile(manifestPath);
const existingChecksum = await sha256(existing);
if (existingChecksum === manifestChecksum) {
console.log(' Archives already uploaded (manifest checksum matches). Skipping upload.');
archivesAlreadyExist = true;
}
} catch { /* not found or parse error — upload fresh */ }
}
if (!archivesAlreadyExist) {
const uploadOps = archives.map(a => ({
key: 'file',
value: { content: utf8ToBase64(a.content), path: a.path, encoding: 'base64' },
}));
uploadOps.push({
key: 'file',
value: {
content: utf8ToBase64(JSON.stringify(invalidRecords, null, 2)),
path: 'archive/legacy/invalid-records.json',
content: utf8ToBase64(manifestContent),
path: manifestPath,
encoding: 'base64',
},
});
if (invalidRecordsContent) {
uploadOps.push({
key: 'file',
value: {
content: utf8ToBase64(invalidRecordsContent),
path: `${runDir}/invalid-records.json`,
encoding: 'base64',
},
});
}
await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files (run ${currentRunId})`);
console.log(' Archives committed.');
}
await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files`);
console.log(' Archives committed.');
// ── Step 6: Verify uploaded archives (re-download + checksum) ─────
// ── Step 7: Verify ALL uploaded artifacts ─────────────────────────
console.log('');
console.log('Step 6: Verifying uploaded archives...');
console.log('Step 7: Verifying uploaded artifacts...');
// Verify manifest
const dlManifest = await downloadFile(manifestPath);
const dlManifestChecksum = await sha256(dlManifest);
if (dlManifestChecksum !== manifestChecksum) {
console.error(` MANIFEST CHECKSUM MISMATCH!`);
console.error(` Expected: ${manifestChecksum}`);
console.error(` Got: ${dlManifestChecksum}`);
console.error(' ABORTING — will not delete sources.');
process.exit(1);
}
console.log(` ${manifestPath}: checksum verified`);
// Verify each archive
for (const archive of archives) {
const downloaded = await downloadFile(archive.path);
const remoteChecksum = await sha256(downloaded);
@@ -281,16 +472,28 @@ async function main() {
console.error(` Expected: ${archive.checksum}`);
console.error(` Got: ${remoteChecksum}`);
console.error(' ABORTING — archives uploaded but legacy files NOT deleted.');
console.error(' Investigate manually before retrying.');
process.exit(1);
}
console.log(` ${archive.path}: checksum verified`);
}
console.log(' All archives verified.');
// ── Step 7: Delete legacy files in batches ────────────────────────
// Verify invalid-records if present
if (invalidRecordsContent) {
const dlInvalid = await downloadFile(`${runDir}/invalid-records.json`);
const dlInvalidChecksum = await sha256(dlInvalid);
if (dlInvalidChecksum !== invalidRecordsChecksum) {
console.error(` INVALID-RECORDS CHECKSUM MISMATCH!`);
console.error(' ABORTING — will not delete sources.');
process.exit(1);
}
console.log(` ${runDir}/invalid-records.json: checksum verified`);
}
console.log(' All artifacts verified.');
// ── Step 8: Delete legacy files in batches ────────────────────────
console.log('');
console.log(`Step 7: Deleting ${sourceFileCount} legacy files in batches of ${DELETE_BATCH_SIZE}...`);
console.log(`Step 8: Deleting ${sourceFileCount} legacy files in batches of ${DELETE_BATCH_SIZE}...`);
const batches = [];
for (let i = 0; i < jsonlFiles.length; i += DELETE_BATCH_SIZE) {
batches.push(jsonlFiles.slice(i, i + DELETE_BATCH_SIZE));
@@ -306,9 +509,29 @@ async function main() {
console.log(` Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`);
}
// ── Step 9: Post-deletion verification ────────────────────────────
console.log('');
console.log('Step 9: Post-deletion verification...');
const postDeleteEntries = await listAllFiles('telemetry');
const postDeleteJsonl = postDeleteEntries.filter(f => f.type === 'file' && f.path.endsWith('.jsonl'));
const sourcePathSet = new Set(jsonlFiles.map(f => f.path));
const leftover = postDeleteJsonl.filter(f => sourcePathSet.has(f.path));
if (leftover.length > 0) {
console.error(` WARNING: ${leftover.length} source files still exist after deletion!`);
console.error(' First 10:', leftover.slice(0, 10).map(f => f.path).join(', '));
console.error(' Rerun with --apply to retry deletion of remaining files.');
process.exit(1);
}
console.log(` Confirmed: all ${sourceFileCount} manifest sources deleted.`);
if (postDeleteJsonl.length > 0) {
console.log(` Note: ${postDeleteJsonl.length} non-manifest JSONL files remain (new incoming data).`);
}
console.log('');
console.log('DONE. Legacy telemetry compacted successfully.');
console.log(` ${archives.length} monthly archives in archive/legacy/`);
console.log(` Run ID: ${currentRunId}`);
console.log(` ${archives.length} monthly archives in ${runDir}/`);
console.log(` ${sourceFileCount} legacy files removed`);
console.log(` ${totalEvents} events preserved`);
}