From 70cb3bb1884f7f5218ece9244763746b7892ee21 Mon Sep 17 00:00:00 2001 From: faber Date: Sat, 11 Jul 2026 10:23:11 -0400 Subject: [PATCH 1/5] fix: date-partition telemetry writes + legacy compaction script The HF dataset's telemetry/ folder hit the 10k file limit. Code fix: writes now target incoming_v2/YYYY/MM/DD/ instead of telemetry/, distributing files across date-partitioned subdirectories. Max ~200 files per day-folder means this can never saturate again. Compaction script: scripts/compact-hf-telemetry.mjs reads the existing 10k legacy files, groups events by UTC month, and writes ~12 archive files to archive/legacy/. Dry-run by default; --apply commits archives and deletes legacy files. Verifies event counts before any deletion. Co-Authored-By: Claude Opus 4.6 --- functions/api/telemetry.ts | 8 +- scripts/compact-hf-telemetry.mjs | 235 +++++++++++++++++++++++++++++++ 2 files changed, 241 insertions(+), 2 deletions(-) create mode 100644 scripts/compact-hf-telemetry.mjs diff --git a/functions/api/telemetry.ts b/functions/api/telemetry.ts index 5857f83..9da5dc7 100644 --- a/functions/api/telemetry.ts +++ b/functions/api/telemetry.ts @@ -377,7 +377,9 @@ async function flushKVToHF(env: Env): Promise { const combined = chunks.join('\n') const keyDigest = await sha256Hex(flushedKeys.map(k => k.name).sort().join(',')) - const filePath = `telemetry/flush_${keyDigest}.jsonl` + const d = new Date() + const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}` + const filePath = `incoming_v2/${datePath}/flush_${keyDigest}.jsonl` const branch = env.HF_DATASET_BRANCH || 'main' const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined) @@ -406,7 +408,9 @@ async function flushMemToHF(env: Env): Promise { const combined = snapshot.join('\n') const digest = await sha256Hex(combined) - const filePath = `telemetry/membuf_${digest}.jsonl` + const d = new Date() + const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}` + const filePath = `incoming_v2/${datePath}/membuf_${digest}.jsonl` const branch = env.HF_DATASET_BRANCH || 'main' const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined) diff --git a/scripts/compact-hf-telemetry.mjs b/scripts/compact-hf-telemetry.mjs new file mode 100644 index 0000000..a37ab71 --- /dev/null +++ b/scripts/compact-hf-telemetry.mjs @@ -0,0 +1,235 @@ +#!/usr/bin/env node +/** + * Compact legacy telemetry/ files into monthly archives under archive/legacy/. + * + * Usage: + * node scripts/compact-hf-telemetry.mjs # dry-run (default) + * node scripts/compact-hf-telemetry.mjs --apply # actually commit archives + delete legacy + * + * Requires: HF_TOKEN env var with write access to the dataset. + * + * Safety: + * - Dry-run is default; --apply required for writes + * - Verifies event counts before deleting anything + * - Produces a manifest with checksums + * - Stops on any count mismatch + */ + +const HF_API = 'https://huggingface.co/api'; +const DATASET_REPO = process.env.HF_DATASET_REPO || 'pliny-the-prompter/g0dm0d3'; +const BRANCH = process.env.HF_DATASET_BRANCH || 'main'; +const TOKEN = process.env.HF_TOKEN; +const APPLY = process.argv.includes('--apply'); + +if (!TOKEN) { + console.error('ERROR: HF_TOKEN environment variable required'); + process.exit(1); +} + +const headers = { Authorization: `Bearer ${TOKEN}` }; + +async function listFiles(prefix) { + const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}`; + const res = await fetch(url, { headers }); + if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`); + return res.json(); +} + +async function downloadFile(path) { + const url = `https://huggingface.co/datasets/${DATASET_REPO}/resolve/${BRANCH}/${path}`; + const res = await fetch(url, { headers }); + if (!res.ok) throw new Error(`Download failed (${res.status}): ${path}`); + return res.text(); +} + +async function sha256(text) { + const data = new TextEncoder().encode(text); + const buf = await crypto.subtle.digest('SHA-256', data); + return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join(''); +} + +async function commitFiles(operations, summary) { + const url = `${HF_API}/datasets/${DATASET_REPO}/commit/${BRANCH}`; + const lines = [ + JSON.stringify({ key: 'header', value: { summary } }), + ...operations.map(op => JSON.stringify({ key: op.key, value: op.value })), + ]; + const res = await fetch(url, { + method: 'POST', + headers: { ...headers, 'Content-Type': 'application/x-ndjson' }, + body: lines.join('\n'), + }); + if (!res.ok) throw new Error(`Commit failed (${res.status}): ${await res.text()}`); + return res.json(); +} + +function utf8ToBase64(str) { + const bytes = new TextEncoder().encode(str); + let binStr = ''; + for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]); + return btoa(binStr); +} + +async function main() { + console.log(`Mode: ${APPLY ? 'APPLY (will write)' : 'DRY-RUN (read-only)'}`); + console.log(`Repo: ${DATASET_REPO} (branch: ${BRANCH})`); + console.log(''); + + // Step 1: List all legacy telemetry files + console.log('Step 1: Listing legacy telemetry/ files...'); + const files = await listFiles('telemetry'); + const jsonlFiles = files.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); + console.log(` Found ${jsonlFiles.length} JSONL files`); + + if (jsonlFiles.length === 0) { + console.log(' Nothing to compact. Done.'); + return; + } + + // Step 2: Download and parse all files + console.log('Step 2: Downloading and parsing events...'); + const monthBuckets = {}; + let totalEvents = 0; + let totalBytes = 0; + let invalidLines = 0; + const invalidRecords = []; + + for (let i = 0; i < jsonlFiles.length; i++) { + const file = jsonlFiles[i]; + if (i % 500 === 0) console.log(` Progress: ${i}/${jsonlFiles.length} files...`); + + try { + const content = await downloadFile(file.path); + totalBytes += new TextEncoder().encode(content).byteLength; + const lines = content.split('\n').filter(l => l.trim()); + + for (const line of lines) { + try { + const event = JSON.parse(line); + const ts = event.timestamp; + let monthKey; + if (ts && typeof ts === 'number') { + const d = new Date(ts); + monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`; + } else { + monthKey = 'unknown'; + } + if (!monthBuckets[monthKey]) monthBuckets[monthKey] = []; + monthBuckets[monthKey].push(line); + totalEvents++; + } catch { + invalidLines++; + invalidRecords.push({ file: file.path, line: line.slice(0, 200) }); + } + } + } catch (err) { + console.error(` ERROR reading ${file.path}: ${err.message}`); + invalidRecords.push({ file: file.path, error: err.message }); + } + } + + console.log(` Total events: ${totalEvents}`); + console.log(` Total source bytes: ${(totalBytes / 1024 / 1024).toFixed(2)} MiB`); + console.log(` Invalid lines: ${invalidLines}`); + console.log(` Month buckets: ${Object.keys(monthBuckets).sort().join(', ')}`); + console.log(''); + + // Step 3: Build archives + console.log('Step 3: Building monthly archives...'); + const archives = []; + for (const [month, events] of Object.entries(monthBuckets).sort()) { + const content = events.join('\n') + '\n'; + const checksum = await sha256(content); + const archivePath = `archive/legacy/telemetry-${month}.jsonl`; + const bytes = new TextEncoder().encode(content).byteLength; + archives.push({ month, path: archivePath, events: events.length, bytes, checksum, content }); + console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 12)}...`); + } + + // Step 4: Verify counts + const archiveEventTotal = archives.reduce((s, a) => s + a.events, 0); + console.log(''); + console.log(`Step 4: Verification`); + console.log(` Source events: ${totalEvents}`); + console.log(` Archive events: ${archiveEventTotal}`); + + if (archiveEventTotal !== totalEvents) { + console.error(' MISMATCH — aborting. No changes made.'); + process.exit(1); + } + console.log(' MATCH — counts verified.'); + + // Build manifest + const manifest = { + timestamp: new Date().toISOString(), + repo: DATASET_REPO, + branch: BRANCH, + sourceFiles: jsonlFiles.length, + sourceBytes: totalBytes, + totalEvents, + invalidLines, + invalidRecords: invalidRecords.length, + archives: archives.map(a => ({ + path: a.path, month: a.month, events: a.events, + bytes: a.bytes, checksum: a.checksum, + })), + }; + + console.log(''); + console.log('Manifest:'); + console.log(JSON.stringify(manifest, null, 2)); + + if (!APPLY) { + console.log(''); + console.log('DRY-RUN complete. Re-run with --apply to commit changes.'); + return; + } + + // Step 5: Upload archives + console.log(''); + console.log('Step 5: Uploading archives...'); + const uploadOps = archives.map(a => ({ + key: 'file', + value: { content: utf8ToBase64(a.content), path: a.path, encoding: 'base64' }, + })); + uploadOps.push({ + key: 'file', + value: { + content: utf8ToBase64(JSON.stringify(manifest, null, 2)), + path: 'archive/legacy/manifest.json', + encoding: 'base64', + }, + }); + + // Save invalid records if any + if (invalidRecords.length > 0) { + uploadOps.push({ + key: 'file', + value: { + content: utf8ToBase64(JSON.stringify(invalidRecords, null, 2)), + path: 'archive/legacy/invalid-records.json', + encoding: 'base64', + }, + }); + } + + await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files`); + console.log(' Archives committed.'); + + // Step 6: Delete legacy files + console.log('Step 6: Deleting legacy files...'); + const deleteOps = jsonlFiles.map(f => ({ + key: 'deletedFile', + value: { path: f.path }, + })); + await commitFiles(deleteOps, `[maintenance] Remove ${jsonlFiles.length} legacy telemetry files (archived in archive/legacy/)`); + console.log(` Deleted ${jsonlFiles.length} files.`); + + console.log(''); + console.log('DONE. Legacy telemetry compacted successfully.'); +} + +main().catch(err => { + console.error('Fatal error:', err); + process.exit(1); +}); From 2896b530accee6be729bdfdcce36d23c66865aa0 Mon Sep 17 00:00:00 2001 From: faber Date: Sat, 11 Jul 2026 10:56:51 -0400 Subject: [PATCH 2/5] fix: deterministic date partitions + compaction script rewrite Telemetry writer: derive partition dates from batch timestamps (KV key timestamp / memBufferFirstTs) instead of `new Date()` so retries always produce the same path. Compaction script: full rewrite addressing all audit P0/P1 findings: - Paginated listing (follows rel="next" until all 10k+ files listed) - Fail-closed: aborts entirely if ANY download fails - Remote verification: re-downloads archives and checks SHA-256 before deleting sources - Batched deletes: max 50 operations per HF commit - Preserves FULL invalid lines in separate artifact (not truncated) - Dry-run is default; --apply required for writes Co-Authored-By: Claude Opus 4.6 --- functions/api/telemetry.ts | 8 +- scripts/compact-hf-telemetry.mjs | 246 +++++++++++++++++++++---------- 2 files changed, 171 insertions(+), 83 deletions(-) diff --git a/functions/api/telemetry.ts b/functions/api/telemetry.ts index 9da5dc7..c87aea2 100644 --- a/functions/api/telemetry.ts +++ b/functions/api/telemetry.ts @@ -377,7 +377,9 @@ async function flushKVToHF(env: Env): Promise { const combined = chunks.join('\n') const keyDigest = await sha256Hex(flushedKeys.map(k => k.name).sort().join(',')) - const d = new Date() + // Derive partition from oldest batch key timestamp (stable across retries) + const oldestTs = parseInt(flushedKeys[0].name.split(':')[1], 10) + const d = !isNaN(oldestTs) ? new Date(oldestTs) : new Date() const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}` const filePath = `incoming_v2/${datePath}/flush_${keyDigest}.jsonl` const branch = env.HF_DATASET_BRANCH || 'main' @@ -403,12 +405,14 @@ async function flushMemToHF(env: Env): Promise { // accumulate into a fresh buffer, not into the one we're flushing. const snapshot = memBuffer.splice(0, memBuffer.length) const snapshotBytes = memBufferBytes + const snapshotTs = memBufferFirstTs memBufferBytes = 0 memBufferFirstTs = 0 const combined = snapshot.join('\n') const digest = await sha256Hex(combined) - const d = new Date() + // Derive partition from buffer start time (stable across retries) + const d = snapshotTs > 0 ? new Date(snapshotTs) : new Date() const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}` const filePath = `incoming_v2/${datePath}/membuf_${digest}.jsonl` const branch = env.HF_DATASET_BRANCH || 'main' diff --git a/scripts/compact-hf-telemetry.mjs b/scripts/compact-hf-telemetry.mjs index a37ab71..60eebed 100644 --- a/scripts/compact-hf-telemetry.mjs +++ b/scripts/compact-hf-telemetry.mjs @@ -4,15 +4,18 @@ * * Usage: * node scripts/compact-hf-telemetry.mjs # dry-run (default) - * node scripts/compact-hf-telemetry.mjs --apply # actually commit archives + delete legacy + * node scripts/compact-hf-telemetry.mjs --apply # commit archives + delete legacy * * Requires: HF_TOKEN env var with write access to the dataset. * - * Safety: - * - Dry-run is default; --apply required for writes - * - Verifies event counts before deleting anything - * - Produces a manifest with checksums - * - Stops on any count mismatch + * Safety guarantees: + * - Dry-run is default; --apply required for any writes + * - FAIL-CLOSED: aborts on ANY download failure or parse error threshold + * - Paginates the full file listing (HF API returns max 1000/page) + * - Verifies uploaded archives by re-downloading and checking checksums + * - Batches delete operations (max 50 per HF commit) + * - Preserves FULL invalid lines (not truncated) in a separate artifact + * - Stops and reports if any count mismatch is detected */ const HF_API = 'https://huggingface.co/api'; @@ -20,6 +23,7 @@ const DATASET_REPO = process.env.HF_DATASET_REPO || 'pliny-the-prompter/g0dm0d3' const BRANCH = process.env.HF_DATASET_BRANCH || 'main'; const TOKEN = process.env.HF_TOKEN; const APPLY = process.argv.includes('--apply'); +const DELETE_BATCH_SIZE = 50; if (!TOKEN) { console.error('ERROR: HF_TOKEN environment variable required'); @@ -28,11 +32,35 @@ if (!TOKEN) { const headers = { Authorization: `Bearer ${TOKEN}` }; -async function listFiles(prefix) { - const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}`; - const res = await fetch(url, { headers }); - if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`); - return res.json(); +// ── Paginated file listing ────────────────────────────────────────── +async function listAllFiles(prefix) { + const allFiles = []; + let cursor = null; + + while (true) { + const params = new URLSearchParams(); + if (cursor) params.set('cursor', cursor); + const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}?${params}`; + const res = await fetch(url, { headers }); + if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`); + + const page = await res.json(); + if (!Array.isArray(page) || page.length === 0) break; + + allFiles.push(...page); + console.log(` Listed ${allFiles.length} entries so far...`); + + // HF returns Link header with rel="next" for pagination + const linkHeader = res.headers.get('link') || ''; + const nextMatch = linkHeader.match(/<[^>]*[?&]cursor=([^&>]+)[^>]*>;\s*rel="next"/); + if (nextMatch) { + cursor = decodeURIComponent(nextMatch[1]); + } else { + break; + } + } + + return allFiles; } async function downloadFile(path) { @@ -48,6 +76,13 @@ async function sha256(text) { return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join(''); } +function utf8ToBase64(str) { + const bytes = new TextEncoder().encode(str); + let binStr = ''; + for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]); + return btoa(binStr); +} + async function commitFiles(operations, summary) { const url = `${HF_API}/datasets/${DATASET_REPO}/commit/${BRANCH}`; const lines = [ @@ -59,82 +94,94 @@ async function commitFiles(operations, summary) { headers: { ...headers, 'Content-Type': 'application/x-ndjson' }, body: lines.join('\n'), }); - if (!res.ok) throw new Error(`Commit failed (${res.status}): ${await res.text()}`); + if (!res.ok) throw new Error(`Commit failed (${res.status}): ${(await res.text()).slice(0, 500)}`); return res.json(); } -function utf8ToBase64(str) { - const bytes = new TextEncoder().encode(str); - let binStr = ''; - for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]); - return btoa(binStr); -} - async function main() { console.log(`Mode: ${APPLY ? 'APPLY (will write)' : 'DRY-RUN (read-only)'}`); console.log(`Repo: ${DATASET_REPO} (branch: ${BRANCH})`); console.log(''); - // Step 1: List all legacy telemetry files - console.log('Step 1: Listing legacy telemetry/ files...'); - const files = await listFiles('telemetry'); - const jsonlFiles = files.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); - console.log(` Found ${jsonlFiles.length} JSONL files`); + // ── Step 1: Paginated listing ───────────────────────────────────── + console.log('Step 1: Listing ALL legacy telemetry/ files (paginated)...'); + const allEntries = await listAllFiles('telemetry'); + const jsonlFiles = allEntries.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); + console.log(` Total entries: ${allEntries.length}`); + console.log(` JSONL files: ${jsonlFiles.length}`); if (jsonlFiles.length === 0) { console.log(' Nothing to compact. Done.'); return; } - // Step 2: Download and parse all files - console.log('Step 2: Downloading and parsing events...'); + // ── Step 2: Download and parse ALL files (fail-closed) ──────────── + console.log(''); + console.log('Step 2: Downloading and parsing events (fail-closed)...'); const monthBuckets = {}; let totalEvents = 0; - let totalBytes = 0; + let totalSourceBytes = 0; + let downloadFailures = 0; let invalidLines = 0; const invalidRecords = []; + const failedFiles = []; for (let i = 0; i < jsonlFiles.length; i++) { const file = jsonlFiles[i]; - if (i % 500 === 0) console.log(` Progress: ${i}/${jsonlFiles.length} files...`); + if (i % 200 === 0 || i === jsonlFiles.length - 1) { + console.log(` Progress: ${i + 1}/${jsonlFiles.length} files...`); + } + let content; try { - const content = await downloadFile(file.path); - totalBytes += new TextEncoder().encode(content).byteLength; - const lines = content.split('\n').filter(l => l.trim()); - - for (const line of lines) { - try { - const event = JSON.parse(line); - const ts = event.timestamp; - let monthKey; - if (ts && typeof ts === 'number') { - const d = new Date(ts); - monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`; - } else { - monthKey = 'unknown'; - } - if (!monthBuckets[monthKey]) monthBuckets[monthKey] = []; - monthBuckets[monthKey].push(line); - totalEvents++; - } catch { - invalidLines++; - invalidRecords.push({ file: file.path, line: line.slice(0, 200) }); - } - } + content = await downloadFile(file.path); } catch (err) { - console.error(` ERROR reading ${file.path}: ${err.message}`); - invalidRecords.push({ file: file.path, error: err.message }); + downloadFailures++; + failedFiles.push({ path: file.path, error: err.message }); + continue; + } + + totalSourceBytes += new TextEncoder().encode(content).byteLength; + const lines = content.split('\n').filter(l => l.trim()); + + for (const line of lines) { + try { + const event = JSON.parse(line); + const ts = event.timestamp; + let monthKey; + if (ts && typeof ts === 'number') { + const d = new Date(ts); + monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`; + } else { + monthKey = 'unknown'; + } + if (!monthBuckets[monthKey]) monthBuckets[monthKey] = []; + monthBuckets[monthKey].push(line); + totalEvents++; + } catch { + invalidLines++; + invalidRecords.push({ file: file.path, line }); + } } } - console.log(` Total events: ${totalEvents}`); - console.log(` Total source bytes: ${(totalBytes / 1024 / 1024).toFixed(2)} MiB`); + console.log(` Total events parsed: ${totalEvents}`); + console.log(` Total source bytes: ${(totalSourceBytes / 1024 / 1024).toFixed(2)} MiB`); + console.log(` Download failures: ${downloadFailures}`); console.log(` Invalid lines: ${invalidLines}`); console.log(` Month buckets: ${Object.keys(monthBuckets).sort().join(', ')}`); - console.log(''); - // Step 3: Build archives + // ── FAIL-CLOSED: abort if any downloads failed ──────────────────── + if (downloadFailures > 0) { + console.error(''); + console.error(`ABORT: ${downloadFailures} file(s) failed to download.`); + console.error('Cannot guarantee data preservation. Fix network/auth issues and retry.'); + console.error('Failed files:', failedFiles.map(f => f.path).join(', ')); + process.exit(1); + } + + // ── Step 3: Build archives ──────────────────────────────────────── + console.log(''); console.log('Step 3: Building monthly archives...'); const archives = []; for (const [month, events] of Object.entries(monthBuckets).sort()) { @@ -143,32 +190,39 @@ async function main() { const archivePath = `archive/legacy/telemetry-${month}.jsonl`; const bytes = new TextEncoder().encode(content).byteLength; archives.push({ month, path: archivePath, events: events.length, bytes, checksum, content }); - console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 12)}...`); + console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 16)}...`); } - // Step 4: Verify counts + // ── Step 4: Verify counts (tautology-free) ──────────────────────── const archiveEventTotal = archives.reduce((s, a) => s + a.events, 0); - console.log(''); - console.log(`Step 4: Verification`); - console.log(` Source events: ${totalEvents}`); - console.log(` Archive events: ${archiveEventTotal}`); + const sourceFileCount = jsonlFiles.length; + const expectedTotal = totalEvents; // all successfully parsed events - if (archiveEventTotal !== totalEvents) { - console.error(' MISMATCH — aborting. No changes made.'); + console.log(''); + console.log('Step 4: Verification'); + console.log(` Source files examined: ${sourceFileCount}`); + console.log(` Download failures: ${downloadFailures} (must be 0)`); + console.log(` Events parsed from sources: ${totalEvents}`); + console.log(` Events in archives: ${archiveEventTotal}`); + console.log(` Invalid lines preserved: ${invalidLines}`); + + if (archiveEventTotal !== expectedTotal) { + console.error(' COUNT MISMATCH — aborting. No changes made.'); process.exit(1); } - console.log(' MATCH — counts verified.'); + console.log(' COUNTS MATCH.'); - // Build manifest + // ── Build manifest ──────────────────────────────────────────────── const manifest = { timestamp: new Date().toISOString(), repo: DATASET_REPO, branch: BRANCH, - sourceFiles: jsonlFiles.length, - sourceBytes: totalBytes, + sourceFiles: sourceFileCount, + sourceBytes: totalSourceBytes, totalEvents, invalidLines, - invalidRecords: invalidRecords.length, + invalidRecordCount: invalidRecords.length, + downloadFailures, archives: archives.map(a => ({ path: a.path, month: a.month, events: a.events, bytes: a.bytes, checksum: a.checksum, @@ -182,10 +236,12 @@ async function main() { if (!APPLY) { console.log(''); console.log('DRY-RUN complete. Re-run with --apply to commit changes.'); + console.log(`Would upload ${archives.length} archive files + manifest.`); + console.log(`Would delete ${sourceFileCount} legacy files in ${Math.ceil(sourceFileCount / DELETE_BATCH_SIZE)} batches.`); return; } - // Step 5: Upload archives + // ── Step 5: Upload archives ─────────────────────────────────────── console.log(''); console.log('Step 5: Uploading archives...'); const uploadOps = archives.map(a => ({ @@ -200,8 +256,6 @@ async function main() { encoding: 'base64', }, }); - - // Save invalid records if any if (invalidRecords.length > 0) { uploadOps.push({ key: 'file', @@ -216,17 +270,47 @@ async function main() { await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files`); console.log(' Archives committed.'); - // Step 6: Delete legacy files - console.log('Step 6: Deleting legacy files...'); - const deleteOps = jsonlFiles.map(f => ({ - key: 'deletedFile', - value: { path: f.path }, - })); - await commitFiles(deleteOps, `[maintenance] Remove ${jsonlFiles.length} legacy telemetry files (archived in archive/legacy/)`); - console.log(` Deleted ${jsonlFiles.length} files.`); + // ── Step 6: Verify uploaded archives (re-download + checksum) ───── + console.log(''); + console.log('Step 6: Verifying uploaded archives...'); + for (const archive of archives) { + const downloaded = await downloadFile(archive.path); + const remoteChecksum = await sha256(downloaded); + if (remoteChecksum !== archive.checksum) { + console.error(` CHECKSUM MISMATCH for ${archive.path}!`); + console.error(` Expected: ${archive.checksum}`); + console.error(` Got: ${remoteChecksum}`); + console.error(' ABORTING — archives uploaded but legacy files NOT deleted.'); + console.error(' Investigate manually before retrying.'); + process.exit(1); + } + console.log(` ${archive.path}: checksum verified`); + } + console.log(' All archives verified.'); + + // ── Step 7: Delete legacy files in batches ──────────────────────── + console.log(''); + console.log(`Step 7: Deleting ${sourceFileCount} legacy files in batches of ${DELETE_BATCH_SIZE}...`); + const batches = []; + for (let i = 0; i < jsonlFiles.length; i += DELETE_BATCH_SIZE) { + batches.push(jsonlFiles.slice(i, i + DELETE_BATCH_SIZE)); + } + + for (let i = 0; i < batches.length; i++) { + const batch = batches[i]; + const deleteOps = batch.map(f => ({ + key: 'deletedFile', + value: { path: f.path }, + })); + await commitFiles(deleteOps, `[maintenance] Remove legacy telemetry batch ${i + 1}/${batches.length} (${batch.length} files)`); + console.log(` Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`); + } console.log(''); console.log('DONE. Legacy telemetry compacted successfully.'); + console.log(` ${archives.length} monthly archives in archive/legacy/`); + console.log(` ${sourceFileCount} legacy files removed`); + console.log(` ${totalEvents} events preserved`); } main().catch(err => { From 9f4f898b8e15e043a3af63d694ca578f901524d8 Mon Sep 17 00:00:00 2001 From: faber Date: Sat, 11 Jul 2026 13:03:34 -0400 Subject: [PATCH 3/5] fix: resumable compaction + restore memBufferFirstTs on flush failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compaction script (P0 restart-safe migration): - Derives stable run ID from hash of sorted source file paths - Archives stored under versioned archive/legacy// path - Manifest includes full sourceFilePaths list for resume detection - On rerun: detects existing manifest, verifies archives, resumes deletion from the original source list — never rebuilds from a partial file set - Verifies ALL artifacts (manifest + archives + invalid-records) - Post-deletion relist confirms every manifest source is gone - Idempotent upload: skips if manifest checksum already matches Telemetry writer (memBufferFirstTs bug): - On flush failure, restore snapshotTs instead of Date.now() so cross-midnight retries don't change partition paths Co-Authored-By: Claude Opus 4.6 --- functions/api/telemetry.ts | 2 +- scripts/compact-hf-telemetry.mjs | 315 ++++++++++++++++++++++++++----- 2 files changed, 270 insertions(+), 47 deletions(-) diff --git a/functions/api/telemetry.ts b/functions/api/telemetry.ts index c87aea2..4e09eb2 100644 --- a/functions/api/telemetry.ts +++ b/functions/api/telemetry.ts @@ -428,7 +428,7 @@ async function flushMemToHF(env: Env): Promise { // Prepend so ordering is roughly preserved. memBuffer.unshift(...snapshot) memBufferBytes += snapshotBytes - if (memBufferFirstTs === 0) memBufferFirstTs = Date.now() + if (memBufferFirstTs === 0) memBufferFirstTs = snapshotTs console.error(`[Telemetry] Memory flush failed — ${snapshot.length} batches returned to buffer`) return 0 } diff --git a/scripts/compact-hf-telemetry.mjs b/scripts/compact-hf-telemetry.mjs index 60eebed..d23e4ec 100644 --- a/scripts/compact-hf-telemetry.mjs +++ b/scripts/compact-hf-telemetry.mjs @@ -10,12 +10,18 @@ * * Safety guarantees: * - Dry-run is default; --apply required for any writes - * - FAIL-CLOSED: aborts on ANY download failure or parse error threshold + * - FAIL-CLOSED: aborts on ANY download failure * - Paginates the full file listing (HF API returns max 1000/page) - * - Verifies uploaded archives by re-downloading and checking checksums + * - RESUMABLE: derives a stable run ID from source file list hash; + * stores archives under archive/legacy//. On rerun after partial + * deletion, detects existing manifest and resumes deletion from the + * manifest's original source list — never rebuilds archives from a + * reduced file set. + * - Verifies ALL uploaded artifacts (archives + manifest + invalid-records) + * by re-downloading and checking checksums * - Batches delete operations (max 50 per HF commit) - * - Preserves FULL invalid lines (not truncated) in a separate artifact - * - Stops and reports if any count mismatch is detected + * - Preserves FULL invalid lines in a separate artifact + * - Post-deletion relist confirms all manifest sources are gone */ const HF_API = 'https://huggingface.co/api'; @@ -50,7 +56,6 @@ async function listAllFiles(prefix) { allFiles.push(...page); console.log(` Listed ${allFiles.length} entries so far...`); - // HF returns Link header with rel="next" for pagination const linkHeader = res.headers.get('link') || ''; const nextMatch = linkHeader.match(/<[^>]*[?&]cursor=([^&>]+)[^>]*>;\s*rel="next"/); if (nextMatch) { @@ -70,6 +75,12 @@ async function downloadFile(path) { return res.text(); } +async function fileExists(path) { + const url = `https://huggingface.co/datasets/${DATASET_REPO}/resolve/${BRANCH}/${path}`; + const res = await fetch(url, { method: 'HEAD', headers }); + return res.ok; +} + async function sha256(text) { const data = new TextEncoder().encode(text); const buf = await crypto.subtle.digest('SHA-256', data); @@ -98,6 +109,13 @@ async function commitFiles(operations, summary) { return res.json(); } +// ── Derive stable run ID from the sorted source file list ─────────── +async function deriveRunId(sourceFiles) { + const pathList = sourceFiles.map(f => f.path).sort().join('\n'); + const hash = await sha256(pathList); + return hash.slice(0, 12); +} + async function main() { console.log(`Mode: ${APPLY ? 'APPLY (will write)' : 'DRY-RUN (read-only)'}`); console.log(`Repo: ${DATASET_REPO} (branch: ${BRANCH})`); @@ -115,9 +133,142 @@ async function main() { return; } - // ── Step 2: Download and parse ALL files (fail-closed) ──────────── + // ── Step 2: Check for existing manifest (resume detection) ──────── + // Derive run ID from the FULL original source list. If a prior run + // already archived everything, its manifest contains the original + // source paths — we can resume deletion without re-downloading. + const currentRunId = await deriveRunId(jsonlFiles); + const runDir = `archive/legacy/${currentRunId}`; + const manifestPath = `${runDir}/manifest.json`; + console.log(''); - console.log('Step 2: Downloading and parsing events (fail-closed)...'); + console.log(`Step 2: Checking for existing manifest (run ID: ${currentRunId})...`); + + // Also check if a DIFFERENT run's manifest exists (partial deletion scenario) + const archiveLegacyEntries = await listAllFiles('archive/legacy'); + const existingManifests = archiveLegacyEntries.filter( + f => f.type === 'file' && f.path.endsWith('/manifest.json') + ); + + let resumeManifest = null; + + if (existingManifests.length > 0) { + // Check each manifest to see if it contains source files that are now missing + // (meaning deletion already started from that run) + for (const mf of existingManifests) { + try { + const mfContent = await downloadFile(mf.path); + const mfData = JSON.parse(mfContent); + if (mfData.sourceFilePaths && Array.isArray(mfData.sourceFilePaths)) { + const currentPaths = new Set(jsonlFiles.map(f => f.path)); + const manifestPaths = new Set(mfData.sourceFilePaths); + + // If manifest has paths that no longer exist, deletion already started + const alreadyDeleted = mfData.sourceFilePaths.filter(p => !currentPaths.has(p)); + if (alreadyDeleted.length > 0) { + console.log(` Found prior manifest at ${mf.path}`); + console.log(` Original sources: ${mfData.sourceFilePaths.length}`); + console.log(` Already deleted: ${alreadyDeleted.length}`); + console.log(` Remaining: ${mfData.sourceFilePaths.length - alreadyDeleted.length}`); + console.log(' RESUMING from prior manifest (archives already verified).'); + resumeManifest = mfData; + break; + } + + // If current files exactly match or are a superset, this is a fresh run + // for the same data — use the existing archives if checksums match + if (mf.path === manifestPath) { + console.log(` Found matching manifest at ${manifestPath}`); + console.log(' Verifying existing archives...'); + let allValid = true; + for (const arch of mfData.archives) { + try { + const content = await downloadFile(arch.path); + const checksum = await sha256(content); + if (checksum !== arch.checksum) { + console.error(` CHECKSUM MISMATCH: ${arch.path}`); + allValid = false; + break; + } + } catch { + console.error(` MISSING: ${arch.path}`); + allValid = false; + break; + } + } + if (allValid) { + console.log(' All archives verified. Resuming deletion phase.'); + resumeManifest = mfData; + break; + } else { + console.log(' Archives invalid — rebuilding from scratch.'); + } + } + } + } catch (err) { + console.warn(` Could not parse manifest ${mf.path}: ${err.message}`); + } + } + } + + // ── RESUME PATH: skip download/archive, go straight to deletion ─── + if (resumeManifest) { + if (!APPLY) { + const remaining = resumeManifest.sourceFilePaths.filter( + p => jsonlFiles.some(f => f.path === p) + ); + console.log(''); + console.log('DRY-RUN: Would resume deletion of remaining files.'); + console.log(` Files still to delete: ${remaining.length}`); + console.log(` Batches: ${Math.ceil(remaining.length / DELETE_BATCH_SIZE)}`); + return; + } + + // Delete only files that still exist AND are in the manifest + const manifestPathSet = new Set(resumeManifest.sourceFilePaths); + const toDelete = jsonlFiles.filter(f => manifestPathSet.has(f.path)); + + console.log(''); + console.log(`Resuming deletion: ${toDelete.length} files remaining...`); + + const batches = []; + for (let i = 0; i < toDelete.length; i += DELETE_BATCH_SIZE) { + batches.push(toDelete.slice(i, i + DELETE_BATCH_SIZE)); + } + + for (let i = 0; i < batches.length; i++) { + const batch = batches[i]; + const deleteOps = batch.map(f => ({ + key: 'deletedFile', + value: { path: f.path }, + })); + await commitFiles(deleteOps, `[maintenance] Remove legacy telemetry batch ${i + 1}/${batches.length} (${batch.length} files)`); + console.log(` Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`); + } + + // Post-deletion verification + console.log(''); + console.log('Post-deletion verification: relisting telemetry/...'); + const remaining = await listAllFiles('telemetry'); + const remainingJsonl = remaining.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); + const leftover = remainingJsonl.filter(f => manifestPathSet.has(f.path)); + if (leftover.length > 0) { + console.error(` WARNING: ${leftover.length} manifest sources still exist after deletion!`); + console.error(' Paths:', leftover.map(f => f.path).slice(0, 10).join(', ')); + process.exit(1); + } + console.log(` Confirmed: 0 manifest sources remain. ${remainingJsonl.length} non-manifest files present.`); + + console.log(''); + console.log('DONE (resumed). Legacy telemetry compacted successfully.'); + return; + } + + // ── FRESH PATH: download, archive, verify, delete ───────────────── + + // ── Step 3: Download and parse ALL files (fail-closed) ──────────── + console.log(''); + console.log('Step 3: Downloading and parsing events (fail-closed)...'); const monthBuckets = {}; let totalEvents = 0; let totalSourceBytes = 0; @@ -180,99 +331,139 @@ async function main() { process.exit(1); } - // ── Step 3: Build archives ──────────────────────────────────────── + // ── Step 4: Build archives ──────────────────────────────────────── console.log(''); - console.log('Step 3: Building monthly archives...'); + console.log(`Step 4: Building monthly archives under ${runDir}/...`); const archives = []; for (const [month, events] of Object.entries(monthBuckets).sort()) { const content = events.join('\n') + '\n'; const checksum = await sha256(content); - const archivePath = `archive/legacy/telemetry-${month}.jsonl`; + const archivePath = `${runDir}/telemetry-${month}.jsonl`; const bytes = new TextEncoder().encode(content).byteLength; archives.push({ month, path: archivePath, events: events.length, bytes, checksum, content }); console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 16)}...`); } - // ── Step 4: Verify counts (tautology-free) ──────────────────────── + // ── Step 5: Verify counts ───────────────────────────────────────── const archiveEventTotal = archives.reduce((s, a) => s + a.events, 0); const sourceFileCount = jsonlFiles.length; - const expectedTotal = totalEvents; // all successfully parsed events console.log(''); - console.log('Step 4: Verification'); + console.log('Step 5: Count verification'); console.log(` Source files examined: ${sourceFileCount}`); - console.log(` Download failures: ${downloadFailures} (must be 0)`); console.log(` Events parsed from sources: ${totalEvents}`); console.log(` Events in archives: ${archiveEventTotal}`); console.log(` Invalid lines preserved: ${invalidLines}`); - if (archiveEventTotal !== expectedTotal) { + if (archiveEventTotal !== totalEvents) { console.error(' COUNT MISMATCH — aborting. No changes made.'); process.exit(1); } console.log(' COUNTS MATCH.'); - // ── Build manifest ──────────────────────────────────────────────── + // ── Build manifest (includes full source path list for resumability) ─ + const invalidRecordsContent = invalidRecords.length > 0 + ? JSON.stringify(invalidRecords, null, 2) + : null; + const invalidRecordsChecksum = invalidRecordsContent + ? await sha256(invalidRecordsContent) + : null; + const manifest = { + runId: currentRunId, timestamp: new Date().toISOString(), repo: DATASET_REPO, branch: BRANCH, sourceFiles: sourceFileCount, sourceBytes: totalSourceBytes, + sourceFilePaths: jsonlFiles.map(f => f.path).sort(), totalEvents, invalidLines, invalidRecordCount: invalidRecords.length, - downloadFailures, + invalidRecordsChecksum, archives: archives.map(a => ({ path: a.path, month: a.month, events: a.events, bytes: a.bytes, checksum: a.checksum, })), }; + const manifestContent = JSON.stringify(manifest, null, 2); + const manifestChecksum = await sha256(manifestContent); + console.log(''); console.log('Manifest:'); - console.log(JSON.stringify(manifest, null, 2)); + console.log(manifestContent); if (!APPLY) { console.log(''); console.log('DRY-RUN complete. Re-run with --apply to commit changes.'); - console.log(`Would upload ${archives.length} archive files + manifest.`); + console.log(`Would upload ${archives.length} archive files + manifest + invalid-records.`); console.log(`Would delete ${sourceFileCount} legacy files in ${Math.ceil(sourceFileCount / DELETE_BATCH_SIZE)} batches.`); return; } - // ── Step 5: Upload archives ─────────────────────────────────────── + // ── Step 6: Upload archives ─────────────────────────────────────── console.log(''); - console.log('Step 5: Uploading archives...'); - const uploadOps = archives.map(a => ({ - key: 'file', - value: { content: utf8ToBase64(a.content), path: a.path, encoding: 'base64' }, - })); - uploadOps.push({ - key: 'file', - value: { - content: utf8ToBase64(JSON.stringify(manifest, null, 2)), - path: 'archive/legacy/manifest.json', - encoding: 'base64', - }, - }); - if (invalidRecords.length > 0) { + console.log('Step 6: Uploading archives...'); + + // Check if archives already exist and match (idempotent upload) + let archivesAlreadyExist = false; + if (await fileExists(manifestPath)) { + try { + const existing = await downloadFile(manifestPath); + const existingChecksum = await sha256(existing); + if (existingChecksum === manifestChecksum) { + console.log(' Archives already uploaded (manifest checksum matches). Skipping upload.'); + archivesAlreadyExist = true; + } + } catch { /* not found or parse error — upload fresh */ } + } + + if (!archivesAlreadyExist) { + const uploadOps = archives.map(a => ({ + key: 'file', + value: { content: utf8ToBase64(a.content), path: a.path, encoding: 'base64' }, + })); uploadOps.push({ key: 'file', value: { - content: utf8ToBase64(JSON.stringify(invalidRecords, null, 2)), - path: 'archive/legacy/invalid-records.json', + content: utf8ToBase64(manifestContent), + path: manifestPath, encoding: 'base64', }, }); + if (invalidRecordsContent) { + uploadOps.push({ + key: 'file', + value: { + content: utf8ToBase64(invalidRecordsContent), + path: `${runDir}/invalid-records.json`, + encoding: 'base64', + }, + }); + } + + await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files (run ${currentRunId})`); + console.log(' Archives committed.'); } - await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files`); - console.log(' Archives committed.'); - - // ── Step 6: Verify uploaded archives (re-download + checksum) ───── + // ── Step 7: Verify ALL uploaded artifacts ───────────────────────── console.log(''); - console.log('Step 6: Verifying uploaded archives...'); + console.log('Step 7: Verifying uploaded artifacts...'); + + // Verify manifest + const dlManifest = await downloadFile(manifestPath); + const dlManifestChecksum = await sha256(dlManifest); + if (dlManifestChecksum !== manifestChecksum) { + console.error(` MANIFEST CHECKSUM MISMATCH!`); + console.error(` Expected: ${manifestChecksum}`); + console.error(` Got: ${dlManifestChecksum}`); + console.error(' ABORTING — will not delete sources.'); + process.exit(1); + } + console.log(` ${manifestPath}: checksum verified`); + + // Verify each archive for (const archive of archives) { const downloaded = await downloadFile(archive.path); const remoteChecksum = await sha256(downloaded); @@ -281,16 +472,28 @@ async function main() { console.error(` Expected: ${archive.checksum}`); console.error(` Got: ${remoteChecksum}`); console.error(' ABORTING — archives uploaded but legacy files NOT deleted.'); - console.error(' Investigate manually before retrying.'); process.exit(1); } console.log(` ${archive.path}: checksum verified`); } - console.log(' All archives verified.'); - // ── Step 7: Delete legacy files in batches ──────────────────────── + // Verify invalid-records if present + if (invalidRecordsContent) { + const dlInvalid = await downloadFile(`${runDir}/invalid-records.json`); + const dlInvalidChecksum = await sha256(dlInvalid); + if (dlInvalidChecksum !== invalidRecordsChecksum) { + console.error(` INVALID-RECORDS CHECKSUM MISMATCH!`); + console.error(' ABORTING — will not delete sources.'); + process.exit(1); + } + console.log(` ${runDir}/invalid-records.json: checksum verified`); + } + + console.log(' All artifacts verified.'); + + // ── Step 8: Delete legacy files in batches ──────────────────────── console.log(''); - console.log(`Step 7: Deleting ${sourceFileCount} legacy files in batches of ${DELETE_BATCH_SIZE}...`); + console.log(`Step 8: Deleting ${sourceFileCount} legacy files in batches of ${DELETE_BATCH_SIZE}...`); const batches = []; for (let i = 0; i < jsonlFiles.length; i += DELETE_BATCH_SIZE) { batches.push(jsonlFiles.slice(i, i + DELETE_BATCH_SIZE)); @@ -306,9 +509,29 @@ async function main() { console.log(` Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`); } + // ── Step 9: Post-deletion verification ──────────────────────────── + console.log(''); + console.log('Step 9: Post-deletion verification...'); + const postDeleteEntries = await listAllFiles('telemetry'); + const postDeleteJsonl = postDeleteEntries.filter(f => f.type === 'file' && f.path.endsWith('.jsonl')); + const sourcePathSet = new Set(jsonlFiles.map(f => f.path)); + const leftover = postDeleteJsonl.filter(f => sourcePathSet.has(f.path)); + + if (leftover.length > 0) { + console.error(` WARNING: ${leftover.length} source files still exist after deletion!`); + console.error(' First 10:', leftover.slice(0, 10).map(f => f.path).join(', ')); + console.error(' Rerun with --apply to retry deletion of remaining files.'); + process.exit(1); + } + console.log(` Confirmed: all ${sourceFileCount} manifest sources deleted.`); + if (postDeleteJsonl.length > 0) { + console.log(` Note: ${postDeleteJsonl.length} non-manifest JSONL files remain (new incoming data).`); + } + console.log(''); console.log('DONE. Legacy telemetry compacted successfully.'); - console.log(` ${archives.length} monthly archives in archive/legacy/`); + console.log(` Run ID: ${currentRunId}`); + console.log(` ${archives.length} monthly archives in ${runDir}/`); console.log(` ${sourceFileCount} legacy files removed`); console.log(` ${totalEvents} events preserved`); } From 9f714a09fce3938f2bb687925f320e3971863a2e Mon Sep 17 00:00:00 2001 From: faber Date: Sat, 11 Jul 2026 14:00:25 -0400 Subject: [PATCH 4/5] fix: use recursive listing when scanning archive/legacy for manifests HF tree API defaults to non-recursive, so nested manifest.json files under archive/legacy// were invisible. After a partial deletion, the script would miss the original manifest and incorrectly rebuild archives from the reduced file set. Now listAllFiles accepts a recursive flag, and the archive scan uses it. Co-Authored-By: Claude Opus 4.6 --- scripts/compact-hf-telemetry.mjs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/compact-hf-telemetry.mjs b/scripts/compact-hf-telemetry.mjs index d23e4ec..586bec2 100644 --- a/scripts/compact-hf-telemetry.mjs +++ b/scripts/compact-hf-telemetry.mjs @@ -39,13 +39,14 @@ if (!TOKEN) { const headers = { Authorization: `Bearer ${TOKEN}` }; // ── Paginated file listing ────────────────────────────────────────── -async function listAllFiles(prefix) { +async function listAllFiles(prefix, recursive = false) { const allFiles = []; let cursor = null; while (true) { const params = new URLSearchParams(); if (cursor) params.set('cursor', cursor); + if (recursive) params.set('recursive', 'true'); const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}?${params}`; const res = await fetch(url, { headers }); if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`); @@ -145,7 +146,7 @@ async function main() { console.log(`Step 2: Checking for existing manifest (run ID: ${currentRunId})...`); // Also check if a DIFFERENT run's manifest exists (partial deletion scenario) - const archiveLegacyEntries = await listAllFiles('archive/legacy'); + const archiveLegacyEntries = await listAllFiles('archive/legacy', true); const existingManifests = archiveLegacyEntries.filter( f => f.type === 'file' && f.path.endsWith('/manifest.json') ); From 2e1cf583434f622482a1da2ec3c591bb74c81017 Mon Sep 17 00:00:00 2001 From: faber Date: Tue, 14 Jul 2026 09:59:17 -0400 Subject: [PATCH 5/5] fix: handle missing archive/legacy dir on first compaction run listAllFiles('archive/legacy') throws 404 EntryNotFound when the directory doesn't exist yet on HF. Catch it so the first run proceeds instead of dying at Step 2. Co-Authored-By: Claude Opus 4.6 --- scripts/compact-hf-telemetry.mjs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/compact-hf-telemetry.mjs b/scripts/compact-hf-telemetry.mjs index 586bec2..19de5a6 100644 --- a/scripts/compact-hf-telemetry.mjs +++ b/scripts/compact-hf-telemetry.mjs @@ -146,7 +146,12 @@ async function main() { console.log(`Step 2: Checking for existing manifest (run ID: ${currentRunId})...`); // Also check if a DIFFERENT run's manifest exists (partial deletion scenario) - const archiveLegacyEntries = await listAllFiles('archive/legacy', true); + let archiveLegacyEntries = []; + try { + archiveLegacyEntries = await listAllFiles('archive/legacy', true); + } catch (err) { + if (!/404|EntryNotFound|does not exist/i.test(String(err.message))) throw err; + } const existingManifests = archiveLegacyEntries.filter( f => f.type === 'file' && f.path.endsWith('/manifest.json') );