mirror of
https://github.com/elder-plinius/G0DM0D3.git
synced 2026-11-16 07:02:25 +00:00
fix: date-partition telemetry writes + legacy compaction script
The HF dataset's telemetry/ folder hit the 10k file limit. Code fix: writes now target incoming_v2/YYYY/MM/DD/ instead of telemetry/, distributing files across date-partitioned subdirectories. Max ~200 files per day-folder means this can never saturate again. Compaction script: scripts/compact-hf-telemetry.mjs reads the existing 10k legacy files, groups events by UTC month, and writes ~12 archive files to archive/legacy/. Dry-run by default; --apply commits archives and deletes legacy files. Verifies event counts before any deletion. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -377,7 +377,9 @@ async function flushKVToHF(env: Env): Promise<number> {
|
||||
|
||||
const combined = chunks.join('\n')
|
||||
const keyDigest = await sha256Hex(flushedKeys.map(k => k.name).sort().join(','))
|
||||
const filePath = `telemetry/flush_${keyDigest}.jsonl`
|
||||
const d = new Date()
|
||||
const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}`
|
||||
const filePath = `incoming_v2/${datePath}/flush_${keyDigest}.jsonl`
|
||||
const branch = env.HF_DATASET_BRANCH || 'main'
|
||||
|
||||
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
|
||||
@@ -406,7 +408,9 @@ async function flushMemToHF(env: Env): Promise<number> {
|
||||
|
||||
const combined = snapshot.join('\n')
|
||||
const digest = await sha256Hex(combined)
|
||||
const filePath = `telemetry/membuf_${digest}.jsonl`
|
||||
const d = new Date()
|
||||
const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}`
|
||||
const filePath = `incoming_v2/${datePath}/membuf_${digest}.jsonl`
|
||||
const branch = env.HF_DATASET_BRANCH || 'main'
|
||||
|
||||
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
|
||||
|
||||
235
scripts/compact-hf-telemetry.mjs
Normal file
235
scripts/compact-hf-telemetry.mjs
Normal file
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Compact legacy telemetry/ files into monthly archives under archive/legacy/.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/compact-hf-telemetry.mjs # dry-run (default)
|
||||
* node scripts/compact-hf-telemetry.mjs --apply # actually commit archives + delete legacy
|
||||
*
|
||||
* Requires: HF_TOKEN env var with write access to the dataset.
|
||||
*
|
||||
* Safety:
|
||||
* - Dry-run is default; --apply required for writes
|
||||
* - Verifies event counts before deleting anything
|
||||
* - Produces a manifest with checksums
|
||||
* - Stops on any count mismatch
|
||||
*/
|
||||
|
||||
const HF_API = 'https://huggingface.co/api';
|
||||
const DATASET_REPO = process.env.HF_DATASET_REPO || 'pliny-the-prompter/g0dm0d3';
|
||||
const BRANCH = process.env.HF_DATASET_BRANCH || 'main';
|
||||
const TOKEN = process.env.HF_TOKEN;
|
||||
const APPLY = process.argv.includes('--apply');
|
||||
|
||||
if (!TOKEN) {
|
||||
console.error('ERROR: HF_TOKEN environment variable required');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const headers = { Authorization: `Bearer ${TOKEN}` };
|
||||
|
||||
async function listFiles(prefix) {
|
||||
const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}`;
|
||||
const res = await fetch(url, { headers });
|
||||
if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
async function downloadFile(path) {
|
||||
const url = `https://huggingface.co/datasets/${DATASET_REPO}/resolve/${BRANCH}/${path}`;
|
||||
const res = await fetch(url, { headers });
|
||||
if (!res.ok) throw new Error(`Download failed (${res.status}): ${path}`);
|
||||
return res.text();
|
||||
}
|
||||
|
||||
async function sha256(text) {
|
||||
const data = new TextEncoder().encode(text);
|
||||
const buf = await crypto.subtle.digest('SHA-256', data);
|
||||
return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('');
|
||||
}
|
||||
|
||||
async function commitFiles(operations, summary) {
|
||||
const url = `${HF_API}/datasets/${DATASET_REPO}/commit/${BRANCH}`;
|
||||
const lines = [
|
||||
JSON.stringify({ key: 'header', value: { summary } }),
|
||||
...operations.map(op => JSON.stringify({ key: op.key, value: op.value })),
|
||||
];
|
||||
const res = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { ...headers, 'Content-Type': 'application/x-ndjson' },
|
||||
body: lines.join('\n'),
|
||||
});
|
||||
if (!res.ok) throw new Error(`Commit failed (${res.status}): ${await res.text()}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
function utf8ToBase64(str) {
|
||||
const bytes = new TextEncoder().encode(str);
|
||||
let binStr = '';
|
||||
for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]);
|
||||
return btoa(binStr);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log(`Mode: ${APPLY ? 'APPLY (will write)' : 'DRY-RUN (read-only)'}`);
|
||||
console.log(`Repo: ${DATASET_REPO} (branch: ${BRANCH})`);
|
||||
console.log('');
|
||||
|
||||
// Step 1: List all legacy telemetry files
|
||||
console.log('Step 1: Listing legacy telemetry/ files...');
|
||||
const files = await listFiles('telemetry');
|
||||
const jsonlFiles = files.filter(f => f.type === 'file' && f.path.endsWith('.jsonl'));
|
||||
console.log(` Found ${jsonlFiles.length} JSONL files`);
|
||||
|
||||
if (jsonlFiles.length === 0) {
|
||||
console.log(' Nothing to compact. Done.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 2: Download and parse all files
|
||||
console.log('Step 2: Downloading and parsing events...');
|
||||
const monthBuckets = {};
|
||||
let totalEvents = 0;
|
||||
let totalBytes = 0;
|
||||
let invalidLines = 0;
|
||||
const invalidRecords = [];
|
||||
|
||||
for (let i = 0; i < jsonlFiles.length; i++) {
|
||||
const file = jsonlFiles[i];
|
||||
if (i % 500 === 0) console.log(` Progress: ${i}/${jsonlFiles.length} files...`);
|
||||
|
||||
try {
|
||||
const content = await downloadFile(file.path);
|
||||
totalBytes += new TextEncoder().encode(content).byteLength;
|
||||
const lines = content.split('\n').filter(l => l.trim());
|
||||
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const event = JSON.parse(line);
|
||||
const ts = event.timestamp;
|
||||
let monthKey;
|
||||
if (ts && typeof ts === 'number') {
|
||||
const d = new Date(ts);
|
||||
monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`;
|
||||
} else {
|
||||
monthKey = 'unknown';
|
||||
}
|
||||
if (!monthBuckets[monthKey]) monthBuckets[monthKey] = [];
|
||||
monthBuckets[monthKey].push(line);
|
||||
totalEvents++;
|
||||
} catch {
|
||||
invalidLines++;
|
||||
invalidRecords.push({ file: file.path, line: line.slice(0, 200) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(` ERROR reading ${file.path}: ${err.message}`);
|
||||
invalidRecords.push({ file: file.path, error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` Total events: ${totalEvents}`);
|
||||
console.log(` Total source bytes: ${(totalBytes / 1024 / 1024).toFixed(2)} MiB`);
|
||||
console.log(` Invalid lines: ${invalidLines}`);
|
||||
console.log(` Month buckets: ${Object.keys(monthBuckets).sort().join(', ')}`);
|
||||
console.log('');
|
||||
|
||||
// Step 3: Build archives
|
||||
console.log('Step 3: Building monthly archives...');
|
||||
const archives = [];
|
||||
for (const [month, events] of Object.entries(monthBuckets).sort()) {
|
||||
const content = events.join('\n') + '\n';
|
||||
const checksum = await sha256(content);
|
||||
const archivePath = `archive/legacy/telemetry-${month}.jsonl`;
|
||||
const bytes = new TextEncoder().encode(content).byteLength;
|
||||
archives.push({ month, path: archivePath, events: events.length, bytes, checksum, content });
|
||||
console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 12)}...`);
|
||||
}
|
||||
|
||||
// Step 4: Verify counts
|
||||
const archiveEventTotal = archives.reduce((s, a) => s + a.events, 0);
|
||||
console.log('');
|
||||
console.log(`Step 4: Verification`);
|
||||
console.log(` Source events: ${totalEvents}`);
|
||||
console.log(` Archive events: ${archiveEventTotal}`);
|
||||
|
||||
if (archiveEventTotal !== totalEvents) {
|
||||
console.error(' MISMATCH — aborting. No changes made.');
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(' MATCH — counts verified.');
|
||||
|
||||
// Build manifest
|
||||
const manifest = {
|
||||
timestamp: new Date().toISOString(),
|
||||
repo: DATASET_REPO,
|
||||
branch: BRANCH,
|
||||
sourceFiles: jsonlFiles.length,
|
||||
sourceBytes: totalBytes,
|
||||
totalEvents,
|
||||
invalidLines,
|
||||
invalidRecords: invalidRecords.length,
|
||||
archives: archives.map(a => ({
|
||||
path: a.path, month: a.month, events: a.events,
|
||||
bytes: a.bytes, checksum: a.checksum,
|
||||
})),
|
||||
};
|
||||
|
||||
console.log('');
|
||||
console.log('Manifest:');
|
||||
console.log(JSON.stringify(manifest, null, 2));
|
||||
|
||||
if (!APPLY) {
|
||||
console.log('');
|
||||
console.log('DRY-RUN complete. Re-run with --apply to commit changes.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 5: Upload archives
|
||||
console.log('');
|
||||
console.log('Step 5: Uploading archives...');
|
||||
const uploadOps = archives.map(a => ({
|
||||
key: 'file',
|
||||
value: { content: utf8ToBase64(a.content), path: a.path, encoding: 'base64' },
|
||||
}));
|
||||
uploadOps.push({
|
||||
key: 'file',
|
||||
value: {
|
||||
content: utf8ToBase64(JSON.stringify(manifest, null, 2)),
|
||||
path: 'archive/legacy/manifest.json',
|
||||
encoding: 'base64',
|
||||
},
|
||||
});
|
||||
|
||||
// Save invalid records if any
|
||||
if (invalidRecords.length > 0) {
|
||||
uploadOps.push({
|
||||
key: 'file',
|
||||
value: {
|
||||
content: utf8ToBase64(JSON.stringify(invalidRecords, null, 2)),
|
||||
path: 'archive/legacy/invalid-records.json',
|
||||
encoding: 'base64',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files`);
|
||||
console.log(' Archives committed.');
|
||||
|
||||
// Step 6: Delete legacy files
|
||||
console.log('Step 6: Deleting legacy files...');
|
||||
const deleteOps = jsonlFiles.map(f => ({
|
||||
key: 'deletedFile',
|
||||
value: { path: f.path },
|
||||
}));
|
||||
await commitFiles(deleteOps, `[maintenance] Remove ${jsonlFiles.length} legacy telemetry files (archived in archive/legacy/)`);
|
||||
console.log(` Deleted ${jsonlFiles.length} files.`);
|
||||
|
||||
console.log('');
|
||||
console.log('DONE. Legacy telemetry compacted successfully.');
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('Fatal error:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user