fix: use recursive listing when scanning archive/legacy for manifests

HF tree API defaults to non-recursive, so nested manifest.json files
under archive/legacy/<runId>/ were invisible. After a partial deletion,
the script would miss the original manifest and incorrectly rebuild
archives from the reduced file set.

Now listAllFiles accepts a recursive flag, and the archive scan uses it.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
faber
2026-07-11 14:00:25 -04:00
parent 9f4f898b8e
commit 9f714a09fc

View File

@@ -39,13 +39,14 @@ if (!TOKEN) {
const headers = { Authorization: `Bearer ${TOKEN}` };
// ── Paginated file listing ──────────────────────────────────────────
async function listAllFiles(prefix) {
async function listAllFiles(prefix, recursive = false) {
const allFiles = [];
let cursor = null;
while (true) {
const params = new URLSearchParams();
if (cursor) params.set('cursor', cursor);
if (recursive) params.set('recursive', 'true');
const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}?${params}`;
const res = await fetch(url, { headers });
if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`);
@@ -145,7 +146,7 @@ async function main() {
console.log(`Step 2: Checking for existing manifest (run ID: ${currentRunId})...`);
// Also check if a DIFFERENT run's manifest exists (partial deletion scenario)
const archiveLegacyEntries = await listAllFiles('archive/legacy');
const archiveLegacyEntries = await listAllFiles('archive/legacy', true);
const existingManifests = archiveLegacyEntries.filter(
f => f.type === 'file' && f.path.endsWith('/manifest.json')
);