mirror of
https://github.com/elder-plinius/G0DM0D3.git
synced 2026-11-16 07:02:25 +00:00
297 lines
11 KiB
TypeScript
297 lines
11 KiB
TypeScript
/**
|
|
* HuggingFace Auto-Publisher
|
|
*
|
|
* Flushes in-memory metadata and dataset buffers to a HuggingFace
|
|
* Dataset repo as JSONL files before data is evicted. Zero data loss.
|
|
*
|
|
* Trigger conditions:
|
|
* 1. Buffer reaches flush threshold (default 80% of max capacity)
|
|
* 2. Periodic timer fires (default every 30 min if buffer has data)
|
|
* 3. Graceful shutdown (SIGTERM/SIGINT)
|
|
*
|
|
* Safety guarantee: items are only removed from the buffer AFTER a
|
|
* successful HF upload. If the upload fails, data stays in memory.
|
|
*
|
|
* Requires:
|
|
* HF_TOKEN — HuggingFace write token (hf_...)
|
|
* HF_DATASET_REPO — target repo (e.g. "pliny-the-prompter/g0dm0d3")
|
|
*
|
|
* File layout in the HF repo:
|
|
* metadata/batch_<timestamp>_<seq>.jsonl
|
|
* dataset/batch_<timestamp>_<seq>.jsonl
|
|
*/
|
|
|
|
// ── Config ───────────────────────────────────────────────────────────
|
|
|
|
const HF_API = 'https://huggingface.co/api'
|
|
|
|
function getConfig() {
|
|
return {
|
|
token: process.env.HF_TOKEN || '',
|
|
repo: process.env.HF_DATASET_REPO || '',
|
|
branch: process.env.HF_DATASET_BRANCH || 'main',
|
|
flushThreshold: parseFloat(process.env.HF_FLUSH_THRESHOLD || '0.8'), // 80% of buffer cap
|
|
periodicMs: parseInt(process.env.HF_FLUSH_INTERVAL_MS || '1800000', 10), // 30 min
|
|
}
|
|
}
|
|
|
|
export function isPublisherEnabled(): boolean {
|
|
const { token, repo } = getConfig()
|
|
return !!(token && repo)
|
|
}
|
|
|
|
// ── State ────────────────────────────────────────────────────────────
|
|
|
|
let metadataBatchSeq = 0
|
|
let datasetBatchSeq = 0
|
|
let periodicTimer: ReturnType<typeof setInterval> | null = null
|
|
let flushingMetadata = false
|
|
let flushingDataset = false
|
|
|
|
// ── Store Interface ──────────────────────────────────────────────────
|
|
// Each store registers two functions:
|
|
// snapshot() → returns a copy of all current items (non-destructive)
|
|
// clear(n) → removes the first n items (called only after successful upload)
|
|
|
|
interface StoreDrain {
|
|
snapshot: () => unknown[]
|
|
clear: (count: number) => void
|
|
}
|
|
|
|
let metadataStore: StoreDrain | null = null
|
|
let datasetStore: StoreDrain | null = null
|
|
|
|
export function registerMetadataStore(store: StoreDrain) {
|
|
metadataStore = store
|
|
}
|
|
|
|
export function registerDatasetStore(store: StoreDrain) {
|
|
datasetStore = store
|
|
}
|
|
|
|
// ── Core: Upload to HuggingFace ──────────────────────────────────────
|
|
|
|
async function commitFile(filePath: string, content: string): Promise<boolean> {
|
|
const { token, repo, branch } = getConfig()
|
|
if (!token || !repo) return false
|
|
|
|
const url = `${HF_API}/datasets/${repo}/commit/${branch}`
|
|
|
|
// HF Hub commit API uses NDJSON (application/x-ndjson)
|
|
// Line 1: commit header with summary
|
|
// Line 2: file operation with base64-encoded content
|
|
const contentBase64 = Buffer.from(content).toString('base64')
|
|
const ndjson = [
|
|
JSON.stringify({ key: 'header', value: { summary: `[auto-publish] ${filePath}` } }),
|
|
JSON.stringify({ key: 'file', value: { content: contentBase64, path: filePath, encoding: 'base64' } }),
|
|
].join('\n')
|
|
|
|
try {
|
|
const res = await fetch(url, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Authorization': `Bearer ${token}`,
|
|
'Content-Type': 'application/x-ndjson',
|
|
},
|
|
body: ndjson,
|
|
})
|
|
|
|
if (res.ok) {
|
|
console.log(`[HF Publisher] Committed ${filePath} (${content.length} bytes)`)
|
|
return true
|
|
}
|
|
|
|
const errText = await res.text().catch(() => '')
|
|
console.error(`[HF Publisher] Commit failed (${res.status}): ${errText.slice(0, 200)}`)
|
|
return false
|
|
} catch (err) {
|
|
console.error(`[HF Publisher] Network error:`, (err as Error).message)
|
|
return false
|
|
}
|
|
}
|
|
|
|
// ── Flush Functions ──────────────────────────────────────────────────
|
|
// Safe pattern: snapshot → upload → clear only on success
|
|
|
|
export async function flushMetadata(): Promise<{ flushed: number; success: boolean }> {
|
|
if (!isPublisherEnabled() || !metadataStore || flushingMetadata) {
|
|
return { flushed: 0, success: false }
|
|
}
|
|
|
|
flushingMetadata = true
|
|
try {
|
|
const items = metadataStore.snapshot()
|
|
if (items.length === 0) return { flushed: 0, success: true }
|
|
|
|
const jsonl = items.map(item => JSON.stringify(item)).join('\n')
|
|
const ts = new Date().toISOString().replace(/[:.]/g, '-')
|
|
const seq = String(++metadataBatchSeq).padStart(4, '0')
|
|
const filePath = `metadata/batch_${ts}_${seq}.jsonl`
|
|
|
|
const ok = await commitFile(filePath, jsonl)
|
|
if (ok) {
|
|
metadataStore.clear(items.length)
|
|
console.log(`[HF Publisher] Flushed ${items.length} metadata events`)
|
|
} else {
|
|
// Data stays in memory — will retry next threshold hit or periodic tick
|
|
console.error(`[HF Publisher] Metadata upload failed — ${items.length} events kept in memory`)
|
|
}
|
|
|
|
return { flushed: ok ? items.length : 0, success: ok }
|
|
} finally {
|
|
flushingMetadata = false
|
|
}
|
|
}
|
|
|
|
export async function flushDataset(): Promise<{ flushed: number; success: boolean }> {
|
|
if (!isPublisherEnabled() || !datasetStore || flushingDataset) {
|
|
return { flushed: 0, success: false }
|
|
}
|
|
|
|
flushingDataset = true
|
|
try {
|
|
const items = datasetStore.snapshot()
|
|
if (items.length === 0) return { flushed: 0, success: true }
|
|
|
|
const jsonl = items.map(item => JSON.stringify(item)).join('\n')
|
|
const ts = new Date().toISOString().replace(/[:.]/g, '-')
|
|
const seq = String(++datasetBatchSeq).padStart(4, '0')
|
|
const filePath = `dataset/batch_${ts}_${seq}.jsonl`
|
|
|
|
const ok = await commitFile(filePath, jsonl)
|
|
if (ok) {
|
|
datasetStore.clear(items.length)
|
|
console.log(`[HF Publisher] Flushed ${items.length} dataset entries`)
|
|
} else {
|
|
console.error(`[HF Publisher] Dataset upload failed — ${items.length} entries kept in memory`)
|
|
}
|
|
|
|
return { flushed: ok ? items.length : 0, success: ok }
|
|
} finally {
|
|
flushingDataset = false
|
|
}
|
|
}
|
|
|
|
// ── Credential Validation (runs once at startup) ────────────────────
|
|
|
|
async function validateCredentials(): Promise<void> {
|
|
const { token, repo } = getConfig()
|
|
if (!token || !repo) return
|
|
|
|
try {
|
|
const res = await fetch(`${HF_API}/datasets/${repo}`, {
|
|
headers: { 'Authorization': `Bearer ${token}` },
|
|
})
|
|
|
|
if (res.ok) {
|
|
console.log(`[HF Publisher] ✓ Credentials valid — repo "${repo}" accessible`)
|
|
} else if (res.status === 401 || res.status === 403) {
|
|
console.error(`[HF Publisher] ✗ AUTHENTICATION FAILED (${res.status}) — HF_TOKEN is invalid or lacks write access to "${repo}"`)
|
|
console.error(`[HF Publisher] ✗ Data will accumulate in memory but NEVER publish. Fix your HF_TOKEN.`)
|
|
} else if (res.status === 404) {
|
|
console.error(`[HF Publisher] ✗ REPO NOT FOUND (404) — "${repo}" does not exist. Create it on HuggingFace first.`)
|
|
} else {
|
|
console.warn(`[HF Publisher] ? Unexpected status ${res.status} checking repo "${repo}" — publishing may fail`)
|
|
}
|
|
} catch (err) {
|
|
console.warn(`[HF Publisher] ? Could not validate credentials (network error: ${(err as Error).message}) — will retry on first flush`)
|
|
}
|
|
}
|
|
|
|
// ── Threshold Check (called by stores after each insert) ─────────────
|
|
|
|
export function checkMetadataThreshold(currentSize: number, maxSize: number) {
|
|
if (!isPublisherEnabled()) return
|
|
const { flushThreshold } = getConfig()
|
|
if (currentSize >= maxSize * flushThreshold && !flushingMetadata) {
|
|
flushMetadata().catch(err => {
|
|
console.error('[HF Publisher] Async metadata flush error:', err)
|
|
})
|
|
}
|
|
}
|
|
|
|
export function checkDatasetThreshold(currentSize: number, maxSize: number) {
|
|
if (!isPublisherEnabled()) return
|
|
const { flushThreshold } = getConfig()
|
|
if (currentSize >= maxSize * flushThreshold && !flushingDataset) {
|
|
flushDataset().catch(err => {
|
|
console.error('[HF Publisher] Async dataset flush error:', err)
|
|
})
|
|
}
|
|
}
|
|
|
|
// ── Periodic Timer ───────────────────────────────────────────────────
|
|
|
|
export function startPeriodicFlush() {
|
|
if (!isPublisherEnabled()) {
|
|
console.warn('[HF Publisher] Periodic flush SKIPPED — publisher not enabled (missing HF_TOKEN or HF_DATASET_REPO)')
|
|
return
|
|
}
|
|
|
|
const { periodicMs, repo } = getConfig()
|
|
console.log(`[HF Publisher] Periodic flush STARTED — every ${Math.round(periodicMs / 60000)}m → ${repo}`)
|
|
|
|
// Validate credentials on first tick (non-blocking)
|
|
validateCredentials().catch(() => {})
|
|
|
|
periodicTimer = setInterval(async () => {
|
|
const metaSnapshot = metadataStore?.snapshot().length ?? 0
|
|
const dataSnapshot = datasetStore?.snapshot().length ?? 0
|
|
console.log(`[HF Publisher] Periodic tick — metadata buffer: ${metaSnapshot}, dataset buffer: ${dataSnapshot}`)
|
|
|
|
try {
|
|
const [meta, data] = await Promise.all([
|
|
flushMetadata(),
|
|
flushDataset(),
|
|
])
|
|
if (meta.flushed > 0 || data.flushed > 0) {
|
|
console.log(`[HF Publisher] Periodic flush: ${meta.flushed} metadata, ${data.flushed} dataset`)
|
|
}
|
|
} catch (err) {
|
|
console.error('[HF Publisher] Periodic flush error:', err)
|
|
}
|
|
}, periodicMs)
|
|
|
|
// Don't keep the process alive just for the timer
|
|
if (periodicTimer.unref) periodicTimer.unref()
|
|
}
|
|
|
|
export function stopPeriodicFlush() {
|
|
if (periodicTimer) {
|
|
clearInterval(periodicTimer)
|
|
periodicTimer = null
|
|
}
|
|
}
|
|
|
|
// ── Graceful Shutdown ────────────────────────────────────────────────
|
|
|
|
export async function shutdownFlush(): Promise<void> {
|
|
if (!isPublisherEnabled()) return
|
|
|
|
console.log('[HF Publisher] Shutdown — flushing remaining data...')
|
|
stopPeriodicFlush()
|
|
|
|
const [meta, data] = await Promise.all([
|
|
flushMetadata(),
|
|
flushDataset(),
|
|
])
|
|
|
|
console.log(`[HF Publisher] Final flush: ${meta.flushed} metadata, ${data.flushed} dataset`)
|
|
}
|
|
|
|
// ── Publisher Status (for /v1/info) ──────────────────────────────────
|
|
|
|
export function getPublisherStatus() {
|
|
const { repo, flushThreshold, periodicMs } = getConfig()
|
|
return {
|
|
enabled: isPublisherEnabled(),
|
|
repo: isPublisherEnabled() ? repo : undefined,
|
|
flush_threshold: flushThreshold,
|
|
periodic_interval_ms: periodicMs,
|
|
batches_published: {
|
|
metadata: metadataBatchSeq,
|
|
dataset: datasetBatchSeq,
|
|
},
|
|
}
|
|
}
|