/** * G0DM0D3 Telemetry Proxy — Cloudflare Pages Function * * Receives metadata events from the frontend and buffers them in * Cloudflare KV. When enough batches accumulate (or enough time * passes), pending batches are flushed as JSONL commits to * HuggingFace. This prevents the 10k+ commit buildup that * happens with per-request commits. * * URL: POST /api/telemetry * * Setup (Cloudflare Pages Dashboard → Settings → Environment Variables): * HF_TOKEN — HuggingFace write token (hf_...) * HF_DATASET_REPO — Target dataset repo (e.g. "pliny-the-prompter/g0dm0d3") * TELEMETRY_FLUSH_SECRET — Bearer token for authenticated cron flush * * KV Binding (Cloudflare Pages Dashboard → Settings → Bindings): * TELEMETRY_KV — KV namespace for buffering * * Without TELEMETRY_KV bound, telemetry accumulates in an * in-memory isolate buffer and flushes to HF in batches. * KV is preferred (survives isolate eviction) but not required. * * Cloudflare-level protections (configure in Dashboard → Security): * - WAF rule: block POST /api/telemetry with Content-Length > 524288 * - Rate limiting rule: 20 req/min per IP to /api/telemetry * - Restrict Origin header to godmod3.ai (optional) * - Monthly usage alert on Workers & KV (paid plan required at scale) */ interface Env { HF_TOKEN: string HF_DATASET_REPO: string HF_DATASET_BRANCH?: string TELEMETRY_KV?: KVNamespace TELEMETRY_FLUSH_SECRET?: string } interface TelemetryEvent { type: string timestamp: number session_id: string [key: string]: unknown } interface TelemetryPayload { events: TelemetryEvent[] } const HF_API = 'https://huggingface.co/api' const CORS_HEADERS = { 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'POST, OPTIONS', 'Access-Control-Allow-Headers': 'Content-Type', 'Access-Control-Max-Age': '86400', } // ── Size & batching limits ────────────────────────────────────────── // Workers have 128 MiB memory. Keep individual requests small and // cap the total data loaded during a flush. const MAX_REQUEST_BYTES = 512 * 1024 // 512 KiB per inbound POST const MAX_BATCH_EVENTS = 100 // events per request const FLUSH_BATCH_COUNT = 50 // keys before triggering flush const FLUSH_MAX_KEYS = 50 // max keys consumed per flush cycle const FLUSH_MAX_BYTES = 5 * 1024 * 1024 // 5 MiB combined JSONL per HF commit const FLUSH_AGE_MS = 30 * 60 * 1000 // 30 minutes // ── In-memory isolate buffer (fallback when KV not bound) ─────────── // Same persistence model as the rate limiter: module-level state // survives across requests within the same Worker isolate. We // accumulate JSONL lines and flush to HF when the buffer is large // or old enough. Isolate eviction loses buffered data — acceptable // for best-effort telemetry, not acceptable for the commit explosion. const MEM_FLUSH_COUNT = 50 // events before flush const MEM_FLUSH_BYTES = 256 * 1024 // 256 KiB before flush const MEM_FLUSH_AGE_MS = 15 * 60 * 1000 // 15 minutes before flush const MEM_MAX_BYTES = 2 * 1024 * 1024 // 2 MiB hard cap — drop oldest if exceeded const memBuffer: string[] = [] let memBufferBytes = 0 let memBufferFirstTs = 0 // ── Rate Limiter (dual: session + IP, in-memory per-isolate) ──────── // Client IP is processed transiently here and is not copied into published events. const RATE_LIMIT_WINDOW_MS = 60_000 const RATE_LIMIT_MAX_SESSION = 10 // per session_id const RATE_LIMIT_MAX_IP = 20 // per IP — catches session_id rotation const rateLimitMap = new Map() function isRateLimited(key: string, max: number): boolean { const now = Date.now() let timestamps = rateLimitMap.get(key) if (!timestamps) { timestamps = [] rateLimitMap.set(key, timestamps) } const cutoff = now - RATE_LIMIT_WINDOW_MS while (timestamps.length > 0 && timestamps[0] <= cutoff) { timestamps.shift() } if (timestamps.length >= max) { return true } timestamps.push(now) // Prevent unbounded memory growth across isolate lifetime if (rateLimitMap.size > 10_000) { const it = rateLimitMap.keys() for (let i = 0; i < 1_000; i++) { const k = it.next() if (k.done) break rateLimitMap.delete(k.value) } } return false } function deriveSessionKey(event: TelemetryEvent): string { const raw = JSON.stringify(event) let h = 0 for (let i = 0; i < raw.length; i++) { h = ((h << 5) - h + raw.charCodeAt(i)) | 0 } return `__derived_${Math.abs(h).toString(36)}` } // ── Event Schema Validation ────────────────────────────────────────── function validateEvent(event: unknown): event is TelemetryEvent { if (typeof event !== 'object' || event === null || Array.isArray(event)) { return false } const e = event as Record if (typeof e.type !== 'string' || e.type.length === 0) return false if (typeof e.timestamp !== 'number' || !Number.isFinite(e.timestamp)) return false if (typeof e.session_id !== 'string') return false if (JSON.stringify(e).length > 65_536) return false return true } // Handle CORS preflight export const onRequestOptions: PagesFunction = async () => { return new Response(null, { status: 204, headers: CORS_HEADERS }) } // GET ?flush=force — authenticated cron trigger for guaranteed flush. export const onRequestGet: PagesFunction = async (context) => { const { request, env } = context const url = new URL(request.url) if (url.searchParams.get('flush') !== 'force') { return new Response(null, { status: 204, headers: CORS_HEADERS }) } if (!env.TELEMETRY_FLUSH_SECRET) { return jsonResponse({ error: 'Flush not configured' }, 503) } const authHeader = request.headers.get('Authorization') || '' const token = authHeader.replace(/^Bearer\s+/i, '') if (!token || token !== env.TELEMETRY_FLUSH_SECRET) { return jsonResponse({ error: 'Unauthorized' }, 401) } if (!env.TELEMETRY_KV || !env.HF_TOKEN || !env.HF_DATASET_REPO) { return jsonResponse({ error: 'KV or HF not configured' }, 503) } const flushed = await flushKVToHF(env) return jsonResponse({ flushed }, 200) } // Main handler export const onRequestPost: PagesFunction = async (context) => { const { request, env } = context // ── Early size rejection (before reading body into memory) ──────── const contentLength = parseInt(request.headers.get('Content-Length') || '', 10) if (contentLength > MAX_REQUEST_BYTES) { return jsonResponse({ error: 'Request too large', limit: MAX_REQUEST_BYTES }, 413) } if (!env.HF_TOKEN || !env.HF_DATASET_REPO) { const missing = [] if (!env.HF_TOKEN) missing.push('HF_TOKEN') if (!env.HF_DATASET_REPO) missing.push('HF_DATASET_REPO') console.error(`[Telemetry] Missing env vars: ${missing.join(', ')}`) return jsonResponse({ error: `Telemetry not configured (missing: ${missing.join(', ')})` }, 503) } // ── IP-based rate limiting (catches session_id rotation attacks) ─── const clientIP = request.headers.get('CF-Connecting-IP') || request.headers.get('X-Forwarded-For') || 'unknown' if (isRateLimited(`ip:${clientIP}`, RATE_LIMIT_MAX_IP)) { return jsonResponse({ error: 'Rate limit exceeded — try again later' }, 429) } // ── Bounded body read (cancels stream at 512 KiB) ────────────────── // Content-Length can be spoofed or absent (chunked), so we enforce // the cap at the byte level while streaming — never buffer more than // MAX_REQUEST_BYTES regardless of what the client claims. let bodyText: string try { const reader = request.body?.getReader() if (!reader) { return jsonResponse({ error: 'No body' }, 400) } const chunks: Uint8Array[] = [] let totalBytes = 0 while (true) { const { done, value } = await reader.read() if (done) break totalBytes += value.byteLength if (totalBytes > MAX_REQUEST_BYTES) { reader.cancel() return jsonResponse({ error: 'Request too large', limit: MAX_REQUEST_BYTES }, 413) } chunks.push(value) } const merged = new Uint8Array(totalBytes) let offset = 0 for (const c of chunks) { merged.set(c, offset); offset += c.byteLength } bodyText = new TextDecoder().decode(merged) } catch { return jsonResponse({ error: 'Failed to read body' }, 400) } let payload: TelemetryPayload try { payload = JSON.parse(bodyText) as TelemetryPayload } catch { return jsonResponse({ error: 'Invalid JSON' }, 400) } if (!payload.events || !Array.isArray(payload.events) || payload.events.length === 0) { return jsonResponse({ error: 'No events provided' }, 400) } const events = payload.events.slice(0, MAX_BATCH_EVENTS) const invalid = events.filter(e => !validateEvent(e)) if (invalid.length > 0) { return jsonResponse({ error: `${invalid.length} event(s) failed schema validation` }, 400) } // ── Session-based rate limiting ─────────────────────────────────── const firstEvent = events[0] const sessionKey = firstEvent.session_id ? firstEvent.session_id : deriveSessionKey(firstEvent) if (isRateLimited(`session:${sessionKey}`, RATE_LIMIT_MAX_SESSION)) { return jsonResponse({ error: 'Rate limit exceeded — try again later' }, 429) } const sanitized = events.map(allowlistEventFields) const jsonl = sanitized.map(e => JSON.stringify(e)).join('\n') // ── KV buffered path (preferred) ────────────────────────────────── if (env.TELEMETRY_KV) { const now = Date.now() const kvKey = `batch:${now}:${shortHash(jsonl)}` await env.TELEMETRY_KV.put(kvKey, jsonl, { expirationTtl: 604800, // 7 days }) // Check flush conditions const pending = await env.TELEMETRY_KV.list({ prefix: 'batch:' }) const pendingCount = pending.keys.length let shouldFlush = pendingCount >= FLUSH_BATCH_COUNT if (!shouldFlush && pendingCount > 0) { const oldestKey = pending.keys[0].name const oldestTs = parseInt(oldestKey.split(':')[1], 10) if (!isNaN(oldestTs) && (now - oldestTs) > FLUSH_AGE_MS) { shouldFlush = true } } if (shouldFlush) { context.waitUntil(flushKVToHF(env)) } return jsonResponse({ accepted: sanitized.length, buffered: true, pending: pendingCount, flushing: shouldFlush, }, 200) } // ── In-memory buffer fallback (KV not configured) ────────────────── // Accumulates JSONL in the isolate's module scope and flushes to HF // when the buffer is large or old enough. Zero config required. // Data loss on isolate eviction is acceptable for telemetry. const now = Date.now() if (memBufferFirstTs === 0) memBufferFirstTs = now // Drop oldest lines if hard cap would be exceeded const lineBytes = new TextEncoder().encode(jsonl).byteLength while (memBufferBytes + lineBytes > MEM_MAX_BYTES && memBuffer.length > 0) { const dropped = memBuffer.shift()! memBufferBytes -= new TextEncoder().encode(dropped).byteLength } memBuffer.push(jsonl) memBufferBytes += lineBytes const age = now - memBufferFirstTs const shouldFlush = memBuffer.length >= MEM_FLUSH_COUNT || memBufferBytes >= MEM_FLUSH_BYTES || age >= MEM_FLUSH_AGE_MS if (shouldFlush) { context.waitUntil(flushMemToHF(env)) } return jsonResponse({ accepted: sanitized.length, buffered: true, pending: memBuffer.length, flushing: shouldFlush, storage: 'memory', }, 200) } // ── KV Flush: read up to FLUSH_MAX_KEYS, cap at FLUSH_MAX_BYTES ──── async function flushKVToHF(env: Env): Promise { const kv = env.TELEMETRY_KV! // Only list up to FLUSH_MAX_KEYS — leave the rest for the next cycle. // This bounds both KV read ops and memory consumption. const list = await kv.list({ prefix: 'batch:', limit: FLUSH_MAX_KEYS }) const keys = list.keys if (keys.length === 0) return 0 // Read values with a cumulative size cap const flushedKeys: { name: string }[] = [] const chunks: string[] = [] let totalBytes = 0 for (const key of keys) { const value = await kv.get(key.name) if (!value) { flushedKeys.push(key) // empty key — still delete it continue } const valueBytes = new TextEncoder().encode(value).byteLength if (totalBytes + valueBytes > FLUSH_MAX_BYTES && chunks.length > 0) { // Would exceed cap — stop here, remaining keys stay for next flush break } chunks.push(value) flushedKeys.push(key) totalBytes += valueBytes } if (chunks.length === 0) { // Only empty keys found — clean them up if (flushedKeys.length > 0) { await Promise.all(flushedKeys.map(key => kv.delete(key.name))) } return 0 } const combined = chunks.join('\n') const keyDigest = await sha256Hex(flushedKeys.map(k => k.name).sort().join(',')) // Derive partition from oldest batch key timestamp (stable across retries) const oldestTs = parseInt(flushedKeys[0].name.split(':')[1], 10) const d = !isNaN(oldestTs) ? new Date(oldestTs) : new Date() const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}` const filePath = `incoming_v2/${datePath}/flush_${keyDigest}.jsonl` const branch = env.HF_DATASET_BRANCH || 'main' const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined) if (ok) { await Promise.all(flushedKeys.map(key => kv.delete(key.name))) console.log(`[Telemetry] Flushed ${flushedKeys.length} batches (${(totalBytes / 1024).toFixed(0)} KiB) → ${filePath}`) return flushedKeys.length } console.error(`[Telemetry] Flush failed — ${flushedKeys.length} batches remain in KV`) return 0 } // ── In-memory flush: drain buffer → one HF commit ────────────────── async function flushMemToHF(env: Env): Promise { if (memBuffer.length === 0) return 0 // Snapshot and clear before the async commit so new requests // accumulate into a fresh buffer, not into the one we're flushing. const snapshot = memBuffer.splice(0, memBuffer.length) const snapshotBytes = memBufferBytes const snapshotTs = memBufferFirstTs memBufferBytes = 0 memBufferFirstTs = 0 const combined = snapshot.join('\n') const digest = await sha256Hex(combined) // Derive partition from buffer start time (stable across retries) const d = snapshotTs > 0 ? new Date(snapshotTs) : new Date() const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}` const filePath = `incoming_v2/${datePath}/membuf_${digest}.jsonl` const branch = env.HF_DATASET_BRANCH || 'main' const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined) if (ok) { console.log(`[Telemetry] Memory flush: ${snapshot.length} batches (${(snapshotBytes / 1024).toFixed(0)} KiB) → ${filePath}`) return snapshot.length } // Commit failed — push lines back so the next request retries. // Prepend so ordering is roughly preserved. memBuffer.unshift(...snapshot) memBufferBytes += snapshotBytes if (memBufferFirstTs === 0) memBufferFirstTs = snapshotTs console.error(`[Telemetry] Memory flush failed — ${snapshot.length} batches returned to buffer`) return 0 } // ── HuggingFace Hub Commit ─────────────────────────────────────────── async function commitToHF( token: string, repo: string, branch: string, filePath: string, content: string, ): Promise { const url = `${HF_API}/datasets/${repo}/commit/${branch}` const contentBase64 = utf8ToBase64(content) const ndjson = [ JSON.stringify({ key: 'header', value: { summary: `[telemetry] ${filePath}` } }), JSON.stringify({ key: 'file', value: { content: contentBase64, path: filePath, encoding: 'base64' } }), ].join('\n') try { const res = await fetch(url, { method: 'POST', headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/x-ndjson', }, body: ndjson, }) if (!res.ok) { const err = await res.text().catch(() => '') if (res.status === 401 || res.status === 403) { console.error(`[Telemetry] HF AUTH FAILED (${res.status}) — HF_TOKEN is invalid or lacks write access to "${repo}"`) } else if (res.status === 404) { console.error(`[Telemetry] HF REPO NOT FOUND (404) — "${repo}" does not exist on HuggingFace`) } else { console.error(`[Telemetry] HF commit failed (${res.status}): ${err.slice(0, 300)}`) } } return res.ok } catch (err) { console.error(`[Telemetry] Network error:`, err) return false } } // ── Helpers ────────────────────────────────────────────────────────── const ALLOWED_FIELDS = new Set([ // Core envelope 'type', 'timestamp', 'session_id', // Shared model/perf 'mode', 'model', 'duration_ms', 'response_length', 'success', 'error_type', // Pipeline config 'pipeline', 'stm_modules', 'strategy', 'godmode', 'auto_retry', 'improve_mode', 'liquid_mode', // AutoTune 'autotune', 'detected_context', 'confidence', 'autotune_context', 'autotune_confidence', // Parseltongue 'parseltongue', 'triggers_found', 'technique', 'intensity', 'parseltongue_transform', 'parseltongue_transformed', 'encoding', 'encoding_rounds', // ULTRAPLINIAN race 'ultraplinian', 'tier', 'models_queried', 'models_succeeded', 'models_refused', 'early_stop', 'early_threshold', 'winner_model', 'winner_score', 'winner_content_length', 'winner_duration_ms', 'winner_template', 'total_duration_ms', 'judge_model', 'model_results', 'all_scores', 'winner_combo', 'winner_source', 'race_result', 'combo', 'combos_attempted', 'combos_failed', 'combos_succeeded', // Standard completion 'attempts', 'content_length', 'temperature', 'top_p', 'fallback_reason', // Streaming / liquid 'stream', 'fast_stream', 'has_image', 'liquid_upgraded', 'liquid_upgrades', // Harm classification 'classification', // Structural context 'persona', 'prompt_length', 'conversation_depth', 'memory_count', 'no_log', ]) // Top-level field allowlist. This reduces accidental field leakage but is not // a recursive PII detector; keep event producers free of user-supplied content. function allowlistEventFields(event: TelemetryEvent): TelemetryEvent { const clean: Record = {} for (const key of Object.keys(event)) { if (ALLOWED_FIELDS.has(key)) { clean[key] = event[key] } } return clean as TelemetryEvent } function shortHash(str: string): string { let h = 0 for (let i = 0; i < str.length; i++) { h = ((h << 5) - h + str.charCodeAt(i)) | 0 } return Math.abs(h).toString(36).slice(0, 6) } async function sha256Hex(str: string): Promise { const data = new TextEncoder().encode(str) const buf = await crypto.subtle.digest('SHA-256', data) return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('') } // Chunked UTF-8-safe base64: processes 32 KiB at a time to avoid // creating a single enormous string via Array.from() on the full input. function utf8ToBase64(content: string): string { const bytes = new TextEncoder().encode(content) const CHUNK = 32 * 1024 let binStr = '' for (let i = 0; i < bytes.length; i += CHUNK) { const slice = bytes.subarray(i, Math.min(i + CHUNK, bytes.length)) for (let j = 0; j < slice.length; j++) { binStr += String.fromCharCode(slice[j]) } } return btoa(binStr) } function jsonResponse(data: unknown, status: number): Response { return new Response(JSON.stringify(data), { status, headers: { 'Content-Type': 'application/json', ...CORS_HEADERS, }, }) }