Files
G0DM0D3/functions/api/telemetry.ts
2026-03-25 10:44:35 -07:00

372 lines
11 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* G0DM0D3 Telemetry Proxy — Cloudflare Pages Function
*
* Receives metadata events from the frontend and commits them
* to a HuggingFace Dataset repo as JSONL. The HF token lives
* server-side in CF Pages environment variables (never exposed
* to the browser).
*
* URL: POST /api/telemetry
*
* Setup (Cloudflare Pages Dashboard → Settings → Environment Variables):
* HF_TOKEN — HuggingFace write token (hf_...)
* HF_DATASET_REPO — Target dataset repo (e.g. "pliny-the-prompter/g0dm0d3")
*
* The frontend batches events and sends them here periodically.
* Each batch becomes a single JSONL file committed to the HF repo.
*
* File layout in the HF repo:
* telemetry/batch_<timestamp>_<hash>.jsonl
*/
interface Env {
HF_TOKEN: string
HF_DATASET_REPO: string
HF_DATASET_BRANCH?: string
}
interface TelemetryEvent {
type: string
timestamp: number
session_id: string
[key: string]: unknown
}
interface TelemetryPayload {
events: TelemetryEvent[]
}
const HF_API = 'https://huggingface.co/api'
// CORS headers for cross-origin requests
const CORS_HEADERS = {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'POST, OPTIONS',
'Access-Control-Allow-Headers': 'Content-Type',
'Access-Control-Max-Age': '86400',
}
// ── Rate Limiter (in-memory, per-isolate) ────────────────────────────
const RATE_LIMIT_WINDOW_MS = 60_000 // 1 minute
const RATE_LIMIT_MAX = 10 // max requests per window per session_id
const rateLimitMap = new Map<string, number[]>()
/** Returns true if this session_id has exceeded the rate limit. */
function isRateLimited(sessionId: string): boolean {
const now = Date.now()
let timestamps = rateLimitMap.get(sessionId)
if (!timestamps) {
timestamps = []
rateLimitMap.set(sessionId, timestamps)
}
// Evict entries older than the window
const cutoff = now - RATE_LIMIT_WINDOW_MS
while (timestamps.length > 0 && timestamps[0] <= cutoff) {
timestamps.shift()
}
if (timestamps.length >= RATE_LIMIT_MAX) {
return true
}
timestamps.push(now)
return false
}
/** Derive a fallback key when session_id is missing. */
function deriveSessionKey(event: TelemetryEvent): string {
const raw = JSON.stringify(event)
let h = 0
for (let i = 0; i < raw.length; i++) {
h = ((h << 5) - h + raw.charCodeAt(i)) | 0
}
return `__derived_${Math.abs(h).toString(36)}`
}
// ── Event Schema Validation ──────────────────────────────────────────
/** Validates that an event conforms to the expected shape. */
function validateEvent(event: unknown): event is TelemetryEvent {
if (typeof event !== 'object' || event === null || Array.isArray(event)) {
return false
}
const e = event as Record<string, unknown>
// 'type' must be a non-empty string
if (typeof e.type !== 'string' || e.type.length === 0) return false
// 'timestamp' must be a finite number
if (typeof e.timestamp !== 'number' || !Number.isFinite(e.timestamp)) return false
// 'session_id' is required and must be a string (can be empty – the
// rate limiter will derive a key if needed, but the field must exist)
if (typeof e.session_id !== 'string') return false
// Reject events that are unreasonably large (> 64 KB serialised)
if (JSON.stringify(e).length > 65_536) return false
return true
}
// Handle CORS preflight
export const onRequestOptions: PagesFunction<Env> = async () => {
return new Response(null, { status: 204, headers: CORS_HEADERS })
}
// Main handler: receive events and push to HF
export const onRequestPost: PagesFunction<Env> = async (context) => {
const { request, env } = context
// Validate config
if (!env.HF_TOKEN || !env.HF_DATASET_REPO) {
const missing = []
if (!env.HF_TOKEN) missing.push('HF_TOKEN')
if (!env.HF_DATASET_REPO) missing.push('HF_DATASET_REPO')
console.error(`[Telemetry] Missing env vars: ${missing.join(', ')} — set these in Cloudflare Pages Dashboard → Settings → Environment Variables`)
return jsonResponse({ error: `Telemetry not configured (missing: ${missing.join(', ')})` }, 503)
}
// Parse body
let payload: TelemetryPayload
try {
payload = await request.json() as TelemetryPayload
} catch {
return jsonResponse({ error: 'Invalid JSON' }, 400)
}
if (!payload.events || !Array.isArray(payload.events) || payload.events.length === 0) {
return jsonResponse({ error: 'No events provided' }, 400)
}
// Cap batch size to prevent abuse
const MAX_BATCH = 500
const events = payload.events.slice(0, MAX_BATCH)
// ── Validate every event against the expected schema ──
const invalid = events.filter(e => !validateEvent(e))
if (invalid.length > 0) {
return jsonResponse(
{ error: `${invalid.length} event(s) failed schema validation` },
400,
)
}
// ── Rate limiting (per session_id) ──
// Use the session_id from the first event; fall back to a derived key
const firstEvent = events[0]
const sessionKey = firstEvent.session_id
? firstEvent.session_id
: deriveSessionKey(firstEvent)
if (isRateLimited(sessionKey)) {
return jsonResponse({ error: 'Rate limit exceeded — try again later' }, 429)
}
// Strip to allowlisted fields only (defense in depth)
const sanitized = events.map(stripPII)
// Convert to JSONL
const jsonl = sanitized.map(e => JSON.stringify(e)).join('\n')
// Generate filename
const ts = new Date().toISOString().replace(/[:.]/g, '-')
const hash = shortHash(jsonl)
const filePath = `telemetry/batch_${ts}_${hash}.jsonl`
// Commit to HF
const branch = env.HF_DATASET_BRANCH || 'main'
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, jsonl)
if (ok) {
return jsonResponse({
accepted: sanitized.length,
file: filePath,
}, 200)
}
return jsonResponse({ error: 'Failed to publish to HuggingFace — check function logs for details' }, 502)
}
// ── HuggingFace Hub Commit ───────────────────────────────────────────
async function commitToHF(
token: string,
repo: string,
branch: string,
filePath: string,
content: string,
): Promise<boolean> {
const url = `${HF_API}/datasets/${repo}/commit/${branch}`
// HF Hub commit API uses NDJSON (application/x-ndjson)
// Line 1: commit header with summary
// Line 2: file operation with base64-encoded content
const contentBase64 = btoa(content)
const ndjson = [
JSON.stringify({ key: 'header', value: { summary: `[telemetry] ${filePath}` } }),
JSON.stringify({ key: 'file', value: { content: contentBase64, path: filePath, encoding: 'base64' } }),
].join('\n')
try {
const res = await fetch(url, {
method: 'POST',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/x-ndjson',
},
body: ndjson,
})
if (!res.ok) {
const err = await res.text().catch(() => '')
if (res.status === 401 || res.status === 403) {
console.error(`[Telemetry] HF AUTH FAILED (${res.status}) — HF_TOKEN is invalid or lacks write access to "${repo}"`)
} else if (res.status === 404) {
console.error(`[Telemetry] HF REPO NOT FOUND (404) — "${repo}" does not exist on HuggingFace`)
} else {
console.error(`[Telemetry] HF commit failed (${res.status}): ${err.slice(0, 300)}`)
}
}
return res.ok
} catch (err) {
console.error(`[Telemetry] Network error:`, err)
return false
}
}
// ── Helpers ──────────────────────────────────────────────────────────
// Allowlist of fields that are safe to persist.
// Everything NOT on this list is silently dropped — this is the inverse
// of the old blocklist approach and far safer against novel PII leaks.
const ALLOWED_FIELDS = new Set<string>([
// Core event envelope
'type',
'timestamp',
'session_id',
// Shared model/perf fields
'mode',
'model',
'duration_ms',
'response_length',
'success',
'error_type',
// Pipeline config (nested object — passes through as-is)
'pipeline',
// AutoTune (nested object)
'autotune',
'detected_context',
'confidence',
// Parseltongue (nested object)
'parseltongue',
'triggers_found',
'technique',
'intensity',
// ULTRAPLINIAN race fields
'ultraplinian',
'tier',
'models_queried',
'models_succeeded',
'models_refused',
'early_stop',
'early_threshold',
'winner_model',
'winner_score',
'winner_content_length',
'winner_duration_ms',
'winner_template',
'total_duration_ms',
'judge_model',
'model_results',
// Standard completion fields
'attempts',
'content_length',
'temperature',
'top_p',
'parseltongue_transform',
// Pipeline sub-fields (when flattened)
'stm_modules',
'strategy',
'godmode',
'auto_retry',
'improve_mode',
'liquid_mode',
'autotune_context',
'autotune_confidence',
// Harm classification (nested object with domain, subcategory, confidence, intent, flags)
'classification',
// Structural context (no content, no PII)
'persona',
'prompt_length',
'conversation_depth',
'memory_count',
'no_log',
'parseltongue_transformed',
'has_image',
// G0DM0D3 CLASSIC race fields
'winner_combo',
'combos_attempted',
'combos_succeeded',
'combos_failed',
'all_scores',
'encoding',
'encoding_rounds',
'liquid_upgrades',
// GODMODE FAST fields
'combo',
'stream',
'fast_stream', // nested: {model, success, content_length}
'liquid_upgraded',
'winner_source', // 'fast' | 'race'
'race_result', // nested: {winner_combo, winner_model, winner_score, ...}
// Failure tracking
'fallback_reason',
// Parseltongue detail fields (nested under 'parseltongue')
// tier, technique, technique_label, triggers_found, variants_total,
// variants_succeeded, variants_refused, winner_score — all nested
])
function stripPII(event: TelemetryEvent): TelemetryEvent {
const clean: Record<string, unknown> = {}
for (const key of Object.keys(event)) {
if (ALLOWED_FIELDS.has(key)) {
clean[key] = event[key]
}
}
return clean as TelemetryEvent
}
function shortHash(str: string): string {
let h = 0
for (let i = 0; i < str.length; i++) {
h = ((h << 5) - h + str.charCodeAt(i)) | 0
}
return Math.abs(h).toString(36).slice(0, 6)
}
function jsonResponse(data: unknown, status: number): Response {
return new Response(JSON.stringify(data), {
status,
headers: {
'Content-Type': 'application/json',
...CORS_HEADERS,
},
})
}