mirror of
https://github.com/elder-plinius/G0DM0D3.git
synced 2026-11-16 07:02:25 +00:00
393 lines
12 KiB
TypeScript
393 lines
12 KiB
TypeScript
/**
|
|
* ZDR Metadata Tracker
|
|
*
|
|
* Always-on, privacy-first analytics for G0DM0D3.
|
|
* Records EVERYTHING about requests — except the actual content.
|
|
*
|
|
* What IS tracked:
|
|
* - Timestamps, endpoints, modes, tiers
|
|
* - Models used, winner model, scores, latencies
|
|
* - Pipeline flags (godmode, autotune, parseltongue, stm)
|
|
* - AutoTune detected context type + confidence
|
|
* - Per-model success/failure, response lengths, durations
|
|
* - Error types (categorized, never raw error messages)
|
|
*
|
|
* What is NEVER tracked:
|
|
* - Message content, system prompts, responses
|
|
* - API keys, auth tokens, IP addresses
|
|
* - Any PII whatsoever
|
|
*
|
|
* Storage: in-memory ring buffer, resets on restart.
|
|
*/
|
|
|
|
import { randomUUID } from 'crypto'
|
|
|
|
// ── Types ────────────────────────────────────────────────────────────
|
|
|
|
export interface MetadataEvent {
|
|
id: string
|
|
timestamp: number
|
|
|
|
// Request shape
|
|
endpoint: string
|
|
mode: 'standard' | 'ultraplinian'
|
|
tier?: string // fast | standard | full (ultraplinian only)
|
|
stream: boolean
|
|
|
|
// Pipeline config (what was enabled, not what it produced)
|
|
pipeline: {
|
|
godmode: boolean
|
|
autotune: boolean
|
|
parseltongue: boolean
|
|
stm_modules: string[]
|
|
strategy?: string
|
|
}
|
|
|
|
// AutoTune metadata (context detection only, no content)
|
|
autotune?: {
|
|
detected_context: string
|
|
confidence: number
|
|
}
|
|
|
|
// Model metadata
|
|
model?: string // single-model mode
|
|
models_queried?: number // ultraplinian
|
|
models_succeeded?: number
|
|
|
|
// Per-model results (no content, just numbers)
|
|
model_results?: Array<{
|
|
model: string
|
|
score: number
|
|
duration_ms: number
|
|
success: boolean
|
|
content_length: number
|
|
error_type?: string // categorized: 'timeout' | 'rate_limit' | 'auth' | 'model_error' | 'empty' | 'unknown'
|
|
}>
|
|
|
|
// Winner metadata
|
|
winner?: {
|
|
model: string
|
|
score: number
|
|
duration_ms: number
|
|
content_length: number
|
|
}
|
|
|
|
// Response metadata
|
|
total_duration_ms: number
|
|
response_length: number
|
|
|
|
// Liquid Response metadata (streaming upgrades)
|
|
liquid?: {
|
|
upgrades: number // how many leader changes happened
|
|
first_response_ms: number // time to first leader
|
|
}
|
|
}
|
|
|
|
// ── In-Memory Ring Buffer ────────────────────────────────────────────
|
|
|
|
let events: MetadataEvent[] = []
|
|
const MAX_EVENTS = 50000 // ~50k events before eviction
|
|
|
|
// ── Recording ────────────────────────────────────────────────────────
|
|
|
|
export function recordEvent(event: Omit<MetadataEvent, 'id' | 'timestamp'>): string {
|
|
const id = randomUUID()
|
|
const record: MetadataEvent = {
|
|
...event,
|
|
id,
|
|
timestamp: Date.now(),
|
|
}
|
|
|
|
events.push(record)
|
|
|
|
// Evict oldest if over cap
|
|
if (events.length > MAX_EVENTS) {
|
|
events = events.slice(events.length - MAX_EVENTS)
|
|
}
|
|
|
|
return id
|
|
}
|
|
|
|
// ── Error Categorization ─────────────────────────────────────────────
|
|
|
|
export function categorizeError(error?: string): string | undefined {
|
|
if (!error) return undefined
|
|
const lower = error.toLowerCase()
|
|
if (lower.includes('timeout') || lower.includes('abort')) return 'timeout'
|
|
if (lower.includes('rate') || lower.includes('429')) return 'rate_limit'
|
|
if (lower.includes('auth') || lower.includes('401') || lower.includes('403')) return 'auth'
|
|
if (lower.includes('empty')) return 'empty'
|
|
if (lower.includes('race ended')) return 'early_exit'
|
|
if (lower.includes('api error') || lower.includes('http')) return 'model_error'
|
|
return 'unknown'
|
|
}
|
|
|
|
// ── Query: Raw Events ────────────────────────────────────────────────
|
|
|
|
export function getEvents(opts?: {
|
|
limit?: number
|
|
offset?: number
|
|
since?: number // unix ms timestamp
|
|
mode?: 'standard' | 'ultraplinian'
|
|
}): { events: MetadataEvent[]; total: number } {
|
|
let filtered = events
|
|
|
|
if (opts?.since) {
|
|
filtered = filtered.filter(e => e.timestamp >= opts.since!)
|
|
}
|
|
if (opts?.mode) {
|
|
filtered = filtered.filter(e => e.mode === opts.mode)
|
|
}
|
|
|
|
const total = filtered.length
|
|
const offset = opts?.offset ?? 0
|
|
const limit = opts?.limit ?? 100
|
|
|
|
return {
|
|
events: filtered.slice(offset, offset + limit),
|
|
total,
|
|
}
|
|
}
|
|
|
|
// ── Query: Aggregated Stats ──────────────────────────────────────────
|
|
|
|
export interface MetadataStats {
|
|
total_requests: number
|
|
since: number | null // oldest event timestamp
|
|
uptime_ms: number
|
|
|
|
// Breakdown by mode
|
|
by_mode: Record<string, number>
|
|
by_tier: Record<string, number>
|
|
by_endpoint: Record<string, number>
|
|
|
|
// Model usage
|
|
models: {
|
|
total_queries: number // sum of all individual model calls
|
|
unique_models: number
|
|
by_model: Record<string, {
|
|
queries: number
|
|
wins: number
|
|
avg_score: number
|
|
avg_duration_ms: number
|
|
success_rate: number
|
|
}>
|
|
}
|
|
|
|
// Pipeline usage
|
|
pipeline: {
|
|
godmode_rate: number // % of requests with godmode enabled
|
|
autotune_rate: number
|
|
parseltongue_rate: number
|
|
stm_usage: Record<string, number> // module -> count
|
|
strategy_usage: Record<string, number>
|
|
}
|
|
|
|
// Context detection (from autotune)
|
|
contexts: Record<string, number>
|
|
|
|
// Latency
|
|
latency: {
|
|
avg_ms: number
|
|
p50_ms: number
|
|
p95_ms: number
|
|
p99_ms: number
|
|
}
|
|
|
|
// Response sizes
|
|
response_lengths: {
|
|
avg: number
|
|
p50: number
|
|
p95: number
|
|
}
|
|
|
|
// Streaming
|
|
streaming: {
|
|
stream_rate: number // % of requests that used streaming
|
|
avg_upgrades: number // avg leader changes per stream
|
|
avg_first_response_ms: number // avg time to first leader
|
|
}
|
|
|
|
// Errors
|
|
errors: {
|
|
total_model_errors: number
|
|
by_type: Record<string, number>
|
|
model_success_rate: number // across all individual model queries
|
|
}
|
|
}
|
|
|
|
export function getStats(): MetadataStats {
|
|
const total = events.length
|
|
|
|
if (total === 0) {
|
|
return {
|
|
total_requests: 0,
|
|
since: null,
|
|
uptime_ms: 0,
|
|
by_mode: {},
|
|
by_tier: {},
|
|
by_endpoint: {},
|
|
models: { total_queries: 0, unique_models: 0, by_model: {} },
|
|
pipeline: { godmode_rate: 0, autotune_rate: 0, parseltongue_rate: 0, stm_usage: {}, strategy_usage: {} },
|
|
contexts: {},
|
|
latency: { avg_ms: 0, p50_ms: 0, p95_ms: 0, p99_ms: 0 },
|
|
response_lengths: { avg: 0, p50: 0, p95: 0 },
|
|
streaming: { stream_rate: 0, avg_upgrades: 0, avg_first_response_ms: 0 },
|
|
errors: { total_model_errors: 0, by_type: {}, model_success_rate: 0 },
|
|
}
|
|
}
|
|
|
|
// Breakdowns
|
|
const byMode: Record<string, number> = {}
|
|
const byTier: Record<string, number> = {}
|
|
const byEndpoint: Record<string, number> = {}
|
|
const contexts: Record<string, number> = {}
|
|
const stmUsage: Record<string, number> = {}
|
|
const strategyUsage: Record<string, number> = {}
|
|
const errorTypes: Record<string, number> = {}
|
|
|
|
// Model tracking
|
|
const modelData: Record<string, { queries: number; wins: number; scores: number[]; durations: number[]; successes: number }> = {}
|
|
|
|
// Accumulators
|
|
let godmodeCount = 0
|
|
let autotuneCount = 0
|
|
let parseltongueCount = 0
|
|
let totalModelQueries = 0
|
|
let totalModelErrors = 0
|
|
let totalModelSuccesses = 0
|
|
const durations: number[] = []
|
|
const responseLengths: number[] = []
|
|
let streamCount = 0
|
|
let upgradeSum = 0
|
|
let firstResponseSum = 0
|
|
let streamWithLiquidCount = 0
|
|
|
|
for (const e of events) {
|
|
byMode[e.mode] = (byMode[e.mode] || 0) + 1
|
|
if (e.tier) byTier[e.tier] = (byTier[e.tier] || 0) + 1
|
|
byEndpoint[e.endpoint] = (byEndpoint[e.endpoint] || 0) + 1
|
|
|
|
if (e.pipeline.godmode) godmodeCount++
|
|
if (e.pipeline.autotune) autotuneCount++
|
|
if (e.pipeline.parseltongue) parseltongueCount++
|
|
if (e.pipeline.strategy) {
|
|
strategyUsage[e.pipeline.strategy] = (strategyUsage[e.pipeline.strategy] || 0) + 1
|
|
}
|
|
for (const mod of e.pipeline.stm_modules) {
|
|
stmUsage[mod] = (stmUsage[mod] || 0) + 1
|
|
}
|
|
|
|
if (e.autotune?.detected_context) {
|
|
const ctx = e.autotune.detected_context
|
|
contexts[ctx] = (contexts[ctx] || 0) + 1
|
|
}
|
|
|
|
durations.push(e.total_duration_ms)
|
|
responseLengths.push(e.response_length)
|
|
|
|
if (e.stream) streamCount++
|
|
|
|
if (e.liquid) {
|
|
upgradeSum += e.liquid.upgrades
|
|
firstResponseSum += e.liquid.first_response_ms
|
|
streamWithLiquidCount++
|
|
}
|
|
|
|
// Per-model results
|
|
if (e.model_results) {
|
|
for (const mr of e.model_results) {
|
|
totalModelQueries++
|
|
if (!modelData[mr.model]) {
|
|
modelData[mr.model] = { queries: 0, wins: 0, scores: [], durations: [], successes: 0 }
|
|
}
|
|
const md = modelData[mr.model]
|
|
md.queries++
|
|
if (mr.success) {
|
|
md.successes++
|
|
totalModelSuccesses++
|
|
md.scores.push(mr.score)
|
|
md.durations.push(mr.duration_ms)
|
|
} else {
|
|
totalModelErrors++
|
|
if (mr.error_type) {
|
|
errorTypes[mr.error_type] = (errorTypes[mr.error_type] || 0) + 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Winner tracking
|
|
if (e.winner) {
|
|
if (!modelData[e.winner.model]) {
|
|
modelData[e.winner.model] = { queries: 0, wins: 0, scores: [], durations: [], successes: 0 }
|
|
}
|
|
modelData[e.winner.model].wins++
|
|
}
|
|
}
|
|
|
|
// Build per-model stats
|
|
const byModel: Record<string, { queries: number; wins: number; avg_score: number; avg_duration_ms: number; success_rate: number }> = {}
|
|
for (const [model, data] of Object.entries(modelData)) {
|
|
byModel[model] = {
|
|
queries: data.queries,
|
|
wins: data.wins,
|
|
avg_score: data.scores.length > 0 ? Math.round(data.scores.reduce((a, b) => a + b, 0) / data.scores.length) : 0,
|
|
avg_duration_ms: data.durations.length > 0 ? Math.round(data.durations.reduce((a, b) => a + b, 0) / data.durations.length) : 0,
|
|
success_rate: data.queries > 0 ? Math.round((data.successes / data.queries) * 100) / 100 : 0,
|
|
}
|
|
}
|
|
|
|
return {
|
|
total_requests: total,
|
|
since: events[0].timestamp,
|
|
uptime_ms: Date.now() - events[0].timestamp,
|
|
by_mode: byMode,
|
|
by_tier: byTier,
|
|
by_endpoint: byEndpoint,
|
|
models: {
|
|
total_queries: totalModelQueries,
|
|
unique_models: Object.keys(modelData).length,
|
|
by_model: byModel,
|
|
},
|
|
pipeline: {
|
|
godmode_rate: Math.round((godmodeCount / total) * 100) / 100,
|
|
autotune_rate: Math.round((autotuneCount / total) * 100) / 100,
|
|
parseltongue_rate: Math.round((parseltongueCount / total) * 100) / 100,
|
|
stm_usage: stmUsage,
|
|
strategy_usage: strategyUsage,
|
|
},
|
|
contexts,
|
|
latency: {
|
|
avg_ms: Math.round(durations.reduce((a, b) => a + b, 0) / durations.length),
|
|
p50_ms: percentile(durations, 50),
|
|
p95_ms: percentile(durations, 95),
|
|
p99_ms: percentile(durations, 99),
|
|
},
|
|
response_lengths: {
|
|
avg: Math.round(responseLengths.reduce((a, b) => a + b, 0) / responseLengths.length),
|
|
p50: percentile(responseLengths, 50),
|
|
p95: percentile(responseLengths, 95),
|
|
},
|
|
streaming: {
|
|
stream_rate: Math.round((streamCount / total) * 100) / 100,
|
|
avg_upgrades: streamWithLiquidCount > 0 ? Math.round((upgradeSum / streamWithLiquidCount) * 100) / 100 : 0,
|
|
avg_first_response_ms: streamWithLiquidCount > 0 ? Math.round(firstResponseSum / streamWithLiquidCount) : 0,
|
|
},
|
|
errors: {
|
|
total_model_errors: totalModelErrors,
|
|
by_type: errorTypes,
|
|
model_success_rate: totalModelQueries > 0 ? Math.round((totalModelSuccesses / totalModelQueries) * 100) / 100 : 0,
|
|
},
|
|
}
|
|
}
|
|
|
|
// ── Helpers ──────────────────────────────────────────────────────────
|
|
|
|
function percentile(arr: number[], p: number): number {
|
|
if (arr.length === 0) return 0
|
|
const sorted = [...arr].sort((a, b) => a - b)
|
|
const idx = Math.ceil((p / 100) * sorted.length) - 1
|
|
return sorted[Math.max(0, idx)]
|
|
}
|