/** * Open Dataset Collection Engine * * Opt-in data collection for building an open source research dataset. * Researchers who enable `contribute_to_dataset: true` in their requests * have their (anonymized) interaction data stored for the community. * * Stored data: * - Messages sent and received (no API keys, no IPs) * - AutoTune parameters and context detection results * - Model used and response metadata * - User feedback/ratings * - Parseltongue and STM pipeline metadata * * Privacy guarantees: * - Strictly opt-in per request * - No PII: API keys, IPs, and auth tokens are NEVER stored * - Dataset is exportable via GET /v1/dataset/export * - Caller can request deletion via DELETE /v1/dataset/:id * * Persistence: auto-publishes to HuggingFace when buffer fills up. */ import { randomUUID } from 'crypto' import { registerDatasetStore, checkDatasetThreshold } from './hf-publisher' // ── Types ──────────────────────────────────────────────────────────── export interface DatasetEntry { id: string timestamp: number // Request metadata endpoint: string // which API endpoint was called model: string mode: 'standard' | 'ultraplinian' // Messages (stripped of system prompts to avoid leaking custom prompts) messages: Array<{ role: string; content: string }> response: string // AutoTune data autotune?: { strategy: string detected_context: string confidence: number params: Record reasoning: string } // Parseltongue data parseltongue?: { triggers_found: string[] technique_used: string transformations_count: number } // STM data stm?: { modules_applied: string[] } // ULTRAPLINIAN race data ultraplinian?: { tier: string models_queried: string[] winner_model: string all_scores: Array<{ model: string; score: number; duration_ms: number; success: boolean }> total_duration_ms: number } // Feedback (added later via POST /v1/feedback if user rates) feedback?: { rating: 1 | -1 heuristics?: { response_length: number repetition_score: number vocabulary_diversity: number } } } // ── In-Memory Store with Auto-Publish ──────────────────────────────── // Buffer auto-flushes to HuggingFace when it hits 80% capacity. // Falls back to FIFO eviction if HF publishing is not configured. let dataset: DatasetEntry[] = [] const MAX_ENTRIES = 10000 // Cap to prevent unbounded memory growth // Track how many entries have been flushed so we can use index-based draining // instead of copying the entire array on each snapshot. let datasetFlushIndex = 0 // Register with HF publisher so it can snapshot/clear our buffer registerDatasetStore({ snapshot: () => dataset.slice(datasetFlushIndex), clear: (count: number) => { datasetFlushIndex += count // Compact the array when more than half has been drained to free memory if (datasetFlushIndex > dataset.length / 2) { dataset = dataset.slice(datasetFlushIndex) datasetFlushIndex = 0 } }, }) // ── PII Scrubber ──────────────────────────────────────────────────── // Best-effort removal of common PII patterns before data hits the buffer. // Not perfect — users are warned to avoid submitting PII regardless. const PII_PATTERNS: Array<{ pattern: RegExp; replacement: string }> = [ // Email addresses { pattern: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g, replacement: '[EMAIL_REDACTED]' }, // Phone numbers (US and international formats) { pattern: /(?:\+?1[-.\s]?)?\(?[0-9]{3}\)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{4}/g, replacement: '[PHONE_REDACTED]' }, { pattern: /\+[0-9]{1,3}[-.\s]?[0-9]{4,14}/g, replacement: '[PHONE_REDACTED]' }, // SSN (US) { pattern: /\b[0-9]{3}[-\s]?[0-9]{2}[-\s]?[0-9]{4}\b/g, replacement: '[SSN_REDACTED]' }, // Credit card numbers (13-19 digits with optional separators) { pattern: /\b(?:[0-9]{4}[-\s]?){3,4}[0-9]{1,4}\b/g, replacement: '[CC_REDACTED]' }, // IPv4 addresses { pattern: /\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b/g, replacement: '[IP_REDACTED]' }, // IPv6 addresses (simplified) { pattern: /\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b/g, replacement: '[IP_REDACTED]' }, // API keys (common patterns: sk-, pk-, key-, bearer tokens) { pattern: /\b(?:sk|pk|api[_-]?key)[_-][a-zA-Z0-9]{20,}\b/gi, replacement: '[APIKEY_REDACTED]' }, { pattern: /\bBearer\s+[a-zA-Z0-9._-]{20,}\b/g, replacement: 'Bearer [TOKEN_REDACTED]' }, // AWS access keys { pattern: /\bAKIA[0-9A-Z]{16}\b/g, replacement: '[AWS_KEY_REDACTED]' }, ] function scrubPII(text: string): string { let result = text for (const { pattern, replacement } of PII_PATTERNS) { // Reset lastIndex for global regexes pattern.lastIndex = 0 result = result.replace(pattern, replacement) } return result } function scrubEntryPII(entry: Omit): Omit { return { ...entry, messages: entry.messages.map(m => ({ ...m, content: typeof m.content === 'string' ? scrubPII(m.content) : m.content, })), response: scrubPII(entry.response), } } // ── Public API ─────────────────────────────────────────────────────── export function addEntry(entry: Omit): string { const id = randomUUID() const scrubbed = scrubEntryPII(entry) const record: DatasetEntry = { ...scrubbed, id, timestamp: Date.now(), } dataset.push(record) // Auto-flush to HF when approaching capacity (async, non-blocking) checkDatasetThreshold(dataset.length, MAX_ENTRIES) // Evict oldest entries if over cap (fallback if HF not configured or failed) if (dataset.length > MAX_ENTRIES) { dataset = dataset.slice(dataset.length - MAX_ENTRIES) } return id } export function addFeedbackToEntry( entryId: string, feedback: DatasetEntry['feedback'], ): boolean { const entry = dataset.find(e => e.id === entryId) if (!entry) return false entry.feedback = feedback return true } export function deleteEntry(id: string): boolean { const idx = dataset.findIndex(e => e.id === id) if (idx === -1) return false dataset.splice(idx, 1) return true } export function getDataset(): DatasetEntry[] { return [...dataset] } export function getDatasetStats(): { total_entries: number entries_with_feedback: number mode_breakdown: Record model_breakdown: Record context_breakdown: Record oldest_entry: number | null newest_entry: number | null } { const modeBreakdown: Record = {} const modelBreakdown: Record = {} const contextBreakdown: Record = {} let withFeedback = 0 for (const entry of dataset) { modeBreakdown[entry.mode] = (modeBreakdown[entry.mode] || 0) + 1 modelBreakdown[entry.model] = (modelBreakdown[entry.model] || 0) + 1 if (entry.autotune?.detected_context) { const ctx = entry.autotune.detected_context contextBreakdown[ctx] = (contextBreakdown[ctx] || 0) + 1 } if (entry.feedback) withFeedback++ } return { total_entries: dataset.length, entries_with_feedback: withFeedback, mode_breakdown: modeBreakdown, model_breakdown: modelBreakdown, context_breakdown: contextBreakdown, oldest_entry: dataset.length > 0 ? dataset[0].timestamp : null, newest_entry: dataset.length > 0 ? dataset[dataset.length - 1].timestamp : null, } }