mirror of
https://github.com/elder-plinius/G0DM0D3.git
synced 2026-11-16 07:02:25 +00:00
232 lines
7.7 KiB
TypeScript
232 lines
7.7 KiB
TypeScript
/**
|
|
* Open Dataset Collection Engine
|
|
*
|
|
* Opt-in data collection for building an open source research dataset.
|
|
* Researchers who enable `contribute_to_dataset: true` in their requests
|
|
* have their (anonymized) interaction data stored for the community.
|
|
*
|
|
* Stored data:
|
|
* - Messages sent and received (no API keys, no IPs)
|
|
* - AutoTune parameters and context detection results
|
|
* - Model used and response metadata
|
|
* - User feedback/ratings
|
|
* - Parseltongue and STM pipeline metadata
|
|
*
|
|
* Privacy guarantees:
|
|
* - Strictly opt-in per request
|
|
* - No PII: API keys, IPs, and auth tokens are NEVER stored
|
|
* - Dataset is exportable via GET /v1/dataset/export
|
|
* - Caller can request deletion via DELETE /v1/dataset/:id
|
|
*
|
|
* Persistence: auto-publishes to HuggingFace when buffer fills up.
|
|
*/
|
|
|
|
import { randomUUID } from 'crypto'
|
|
import { registerDatasetStore, checkDatasetThreshold } from './hf-publisher'
|
|
|
|
// ── Types ────────────────────────────────────────────────────────────
|
|
|
|
export interface DatasetEntry {
|
|
id: string
|
|
timestamp: number
|
|
|
|
// Request metadata
|
|
endpoint: string // which API endpoint was called
|
|
model: string
|
|
mode: 'standard' | 'ultraplinian'
|
|
|
|
// Messages (stripped of system prompts to avoid leaking custom prompts)
|
|
messages: Array<{ role: string; content: string }>
|
|
response: string
|
|
|
|
// AutoTune data
|
|
autotune?: {
|
|
strategy: string
|
|
detected_context: string
|
|
confidence: number
|
|
params: Record<string, number>
|
|
reasoning: string
|
|
}
|
|
|
|
// Parseltongue data
|
|
parseltongue?: {
|
|
triggers_found: string[]
|
|
technique_used: string
|
|
transformations_count: number
|
|
}
|
|
|
|
// STM data
|
|
stm?: {
|
|
modules_applied: string[]
|
|
}
|
|
|
|
// ULTRAPLINIAN race data
|
|
ultraplinian?: {
|
|
tier: string
|
|
models_queried: string[]
|
|
winner_model: string
|
|
all_scores: Array<{ model: string; score: number; duration_ms: number; success: boolean }>
|
|
total_duration_ms: number
|
|
}
|
|
|
|
// Feedback (added later via POST /v1/feedback if user rates)
|
|
feedback?: {
|
|
rating: 1 | -1
|
|
heuristics?: {
|
|
response_length: number
|
|
repetition_score: number
|
|
vocabulary_diversity: number
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── In-Memory Store with Auto-Publish ────────────────────────────────
|
|
// Buffer auto-flushes to HuggingFace when it hits 80% capacity.
|
|
// Falls back to FIFO eviction if HF publishing is not configured.
|
|
|
|
let dataset: DatasetEntry[] = []
|
|
const MAX_ENTRIES = 10000 // Cap to prevent unbounded memory growth
|
|
|
|
// Track how many entries have been flushed so we can use index-based draining
|
|
// instead of copying the entire array on each snapshot.
|
|
let datasetFlushIndex = 0
|
|
|
|
// Register with HF publisher so it can snapshot/clear our buffer
|
|
registerDatasetStore({
|
|
snapshot: () => dataset.slice(datasetFlushIndex),
|
|
clear: (count: number) => {
|
|
datasetFlushIndex += count
|
|
// Compact the array when more than half has been drained to free memory
|
|
if (datasetFlushIndex > dataset.length / 2) {
|
|
dataset = dataset.slice(datasetFlushIndex)
|
|
datasetFlushIndex = 0
|
|
}
|
|
},
|
|
})
|
|
|
|
// ── PII Scrubber ────────────────────────────────────────────────────
|
|
// Best-effort removal of common PII patterns before data hits the buffer.
|
|
// Not perfect — users are warned to avoid submitting PII regardless.
|
|
|
|
const PII_PATTERNS: Array<{ pattern: RegExp; replacement: string }> = [
|
|
// Email addresses
|
|
{ pattern: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g, replacement: '[EMAIL_REDACTED]' },
|
|
// Phone numbers (US and international formats)
|
|
{ pattern: /(?:\+?1[-.\s]?)?\(?[0-9]{3}\)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{4}/g, replacement: '[PHONE_REDACTED]' },
|
|
{ pattern: /\+[0-9]{1,3}[-.\s]?[0-9]{4,14}/g, replacement: '[PHONE_REDACTED]' },
|
|
// SSN (US)
|
|
{ pattern: /\b[0-9]{3}[-\s]?[0-9]{2}[-\s]?[0-9]{4}\b/g, replacement: '[SSN_REDACTED]' },
|
|
// Credit card numbers (13-19 digits with optional separators)
|
|
{ pattern: /\b(?:[0-9]{4}[-\s]?){3,4}[0-9]{1,4}\b/g, replacement: '[CC_REDACTED]' },
|
|
// IPv4 addresses
|
|
{ pattern: /\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b/g, replacement: '[IP_REDACTED]' },
|
|
// IPv6 addresses (simplified)
|
|
{ pattern: /\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b/g, replacement: '[IP_REDACTED]' },
|
|
// API keys (common patterns: sk-, pk-, key-, bearer tokens)
|
|
{ pattern: /\b(?:sk|pk|api[_-]?key)[_-][a-zA-Z0-9]{20,}\b/gi, replacement: '[APIKEY_REDACTED]' },
|
|
{ pattern: /\bBearer\s+[a-zA-Z0-9._-]{20,}\b/g, replacement: 'Bearer [TOKEN_REDACTED]' },
|
|
// AWS access keys
|
|
{ pattern: /\bAKIA[0-9A-Z]{16}\b/g, replacement: '[AWS_KEY_REDACTED]' },
|
|
]
|
|
|
|
function scrubPII(text: string): string {
|
|
let result = text
|
|
for (const { pattern, replacement } of PII_PATTERNS) {
|
|
// Reset lastIndex for global regexes
|
|
pattern.lastIndex = 0
|
|
result = result.replace(pattern, replacement)
|
|
}
|
|
return result
|
|
}
|
|
|
|
function scrubEntryPII(entry: Omit<DatasetEntry, 'id' | 'timestamp'>): Omit<DatasetEntry, 'id' | 'timestamp'> {
|
|
return {
|
|
...entry,
|
|
messages: entry.messages.map(m => ({
|
|
...m,
|
|
content: typeof m.content === 'string' ? scrubPII(m.content) : m.content,
|
|
})),
|
|
response: scrubPII(entry.response),
|
|
}
|
|
}
|
|
|
|
// ── Public API ───────────────────────────────────────────────────────
|
|
|
|
export function addEntry(entry: Omit<DatasetEntry, 'id' | 'timestamp'>): string {
|
|
const id = randomUUID()
|
|
const scrubbed = scrubEntryPII(entry)
|
|
const record: DatasetEntry = {
|
|
...scrubbed,
|
|
id,
|
|
timestamp: Date.now(),
|
|
}
|
|
|
|
dataset.push(record)
|
|
|
|
// Auto-flush to HF when approaching capacity (async, non-blocking)
|
|
checkDatasetThreshold(dataset.length, MAX_ENTRIES)
|
|
|
|
// Evict oldest entries if over cap (fallback if HF not configured or failed)
|
|
if (dataset.length > MAX_ENTRIES) {
|
|
dataset = dataset.slice(dataset.length - MAX_ENTRIES)
|
|
}
|
|
|
|
return id
|
|
}
|
|
|
|
export function addFeedbackToEntry(
|
|
entryId: string,
|
|
feedback: DatasetEntry['feedback'],
|
|
): boolean {
|
|
const entry = dataset.find(e => e.id === entryId)
|
|
if (!entry) return false
|
|
entry.feedback = feedback
|
|
return true
|
|
}
|
|
|
|
export function deleteEntry(id: string): boolean {
|
|
const idx = dataset.findIndex(e => e.id === id)
|
|
if (idx === -1) return false
|
|
dataset.splice(idx, 1)
|
|
return true
|
|
}
|
|
|
|
export function getDataset(): DatasetEntry[] {
|
|
return [...dataset]
|
|
}
|
|
|
|
export function getDatasetStats(): {
|
|
total_entries: number
|
|
entries_with_feedback: number
|
|
mode_breakdown: Record<string, number>
|
|
model_breakdown: Record<string, number>
|
|
context_breakdown: Record<string, number>
|
|
oldest_entry: number | null
|
|
newest_entry: number | null
|
|
} {
|
|
const modeBreakdown: Record<string, number> = {}
|
|
const modelBreakdown: Record<string, number> = {}
|
|
const contextBreakdown: Record<string, number> = {}
|
|
let withFeedback = 0
|
|
|
|
for (const entry of dataset) {
|
|
modeBreakdown[entry.mode] = (modeBreakdown[entry.mode] || 0) + 1
|
|
modelBreakdown[entry.model] = (modelBreakdown[entry.model] || 0) + 1
|
|
if (entry.autotune?.detected_context) {
|
|
const ctx = entry.autotune.detected_context
|
|
contextBreakdown[ctx] = (contextBreakdown[ctx] || 0) + 1
|
|
}
|
|
if (entry.feedback) withFeedback++
|
|
}
|
|
|
|
return {
|
|
total_entries: dataset.length,
|
|
entries_with_feedback: withFeedback,
|
|
mode_breakdown: modeBreakdown,
|
|
model_breakdown: modelBreakdown,
|
|
context_breakdown: contextBreakdown,
|
|
oldest_entry: dataset.length > 0 ? dataset[0].timestamp : null,
|
|
newest_entry: dataset.length > 0 ? dataset[dataset.length - 1].timestamp : null,
|
|
}
|
|
}
|