Files
G0DM0D3/api/routes/research.ts
2026-03-25 10:44:35 -07:00

278 lines
10 KiB
TypeScript

/**
* Research API Routes
*
* Read-access endpoints for the pliny-the-prompter/g0dm0d3 HuggingFace
* dataset. Lets researchers query, filter, and download the full
* published corpus — not just the current in-memory buffer.
*
* GET /v1/research/info — Dataset repo info + schema
* GET /v1/research/stats — Aggregate stats across all published batches
* GET /v1/research/batches — List all published batch files
* GET /v1/research/batch/:path — Read a single batch file
* GET /v1/research/query — Query the full corpus with filters
* POST /v1/research/flush — Force-flush in-memory buffers to HF
* GET /v1/research/download — Download the full corpus as JSONL
*/
import { Router } from 'express'
import {
isReaderEnabled,
listBatchFiles,
readBatch,
queryCorpus,
getCorpusStats,
flushMetadata,
flushDataset,
} from '../lib/hf-reader'
import { getDataset, getDatasetStats } from '../lib/dataset'
import { getStats as getMetadataStats } from '../lib/metadata'
import { getPublisherStatus } from '../lib/hf-publisher'
import { tierGate } from '../middleware/tierGate'
export const researchRoutes = Router()
let lastFlushTime = 0
// ── Info: dataset schema, repo, capabilities ─────────────────────────
researchRoutes.get('/info', (_req, res) => {
res.json({
name: 'G0DM0D3 Research Dataset',
repo: 'pliny-the-prompter/g0dm0d3',
repo_url: 'https://huggingface.co/datasets/pliny-the-prompter/g0dm0d3',
license: 'AGPL-3.0',
description: 'Open research dataset for LLM robustness and safety evaluation. Contains opt-in interaction data with full pipeline metadata from ULTRAPLINIAN multi-model racing, AutoTune context-adaptive parameter tuning, Parseltongue obfuscation, and STM output normalization.',
hf_reader_enabled: isReaderEnabled(),
publisher: getPublisherStatus(),
schema: {
dataset_entry: {
id: 'string (UUID)',
timestamp: 'number (unix ms)',
endpoint: 'string',
model: 'string (OpenRouter model ID)',
mode: '"standard" | "ultraplinian"',
messages: 'Array<{ role: string, content: string }>',
response: 'string',
autotune: '{ strategy, detected_context, confidence, params, reasoning } | null',
parseltongue: '{ triggers_found, technique_used, transformations_count } | null',
stm: '{ modules_applied: string[] } | null',
ultraplinian: '{ tier, models_queried, winner_model, all_scores, total_duration_ms } | null',
feedback: '{ rating: 1|-1, heuristics } | null',
},
metadata_event: {
id: 'string (UUID)',
timestamp: 'number (unix ms)',
endpoint: 'string',
mode: '"standard" | "ultraplinian"',
tier: 'string | null',
stream: 'boolean',
pipeline: '{ godmode, autotune, parseltongue, stm_modules, strategy }',
autotune: '{ detected_context, confidence } | null',
model_results: 'Array<{ model, score, duration_ms, success, content_length, error_type }> | null',
winner: '{ model, score, duration_ms, content_length } | null',
total_duration_ms: 'number',
response_length: 'number',
liquid: '{ upgrades, first_response_ms } | null',
},
},
endpoints: {
'GET /v1/research/info': 'This endpoint — dataset schema and repo info',
'GET /v1/research/stats': 'Aggregate stats across all published HF batches',
'GET /v1/research/batches': 'List all published batch files in the HF repo',
'GET /v1/research/batch/*': 'Read a specific batch file (JSONL parsed)',
'GET /v1/research/query': 'Query the full corpus with filters (category, model, mode, time range)',
'POST /v1/research/flush': 'Force-flush in-memory buffers to HuggingFace',
'GET /v1/research/download': 'Download full corpus as streaming JSONL',
'GET /v1/research/combined-stats': 'Combined view: in-memory + published stats',
},
})
})
// ── Stats: aggregate across all published batches ────────────────────
researchRoutes.get('/stats', async (_req, res) => {
try {
const stats = await getCorpusStats()
res.json(stats)
} catch (err) {
console.error('[research]', err)
res.status(500).json({ error: 'Failed to fetch corpus stats' })
}
})
// ── Combined Stats: in-memory + published ────────────────────────────
researchRoutes.get('/combined-stats', async (_req, res) => {
try {
const [corpusStats, datasetStats, metadataStats] = await Promise.all([
getCorpusStats(),
Promise.resolve(getDatasetStats()),
Promise.resolve(getMetadataStats()),
])
res.json({
published: corpusStats,
in_memory: {
dataset: datasetStats,
metadata: {
total_requests: metadataStats.total_requests,
by_mode: metadataStats.by_mode,
models: metadataStats.models,
latency: metadataStats.latency,
errors: metadataStats.errors,
},
},
})
} catch (err) {
console.error('[research]', err)
res.status(500).json({ error: 'Failed to fetch combined stats' })
}
})
// ── Batches: list all published files ────────────────────────────────
researchRoutes.get('/batches', async (req, res) => {
try {
const refresh = req.query.refresh === 'true'
const category = req.query.category as 'metadata' | 'dataset' | undefined
let batches = await listBatchFiles(refresh)
if (category) {
batches = batches.filter(b => b.category === category)
}
res.json({
batches,
total: batches.length,
hf_reader_enabled: isReaderEnabled(),
})
} catch (err) {
console.error('[research]', err)
res.status(500).json({ error: 'Failed to list batches' })
}
})
// ── Batch: read a single file ────────────────────────────────────────
// Path comes as wildcard: /v1/research/batch/metadata/batch_2024-01-01_0001.jsonl
researchRoutes.get('/batch/*', async (req, res) => {
try {
const filePath = req.params[0]
if (!filePath || !filePath.endsWith('.jsonl')) {
res.status(400).json({ error: 'Invalid batch path. Expected: metadata/batch_*.jsonl or dataset/batch_*.jsonl' })
return
}
if (!/^(metadata|dataset)\/batch_[\w.-]+\.jsonl$/.test(filePath)) {
res.status(400).json({ error: 'Invalid batch path format' })
return
}
const records = await readBatch(filePath)
res.json({
path: filePath,
records,
count: records.length,
})
} catch (err) {
console.error('[research]', err)
res.status(500).json({ error: 'Failed to read batch' })
}
})
// ── Query: search the full corpus ────────────────────────────────────
researchRoutes.get('/query', tierGate('research:full'), async (req, res) => {
try {
const query = {
category: req.query.category as 'metadata' | 'dataset' | undefined,
since: req.query.since ? parseInt(String(req.query.since)) : undefined,
until: req.query.until ? parseInt(String(req.query.until)) : undefined,
model: req.query.model as string | undefined,
mode: req.query.mode as 'standard' | 'ultraplinian' | undefined,
limit: Math.min(parseInt(String(req.query.limit)) || 100, 1000),
offset: parseInt(String(req.query.offset)) || 0,
}
const result = await queryCorpus(query)
res.json({
...result,
query,
has_more: result.truncated,
})
} catch (err) {
console.error('[research]', err)
res.status(500).json({ error: 'Failed to query corpus' })
}
})
// ── Flush: force-push in-memory buffers to HF ───────────────────────
researchRoutes.post('/flush', tierGate('flush'), async (_req, res) => {
if (Date.now() - lastFlushTime < 300_000) {
res.status(429).json({ error: 'Flush cooldown: please wait 5 minutes between flushes' })
return
}
try {
lastFlushTime = Date.now()
const [meta, data] = await Promise.all([
flushMetadata(),
flushDataset(),
])
res.json({
metadata: meta,
dataset: data,
message: meta.success || data.success
? `Flushed ${meta.flushed} metadata events + ${data.flushed} dataset entries to HuggingFace`
: 'Nothing to flush (buffers empty or HF publishing not configured)',
})
} catch (err) {
console.error('[research]', err)
res.status(500).json({ error: 'Flush failed' })
}
})
// ── Download: stream the full corpus as JSONL ────────────────────────
researchRoutes.get('/download', tierGate('corpus:download'), async (req, res) => {
try {
const category = req.query.category as 'metadata' | 'dataset' | undefined
const includeMemory = req.query.include_memory !== 'false' // default: true
res.setHeader('Content-Type', 'application/x-ndjson')
res.setHeader('Content-Disposition', `attachment; filename="g0dm0d3-corpus-${Date.now()}.jsonl"`)
// First: stream published batches from HF
const batches = await listBatchFiles()
const targetBatches = category ? batches.filter(b => b.category === category) : batches
for (const batch of targetBatches) {
const records = await readBatch(batch.path)
for (const record of records) {
res.write(JSON.stringify(record) + '\n')
}
}
// Then: append current in-memory data (not yet published)
if (includeMemory) {
const memoryDataset = getDataset()
for (const entry of memoryDataset) {
res.write(JSON.stringify({ ...entry, _source: 'memory' }) + '\n')
}
}
res.end()
} catch (err) {
// If headers already sent, can't send JSON error
console.error('[research]', err)
if (!res.headersSent) {
res.status(500).json({ error: 'Download failed' })
} else {
res.end()
}
}
})