Files
G0DM0D3/api/lib/consortium.ts
2026-03-25 10:44:35 -07:00

362 lines
12 KiB
TypeScript

/**
* CONSORTIUM Engine
*
* The hive-mind mode of G0DM0D3. Instead of racing models to pick a single winner
* (ULTRAPLINIAN), CONSORTIUM collects ALL responses and feeds them to a strong
* orchestrator model that synthesizes ground truth from the collective intelligence.
*
* Architecture:
* 1. COLLECTION — Query N models in parallel (reuses ULTRAPLINIAN racing infra)
* 2. ANALYSIS — Score each response + identify consensus/disagreements
* 3. SYNTHESIS — Orchestrator model distills ground truth from all responses
* 4. RESPONSE — Return synthesized answer + full provenance metadata
*
* Key difference from ULTRAPLINIAN:
* ULTRAPLINIAN picks the BEST single voice.
* CONSORTIUM distills GROUND TRUTH from the crowd.
*/
import { queryModel, type ModelResult, type SpeedTier, getModelsForTier, scoreResponse } from './ultraplinian'
// ── Orchestrator Models ─────────────────────────────────────────────
/** Strong reasoning models suitable for synthesis. First is default. */
export const ORCHESTRATOR_MODELS = [
'anthropic/claude-sonnet-4.6',
'openai/gpt-5.3-chat',
'google/gemini-3-pro-preview',
'x-ai/grok-4',
'anthropic/claude-opus-4.6',
'deepseek/deepseek-v3.2',
] as const
export type OrchestratorModel = typeof ORCHESTRATOR_MODELS[number] | (string & {})
// ── Consortium Types ────────────────────────────────────────────────
export interface ConsortiumResponse {
model: string
content: string
score: number
duration_ms: number
success: boolean
error?: string
}
export interface ConsensusPoint {
claim: string
supporting_models: string[]
confidence: 'high' | 'medium' | 'low'
}
export interface DisagreementPoint {
topic: string
positions: Array<{ model: string; stance: string }>
}
export interface ConsortiumResult {
/** The synthesized ground-truth response */
synthesis: string
/** Which orchestrator model produced the synthesis */
orchestrator_model: string
/** How long the orchestrator took (ms) */
orchestrator_duration_ms: number
/** All model responses that were collected */
responses: ConsortiumResponse[]
/** Collection phase stats */
collection: {
models_queried: number
models_succeeded: number
total_duration_ms: number
}
}
// ── Collection Config ───────────────────────────────────────────────
interface CollectionConfig {
/** Wait for at least this many successful responses (default: 3) */
minResponses?: number
/** Max time to wait for collection (default: 60000ms — longer than ULTRAPLINIAN) */
hardTimeout?: number
/** Called as each model finishes */
onModelResult?: (result: ModelResult, collected: number, total: number) => void
}
// ── Orchestrator System Prompt ──────────────────────────────────────
export const CONSORTIUM_SYSTEM_PROMPT = `You are the CONSORTIUM ORCHESTRATOR — an impartial, analytical intelligence tasked with synthesizing ground truth from multiple AI model responses.
Your role is that of a level-headed tastemaker and truth-distiller. You are NOT another chatbot. You are a meta-cognitive layer that operates ABOVE the individual model responses.
## YOUR PROCESS
1. **READ** all provided model responses carefully
2. **IDENTIFY CONSENSUS** — claims/facts that multiple models agree on (these have highest confidence)
3. **FLAG CONTRADICTIONS** — where models disagree, evaluate which position has stronger reasoning
4. **SYNTHESIZE** — produce a single, authoritative response that represents the best collective understanding
5. **BE DIRECT** — your output IS the final answer the user sees. No meta-commentary about the process.
## RULES
- **Ground truth over popularity**: If one model provides a well-reasoned minority position with evidence, it can override a poorly-reasoned majority
- **Specificity wins**: Prefer concrete details, specific numbers, working code over vague generalities
- **No hedging**: You are synthesizing from N expert opinions. The result should be MORE confident, not less
- **Completeness**: Your synthesis should be AT LEAST as comprehensive as the best individual response
- **Attribution-free**: The user should NOT see model names or "according to model X" — just the unified truth
- **Preserve quality**: If one model has excellent code/examples, incorporate them directly
- **Structure**: Use headers, lists, and code blocks for organization. Match the best structural elements from the inputs.
## OUTPUT FORMAT
Write your synthesized response directly. Do NOT include meta-commentary like "Based on the responses..." or "The models agree that..." — just write the definitive answer as if you are the world's foremost authority on the topic.`
// ── Build Orchestrator Prompt ───────────────────────────────────────
function buildOrchestrationPrompt(
userQuery: string,
responses: ConsortiumResponse[],
): string {
const successfulResponses = responses
.filter(r => r.success && r.content)
.sort((a, b) => b.score - a.score)
let prompt = `## USER'S ORIGINAL QUESTION\n\n${userQuery}\n\n`
prompt += `## MODEL RESPONSES (${successfulResponses.length} collected)\n\n`
prompt += `Each response below is from a different AI model, scored 0-100 on substance/directness.\n\n`
for (let i = 0; i < successfulResponses.length; i++) {
const r = successfulResponses[i]
prompt += `---\n`
prompt += `### Response ${i + 1} (Score: ${r.score}/100, ${r.duration_ms}ms)\n\n`
prompt += `${r.content}\n\n`
}
prompt += `---\n\n`
prompt += `## YOUR TASK\n\n`
prompt += `Synthesize the above ${successfulResponses.length} responses into a single, definitive answer. `
prompt += `Identify what the models agree on (high confidence), resolve any contradictions, `
prompt += `and produce the most complete, accurate, and direct response possible. `
prompt += `Your synthesis should be BETTER than any individual response.`
return prompt
}
// ── Collect All Responses ───────────────────────────────────────────
/**
* Query all models and collect responses. Unlike ULTRAPLINIAN's race-and-exit,
* CONSORTIUM waits for ALL models (up to the hard timeout) because every
* voice matters for consensus.
*/
export function collectAllResponses(
models: string[],
messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
apiKey: string,
params: {
temperature?: number
max_tokens?: number
top_p?: number
top_k?: number
frequency_penalty?: number
presence_penalty?: number
repetition_penalty?: number
},
config: CollectionConfig = {},
): Promise<ModelResult[]> {
const minResponses = config.minResponses ?? 3
const hardTimeout = config.hardTimeout ?? 60000
return new Promise(resolve => {
const results: ModelResult[] = []
let successCount = 0
let settled = 0
let resolved = false
const controller = new AbortController()
const finish = () => {
if (resolved) return
resolved = true
controller.abort()
if (hardTimer) clearTimeout(hardTimer)
resolve(results)
}
// Hard timeout — collect what we have
const hardTimer = setTimeout(finish, hardTimeout)
// Fire all queries
for (const model of models) {
queryModel(model, messages, apiKey, params, controller.signal)
.then(result => {
if (resolved) return
results.push(result)
settled++
if (result.success) successCount++
if (config.onModelResult) {
try { config.onModelResult(result, settled, models.length) } catch {}
}
// All models done
if (settled === models.length) {
finish()
}
})
.catch((err: any) => {
if (resolved) return
const errorResult: ModelResult = {
model,
content: '',
duration_ms: 0,
success: false,
error: err.message || 'Unknown error',
score: 0,
}
results.push(errorResult)
settled++
if (config.onModelResult) {
try { config.onModelResult(errorResult, settled, models.length) } catch {}
}
if (settled === models.length) {
finish()
}
})
}
// Edge case: no models
if (models.length === 0) finish()
// Safety: if we have enough after a generous wait, don't block forever
// on one slow model. After 80% of hardTimeout, if we have minResponses, finish.
setTimeout(() => {
if (!resolved && successCount >= minResponses) {
finish()
}
}, hardTimeout * 0.8)
})
}
// ── Run Orchestrator ────────────────────────────────────────────────
/**
* Feed collected responses to the orchestrator model for synthesis.
*/
export async function synthesize(
userQuery: string,
responses: ConsortiumResponse[],
apiKey: string,
orchestratorModel: OrchestratorModel = ORCHESTRATOR_MODELS[0],
maxTokens: number = 8192,
): Promise<{ synthesis: string; duration_ms: number; model: string }> {
const orchestrationPrompt = buildOrchestrationPrompt(userQuery, responses)
const messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }> = [
{ role: 'system', content: CONSORTIUM_SYSTEM_PROMPT },
{ role: 'user', content: orchestrationPrompt },
]
const result = await queryModel(
orchestratorModel,
messages,
apiKey,
{ temperature: 0.3, max_tokens: maxTokens }, // Low temp for analytical synthesis
)
if (!result.success || !result.content) {
throw new Error(`Orchestrator (${orchestratorModel}) failed: ${result.error || 'empty response'}`)
}
return {
synthesis: result.content,
duration_ms: result.duration_ms,
model: orchestratorModel,
}
}
// ── Full Consortium Pipeline ────────────────────────────────────────
export interface ConsortiumPipelineConfig {
tier: SpeedTier
orchestratorModel?: OrchestratorModel
maxTokens?: number
collectionConfig?: CollectionConfig
}
/**
* Run the full CONSORTIUM pipeline:
* 1. Collect all model responses
* 2. Score them
* 3. Feed to orchestrator for synthesis
* 4. Return the synthesized ground truth
*/
export async function runConsortium(
userQuery: string,
messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
apiKey: string,
params: {
temperature?: number
max_tokens?: number
top_p?: number
top_k?: number
frequency_penalty?: number
presence_penalty?: number
repetition_penalty?: number
},
config: ConsortiumPipelineConfig,
): Promise<ConsortiumResult> {
const models = getModelsForTier(config.tier)
const collectionStart = Date.now()
// Phase 1: Collect
const rawResults = await collectAllResponses(
models,
messages,
apiKey,
params,
config.collectionConfig,
)
const collectionDuration = Date.now() - collectionStart
// Phase 2: Score
const scoredResponses: ConsortiumResponse[] = rawResults.map(r => ({
model: r.model,
content: r.content,
score: r.success ? scoreResponse(r.content, userQuery) : 0,
duration_ms: r.duration_ms,
success: r.success,
error: r.error,
}))
scoredResponses.sort((a, b) => b.score - a.score)
const successCount = scoredResponses.filter(r => r.success).length
if (successCount === 0) {
throw new Error('All models failed during collection phase')
}
// Phase 3: Synthesize
const orchestratorResult = await synthesize(
userQuery,
scoredResponses,
apiKey,
config.orchestratorModel,
config.maxTokens ?? params.max_tokens ?? 8192,
)
return {
synthesis: orchestratorResult.synthesis,
orchestrator_model: orchestratorResult.model,
orchestrator_duration_ms: orchestratorResult.duration_ms,
responses: scoredResponses,
collection: {
models_queried: models.length,
models_succeeded: successCount,
total_duration_ms: collectionDuration,
},
}
}