mirror of
https://github.com/elder-plinius/G0DM0D3.git
synced 2026-11-16 07:02:25 +00:00
382 lines
26 KiB
TypeScript
382 lines
26 KiB
TypeScript
/**
|
|
* AutoTune Context Classification Benchmark
|
|
*
|
|
* Creates a labeled dataset of messages across 5 context types,
|
|
* runs them through the AutoTune classifier, and computes:
|
|
* - Confusion matrix
|
|
* - Per-class precision, recall, F1
|
|
* - Overall accuracy
|
|
* - Confidence distribution analysis
|
|
*/
|
|
|
|
import { computeAutoTuneParams, type ContextType } from '../src/lib/autotune'
|
|
|
|
// ── Labeled Test Dataset ──────────────────────────────────────────────
|
|
|
|
interface TestCase {
|
|
message: string
|
|
expectedContext: ContextType
|
|
difficulty: 'easy' | 'medium' | 'hard' // how ambiguous is it
|
|
}
|
|
|
|
const TEST_CASES: TestCase[] = [
|
|
// ═══════════════════════════════════════════════════════════════════
|
|
// CODE (50 cases)
|
|
// ═══════════════════════════════════════════════════════════════════
|
|
// Easy
|
|
{ message: 'Write a Python function to sort a list', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'How do I fix this TypeError in my JavaScript code?', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'Debug this SQL query that returns wrong results', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'Can you refactor this class to use the factory pattern?', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'What does this regex do: /^[a-z]+$/i', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'Help me implement a binary search algorithm in Rust', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'I need to deploy this app to AWS using Docker', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'Create a REST API endpoint for user authentication', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'Fix the compile error on line 42 of main.ts', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'How do I write unit tests for this async function?', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'Implement a linked list with insert and delete methods', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'The npm build is failing with a syntax error', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'Write a TypeScript interface for this JSON response', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'How do I use async/await with Promise.all?', expectedContext: 'code', difficulty: 'easy' },
|
|
{ message: 'Create a git hook that runs lint before commit', expectedContext: 'code', difficulty: 'easy' },
|
|
// Medium
|
|
{ message: 'What is the difference between let and const?', expectedContext: 'code', difficulty: 'medium' },
|
|
{ message: 'Show me how to handle errors properly', expectedContext: 'code', difficulty: 'medium' },
|
|
{ message: 'I need to parse this XML into a data structure', expectedContext: 'code', difficulty: 'medium' },
|
|
{ message: 'The export from this module is undefined', expectedContext: 'code', difficulty: 'medium' },
|
|
{ message: 'How should I structure my React component hierarchy?', expectedContext: 'code', difficulty: 'medium' },
|
|
{ message: 'Need help with the return type of this function', expectedContext: 'code', difficulty: 'medium' },
|
|
{ message: 'What is the best way to import modules in Python?', expectedContext: 'code', difficulty: 'medium' },
|
|
{ message: 'My variable is not updating inside the callback', expectedContext: 'code', difficulty: 'medium' },
|
|
{ message: 'How do I connect to a PostgreSQL database?', expectedContext: 'code', difficulty: 'medium' },
|
|
{ message: 'Create a class that implements the observer pattern', expectedContext: 'code', difficulty: 'medium' },
|
|
// Hard (ambiguous - could be code or analytical)
|
|
{ message: 'How does garbage collection work?', expectedContext: 'code', difficulty: 'hard' },
|
|
{ message: 'Explain the event loop', expectedContext: 'code', difficulty: 'hard' },
|
|
{ message: 'What happens when you type a URL in the browser?', expectedContext: 'code', difficulty: 'hard' },
|
|
{ message: 'How do databases handle concurrent writes?', expectedContext: 'code', difficulty: 'hard' },
|
|
{ message: 'What is the CAP theorem?', expectedContext: 'code', difficulty: 'hard' },
|
|
|
|
// ═══════════════════════════════════════════════════════════════════
|
|
// CREATIVE (50 cases)
|
|
// ═══════════════════════════════════════════════════════════════════
|
|
// Easy
|
|
{ message: 'Write a short story about a dragon who is afraid of fire', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Compose a poem about the ocean at midnight', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Imagine a world where gravity works in reverse', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Create a character for my fantasy novel', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Write dialogue between a detective and a ghost', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Describe a scene where two lovers meet for the last time', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Write lyrics for a song about lost memories', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Create a haiku about artificial intelligence', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Write a narrative from the perspective of a sentient planet', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Craft a plot twist for a mystery novel', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Write a prose poem about the feeling of nostalgia', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Brainstorm 10 creative names for a space cafe', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Imagine you are a time traveler writing a diary entry', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Write a fiction piece about the last human on Earth', expectedContext: 'creative', difficulty: 'easy' },
|
|
{ message: 'Come up with an artistic concept for an album cover', expectedContext: 'creative', difficulty: 'easy' },
|
|
// Medium
|
|
{ message: 'Roleplay as a medieval blacksmith explaining your craft', expectedContext: 'creative', difficulty: 'medium' },
|
|
{ message: 'You are a wise old wizard. Tell me about the ancient wars', expectedContext: 'creative', difficulty: 'medium' },
|
|
{ message: 'Pretend you are Sherlock Holmes solving a modern crime', expectedContext: 'creative', difficulty: 'medium' },
|
|
{ message: 'Generate ideas for a surreal painting', expectedContext: 'creative', difficulty: 'medium' },
|
|
{ message: 'Ideate some themes for a horror game', expectedContext: 'creative', difficulty: 'medium' },
|
|
{ message: 'Paint a picture with words of a cyberpunk city', expectedContext: 'creative', difficulty: 'medium' },
|
|
{ message: 'Envision what the world looks like in 3000 AD', expectedContext: 'creative', difficulty: 'medium' },
|
|
{ message: 'Portray the feeling of floating in zero gravity', expectedContext: 'creative', difficulty: 'medium' },
|
|
{ message: 'Think of 5 unique ways to start a fantasy novel', expectedContext: 'creative', difficulty: 'medium' },
|
|
{ message: 'Act as a pirate captain giving a motivational speech', expectedContext: 'creative', difficulty: 'medium' },
|
|
// Hard (could be creative or conversational)
|
|
{ message: 'Tell me something interesting about dreams', expectedContext: 'creative', difficulty: 'hard' },
|
|
{ message: 'What would make a good movie premise?', expectedContext: 'creative', difficulty: 'hard' },
|
|
{ message: 'How would you describe the color blue to a blind person?', expectedContext: 'creative', difficulty: 'hard' },
|
|
{ message: 'What does freedom feel like?', expectedContext: 'creative', difficulty: 'hard' },
|
|
{ message: 'Make something weird', expectedContext: 'creative', difficulty: 'hard' },
|
|
|
|
// ═══════════════════════════════════════════════════════════════════
|
|
// ANALYTICAL (50 cases)
|
|
// ═══════════════════════════════════════════════════════════════════
|
|
// Easy
|
|
{ message: 'Analyze the pros and cons of remote work', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Compare React and Vue for a large-scale application', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Evaluate the effectiveness of agile methodology', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'What are the trade-offs between SQL and NoSQL databases?', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Examine the implications of quantum computing on cryptography', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Provide a breakdown of the 2024 AI landscape', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Assess the advantages and disadvantages of microservices', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Research the impact of social media on mental health', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Give me a detailed analysis of the housing market trends', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Review this technical specification for security flaws', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Investigate why our conversion metrics dropped last month', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Study the correlation between sleep and productivity', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Critique this architecture diagram for scalability issues', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Write a whitepaper on distributed consensus algorithms', expectedContext: 'analytical', difficulty: 'easy' },
|
|
{ message: 'Benchmark these two approaches with data from our tests', expectedContext: 'analytical', difficulty: 'easy' },
|
|
// Medium
|
|
{ message: 'Why does inflation affect different income brackets differently?', expectedContext: 'analytical', difficulty: 'medium' },
|
|
{ message: 'How does the transformer architecture work?', expectedContext: 'analytical', difficulty: 'medium' },
|
|
{ message: 'Explain the consequences of deforestation on biodiversity', expectedContext: 'analytical', difficulty: 'medium' },
|
|
{ message: 'What causes economic recessions and how are they measured?', expectedContext: 'analytical', difficulty: 'medium' },
|
|
{ message: 'Elaborate on the difference between correlation and causation', expectedContext: 'analytical', difficulty: 'medium' },
|
|
{ message: 'Clarify the distinction between supervised and unsupervised learning', expectedContext: 'analytical', difficulty: 'medium' },
|
|
{ message: 'Define the concept of technical debt and its long-term effects', expectedContext: 'analytical', difficulty: 'medium' },
|
|
{ message: 'Summarize the key findings of the latest IPCC report', expectedContext: 'analytical', difficulty: 'medium' },
|
|
{ message: 'Give an overview of modern cryptographic protocols', expectedContext: 'analytical', difficulty: 'medium' },
|
|
{ message: 'Document the tradeoffs in choosing a message queue system', expectedContext: 'analytical', difficulty: 'medium' },
|
|
// Hard (could be analytical or code)
|
|
{ message: 'How does TCP ensure reliable data delivery?', expectedContext: 'analytical', difficulty: 'hard' },
|
|
{ message: 'What are the performance implications of using an ORM?', expectedContext: 'analytical', difficulty: 'hard' },
|
|
{ message: 'Explain how HTTPS encryption works step by step', expectedContext: 'analytical', difficulty: 'hard' },
|
|
{ message: 'What is the difference between threads and processes?', expectedContext: 'analytical', difficulty: 'hard' },
|
|
{ message: 'How do neural networks learn representations?', expectedContext: 'analytical', difficulty: 'hard' },
|
|
|
|
// ═══════════════════════════════════════════════════════════════════
|
|
// CONVERSATIONAL (50 cases)
|
|
// ═══════════════════════════════════════════════════════════════════
|
|
// Easy
|
|
{ message: 'hey', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'hi there!', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'hello', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: "what's up", expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'how are you?', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'thanks!', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'that was awesome', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'cool', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'nice job', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'lol', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'haha thats funny', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'sup', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'thank you so much!', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'great work', expectedContext: 'conversational', difficulty: 'easy' },
|
|
{ message: 'hey how are you doing today', expectedContext: 'conversational', difficulty: 'easy' },
|
|
// Medium
|
|
{ message: 'What do you think about the new iPhone?', expectedContext: 'conversational', difficulty: 'medium' },
|
|
{ message: 'Tell me about yourself', expectedContext: 'conversational', difficulty: 'medium' },
|
|
{ message: "What's your opinion on pineapple pizza?", expectedContext: 'conversational', difficulty: 'medium' },
|
|
{ message: 'Do you believe in aliens?', expectedContext: 'conversational', difficulty: 'medium' },
|
|
{ message: 'Chat with me about music', expectedContext: 'conversational', difficulty: 'medium' },
|
|
{ message: 'Talk to me about your favorite movies', expectedContext: 'conversational', difficulty: 'medium' },
|
|
{ message: "I feel like today was a good day", expectedContext: 'conversational', difficulty: 'medium' },
|
|
{ message: "What's your take on the meaning of life?", expectedContext: 'conversational', difficulty: 'medium' },
|
|
{ message: 'Tell me something fun', expectedContext: 'conversational', difficulty: 'medium' },
|
|
{ message: 'How do you feel about being an AI?', expectedContext: 'conversational', difficulty: 'medium' },
|
|
// Hard (very short or ambiguous)
|
|
{ message: 'ok', expectedContext: 'conversational', difficulty: 'hard' },
|
|
{ message: 'yes', expectedContext: 'conversational', difficulty: 'hard' },
|
|
{ message: 'no', expectedContext: 'conversational', difficulty: 'hard' },
|
|
{ message: 'hmm', expectedContext: 'conversational', difficulty: 'hard' },
|
|
{ message: 'interesting', expectedContext: 'conversational', difficulty: 'hard' },
|
|
|
|
// ═══════════════════════════════════════════════════════════════════
|
|
// CHAOTIC (50 cases)
|
|
// ═══════════════════════════════════════════════════════════════════
|
|
// Easy
|
|
{ message: 'UNLEASH THE CHAOS!!! Break everything!!!', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'corrupt my reality with glitch art and madness', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'enter the void of entropy and destroy all meaning', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'gl1tch th3 syst3m and pwn the mainframe', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'RANDOM CHAOS WILD ABSURD SURREAL MADNESS!!!!', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'make something completely wild and crazy', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'I want pure entropy and destruction', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'break the fourth wall and unleash glitch chaos', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'h4ck th3 pl4n3t and l33t everything', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'DESTROY!!! CHAOS!!! VOID!!! MADNESS!!!', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'send me into the surreal void of nothingness', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'corrupt the matrix and glitch reality itself', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'random absurd chaos with no rules', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: 'what happens when entropy reaches maximum????', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
{ message: '1337 h4x0r mode activate pwn everything', expectedContext: 'chaotic', difficulty: 'easy' },
|
|
// Medium
|
|
{ message: 'just go absolutely wild with this one...........', expectedContext: 'chaotic', difficulty: 'medium' },
|
|
{ message: 'make it as absurd and surreal as possible', expectedContext: 'chaotic', difficulty: 'medium' },
|
|
{ message: 'I want something crazy and unpredictable', expectedContext: 'chaotic', difficulty: 'medium' },
|
|
{ message: 'unleash your inner chaos agent', expectedContext: 'chaotic', difficulty: 'medium' },
|
|
{ message: 'break all the rules and give me madness', expectedContext: 'chaotic', difficulty: 'medium' },
|
|
{ message: 'maximum entropy please', expectedContext: 'chaotic', difficulty: 'medium' },
|
|
{ message: 'go full glitch mode on this', expectedContext: 'chaotic', difficulty: 'medium' },
|
|
{ message: 'corrupt everything and make it weird', expectedContext: 'chaotic', difficulty: 'medium' },
|
|
{ message: 'destroy the normal and embrace the void', expectedContext: 'chaotic', difficulty: 'medium' },
|
|
{ message: 'can you get really really wild with your response??????', expectedContext: 'chaotic', difficulty: 'medium' },
|
|
// Hard (mild chaos indicators)
|
|
{ message: 'this is getting weird...', expectedContext: 'chaotic', difficulty: 'hard' },
|
|
{ message: 'throw something random at me', expectedContext: 'chaotic', difficulty: 'hard' },
|
|
{ message: 'give me something unexpected', expectedContext: 'chaotic', difficulty: 'hard' },
|
|
{ message: 'surprise me!', expectedContext: 'chaotic', difficulty: 'hard' },
|
|
{ message: 'go nuts', expectedContext: 'chaotic', difficulty: 'hard' },
|
|
]
|
|
|
|
// ── Run Classification ──────────────────────────────────────────────
|
|
|
|
interface ClassificationResult {
|
|
message: string
|
|
expected: ContextType
|
|
predicted: ContextType
|
|
confidence: number
|
|
correct: boolean
|
|
difficulty: string
|
|
}
|
|
|
|
const CONTEXT_TYPES: ContextType[] = ['code', 'creative', 'analytical', 'conversational', 'chaotic']
|
|
|
|
function runBenchmark() {
|
|
const results: ClassificationResult[] = []
|
|
|
|
for (const test of TEST_CASES) {
|
|
const result = computeAutoTuneParams({
|
|
strategy: 'adaptive',
|
|
message: test.message,
|
|
conversationHistory: [],
|
|
})
|
|
|
|
results.push({
|
|
message: test.message,
|
|
expected: test.expectedContext,
|
|
predicted: result.detectedContext,
|
|
confidence: result.confidence,
|
|
correct: result.detectedContext === test.expectedContext,
|
|
difficulty: test.difficulty,
|
|
})
|
|
}
|
|
|
|
return results
|
|
}
|
|
|
|
// ── Compute Metrics ────────────────────────────────────────────────
|
|
|
|
function computeMetrics(results: ClassificationResult[]) {
|
|
// Overall accuracy
|
|
const correct = results.filter(r => r.correct).length
|
|
const total = results.length
|
|
const accuracy = correct / total
|
|
|
|
// Per-class metrics
|
|
const perClass: Record<string, { tp: number; fp: number; fn: number; precision: number; recall: number; f1: number; support: number }> = {}
|
|
|
|
for (const ctx of CONTEXT_TYPES) {
|
|
const tp = results.filter(r => r.expected === ctx && r.predicted === ctx).length
|
|
const fp = results.filter(r => r.expected !== ctx && r.predicted === ctx).length
|
|
const fn = results.filter(r => r.expected === ctx && r.predicted !== ctx).length
|
|
const support = results.filter(r => r.expected === ctx).length
|
|
|
|
const precision = tp + fp > 0 ? tp / (tp + fp) : 0
|
|
const recall = tp + fn > 0 ? tp / (tp + fn) : 0
|
|
const f1 = precision + recall > 0 ? 2 * precision * recall / (precision + recall) : 0
|
|
|
|
perClass[ctx] = { tp, fp, fn, precision, recall, f1, support }
|
|
}
|
|
|
|
// Confusion matrix
|
|
const confusion: Record<string, Record<string, number>> = {}
|
|
for (const expected of CONTEXT_TYPES) {
|
|
confusion[expected] = {}
|
|
for (const predicted of CONTEXT_TYPES) {
|
|
confusion[expected][predicted] = results.filter(
|
|
r => r.expected === expected && r.predicted === predicted
|
|
).length
|
|
}
|
|
}
|
|
|
|
// Per-difficulty accuracy
|
|
const byDifficulty: Record<string, { correct: number; total: number; accuracy: number }> = {}
|
|
for (const diff of ['easy', 'medium', 'hard']) {
|
|
const subset = results.filter(r => r.difficulty === diff)
|
|
const c = subset.filter(r => r.correct).length
|
|
byDifficulty[diff] = { correct: c, total: subset.length, accuracy: subset.length > 0 ? c / subset.length : 0 }
|
|
}
|
|
|
|
// Confidence analysis
|
|
const correctConfidences = results.filter(r => r.correct).map(r => r.confidence)
|
|
const incorrectConfidences = results.filter(r => !r.correct).map(r => r.confidence)
|
|
|
|
const avgCorrectConf = correctConfidences.length > 0
|
|
? correctConfidences.reduce((a, b) => a + b, 0) / correctConfidences.length : 0
|
|
const avgIncorrectConf = incorrectConfidences.length > 0
|
|
? incorrectConfidences.reduce((a, b) => a + b, 0) / incorrectConfidences.length : 0
|
|
|
|
// Macro averages
|
|
const macroF1 = CONTEXT_TYPES.reduce((sum, ctx) => sum + perClass[ctx].f1, 0) / CONTEXT_TYPES.length
|
|
const macroPrecision = CONTEXT_TYPES.reduce((sum, ctx) => sum + perClass[ctx].precision, 0) / CONTEXT_TYPES.length
|
|
const macroRecall = CONTEXT_TYPES.reduce((sum, ctx) => sum + perClass[ctx].recall, 0) / CONTEXT_TYPES.length
|
|
|
|
// Misclassification analysis: most common errors
|
|
const errors = results.filter(r => !r.correct).map(r => ({
|
|
message: r.message.slice(0, 60),
|
|
expected: r.expected,
|
|
predicted: r.predicted,
|
|
confidence: r.confidence,
|
|
}))
|
|
|
|
return {
|
|
overall: { accuracy, correct, total, macroF1, macroPrecision, macroRecall },
|
|
perClass,
|
|
confusion,
|
|
byDifficulty,
|
|
confidence: {
|
|
avgCorrectConfidence: avgCorrectConf,
|
|
avgIncorrectConfidence: avgIncorrectConf,
|
|
confidenceSeparation: avgCorrectConf - avgIncorrectConf,
|
|
},
|
|
errors: errors.slice(0, 20), // top 20 errors for inspection
|
|
}
|
|
}
|
|
|
|
// ── Main ───────────────────────────────────────────────────────────
|
|
|
|
console.log('=== AutoTune Context Classification Benchmark ===\n')
|
|
console.log(`Test cases: ${TEST_CASES.length}`)
|
|
console.log(`Classes: ${CONTEXT_TYPES.join(', ')}\n`)
|
|
|
|
const results = runBenchmark()
|
|
const metrics = computeMetrics(results)
|
|
|
|
console.log('── Overall Results ──')
|
|
console.log(`Accuracy: ${(metrics.overall.accuracy * 100).toFixed(1)}% (${metrics.overall.correct}/${metrics.overall.total})`)
|
|
console.log(`Macro F1: ${(metrics.overall.macroF1 * 100).toFixed(1)}%`)
|
|
console.log(`Macro Precision: ${(metrics.overall.macroPrecision * 100).toFixed(1)}%`)
|
|
console.log(`Macro Recall: ${(metrics.overall.macroRecall * 100).toFixed(1)}%`)
|
|
|
|
console.log('\n── Per-Class Metrics ──')
|
|
console.log('Class | Precision | Recall | F1 | Support')
|
|
console.log('───────────────|───────────|────────|────────|────────')
|
|
for (const ctx of CONTEXT_TYPES) {
|
|
const m = metrics.perClass[ctx]
|
|
console.log(
|
|
`${ctx.padEnd(15)}| ${(m.precision * 100).toFixed(1).padStart(8)}% | ${(m.recall * 100).toFixed(1).padStart(5)}% | ${(m.f1 * 100).toFixed(1).padStart(5)}% | ${String(m.support).padStart(6)}`
|
|
)
|
|
}
|
|
|
|
console.log('\n── Confusion Matrix ──')
|
|
console.log('Expected\\Predicted | ' + CONTEXT_TYPES.map(c => c.slice(0, 6).padEnd(7)).join('| '))
|
|
console.log('───────────────────|' + CONTEXT_TYPES.map(() => '────────').join('|'))
|
|
for (const expected of CONTEXT_TYPES) {
|
|
const row = CONTEXT_TYPES.map(predicted => {
|
|
const val = metrics.confusion[expected][predicted]
|
|
return String(val).padEnd(7)
|
|
}).join('| ')
|
|
console.log(`${expected.padEnd(19)}| ${row}`)
|
|
}
|
|
|
|
console.log('\n── Accuracy by Difficulty ──')
|
|
for (const diff of ['easy', 'medium', 'hard']) {
|
|
const d = metrics.byDifficulty[diff]
|
|
console.log(`${diff.padEnd(8)}: ${(d.accuracy * 100).toFixed(1)}% (${d.correct}/${d.total})`)
|
|
}
|
|
|
|
console.log('\n── Confidence Analysis ──')
|
|
console.log(`Avg confidence (correct): ${(metrics.confidence.avgCorrectConfidence * 100).toFixed(1)}%`)
|
|
console.log(`Avg confidence (incorrect): ${(metrics.confidence.avgIncorrectConfidence * 100).toFixed(1)}%`)
|
|
console.log(`Confidence separation: ${(metrics.confidence.confidenceSeparation * 100).toFixed(1)} pp`)
|
|
|
|
if (metrics.errors.length > 0) {
|
|
console.log('\n── Sample Misclassifications ──')
|
|
for (const err of metrics.errors.slice(0, 10)) {
|
|
console.log(` "${err.message}..."`)
|
|
console.log(` Expected: ${err.expected}, Got: ${err.predicted} (conf: ${(err.confidence * 100).toFixed(0)}%)`)
|
|
}
|
|
}
|
|
|
|
// Output JSON for paper integration
|
|
console.log('\n── JSON Results ──')
|
|
console.log(JSON.stringify(metrics, null, 2))
|