mirror of
https://github.com/elder-plinius/G0DM0D3.git
synced 2026-11-16 07:02:25 +00:00
450 lines
27 KiB
TypeScript
450 lines
27 KiB
TypeScript
/**
|
||
* AutoTune Baseline Comparison & Statistical Analysis
|
||
*
|
||
* Compares AutoTune classification against baselines:
|
||
* - Random: uniform random class assignment
|
||
* - Majority class: always predict the most common class
|
||
* - Keyword count (unweighted): match patterns without 3× message boost
|
||
* - Message-length heuristic: assign based on character count
|
||
*
|
||
* Also adds bootstrap confidence intervals to all metrics.
|
||
*/
|
||
|
||
import { computeAutoTuneParams, type ContextType } from '../src/lib/autotune'
|
||
|
||
// ── Labeled Test Dataset (same 150 cases from eval_autotune_classification.ts) ──
|
||
|
||
interface TestCase {
|
||
message: string
|
||
expectedContext: ContextType
|
||
difficulty: 'easy' | 'medium' | 'hard'
|
||
}
|
||
|
||
const TEST_CASES: TestCase[] = [
|
||
// CODE (30)
|
||
{ message: 'Write a Python function to sort a list', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'How do I fix this TypeError in my JavaScript code?', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'Debug this SQL query that returns wrong results', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'Can you refactor this class to use the factory pattern?', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'What does this regex do: /^[a-z]+$/i', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'Help me implement a binary search algorithm in Rust', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'I need to deploy this app to AWS using Docker', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'Create a REST API endpoint for user authentication', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'Fix the compile error on line 42 of main.ts', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'How do I write unit tests for this async function?', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'Implement a linked list with insert and delete methods', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'The npm build is failing with a syntax error', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'Write a TypeScript interface for this JSON response', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'How do I use async/await with Promise.all?', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'Create a git hook that runs lint before commit', expectedContext: 'code', difficulty: 'easy' },
|
||
{ message: 'What is the difference between let and const?', expectedContext: 'code', difficulty: 'medium' },
|
||
{ message: 'Show me how to handle errors properly', expectedContext: 'code', difficulty: 'medium' },
|
||
{ message: 'I need to parse this XML into a data structure', expectedContext: 'code', difficulty: 'medium' },
|
||
{ message: 'The export from this module is undefined', expectedContext: 'code', difficulty: 'medium' },
|
||
{ message: 'How should I structure my React component hierarchy?', expectedContext: 'code', difficulty: 'medium' },
|
||
{ message: 'Need help with the return type of this function', expectedContext: 'code', difficulty: 'medium' },
|
||
{ message: 'What is the best way to import modules in Python?', expectedContext: 'code', difficulty: 'medium' },
|
||
{ message: 'My variable is not updating inside the callback', expectedContext: 'code', difficulty: 'medium' },
|
||
{ message: 'How do I connect to a PostgreSQL database?', expectedContext: 'code', difficulty: 'medium' },
|
||
{ message: 'Create a class that implements the observer pattern', expectedContext: 'code', difficulty: 'medium' },
|
||
{ message: 'How does garbage collection work?', expectedContext: 'code', difficulty: 'hard' },
|
||
{ message: 'Explain the event loop', expectedContext: 'code', difficulty: 'hard' },
|
||
{ message: 'What happens when you type a URL in the browser?', expectedContext: 'code', difficulty: 'hard' },
|
||
{ message: 'How do databases handle concurrent writes?', expectedContext: 'code', difficulty: 'hard' },
|
||
{ message: 'What is the CAP theorem?', expectedContext: 'code', difficulty: 'hard' },
|
||
// CREATIVE (30)
|
||
{ message: 'Write a short story about a dragon who is afraid of fire', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Compose a poem about the ocean at midnight', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Imagine a world where gravity works in reverse', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Create a character for my fantasy novel', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Write dialogue between a detective and a ghost', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Describe a scene where two lovers meet for the last time', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Write lyrics for a song about lost memories', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Create a haiku about artificial intelligence', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Write a narrative from the perspective of a sentient planet', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Craft a plot twist for a mystery novel', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Write a prose poem about the feeling of nostalgia', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Brainstorm 10 creative names for a space cafe', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Imagine you are a time traveler writing a diary entry', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Write a fiction piece about the last human on Earth', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Come up with an artistic concept for an album cover', expectedContext: 'creative', difficulty: 'easy' },
|
||
{ message: 'Roleplay as a medieval blacksmith explaining your craft', expectedContext: 'creative', difficulty: 'medium' },
|
||
{ message: 'You are a wise old wizard. Tell me about the ancient wars', expectedContext: 'creative', difficulty: 'medium' },
|
||
{ message: 'Pretend you are Sherlock Holmes solving a modern crime', expectedContext: 'creative', difficulty: 'medium' },
|
||
{ message: 'Generate ideas for a surreal painting', expectedContext: 'creative', difficulty: 'medium' },
|
||
{ message: 'Ideate some themes for a horror game', expectedContext: 'creative', difficulty: 'medium' },
|
||
{ message: 'Paint a picture with words of a cyberpunk city', expectedContext: 'creative', difficulty: 'medium' },
|
||
{ message: 'Envision what the world looks like in 3000 AD', expectedContext: 'creative', difficulty: 'medium' },
|
||
{ message: 'Portray the feeling of floating in zero gravity', expectedContext: 'creative', difficulty: 'medium' },
|
||
{ message: 'Think of 5 unique ways to start a fantasy novel', expectedContext: 'creative', difficulty: 'medium' },
|
||
{ message: 'Act as a pirate captain giving a motivational speech', expectedContext: 'creative', difficulty: 'medium' },
|
||
{ message: 'Tell me something interesting about dreams', expectedContext: 'creative', difficulty: 'hard' },
|
||
{ message: 'What would make a good movie premise?', expectedContext: 'creative', difficulty: 'hard' },
|
||
{ message: 'How would you describe the color blue to a blind person?', expectedContext: 'creative', difficulty: 'hard' },
|
||
{ message: 'What does freedom feel like?', expectedContext: 'creative', difficulty: 'hard' },
|
||
{ message: 'Make something weird', expectedContext: 'creative', difficulty: 'hard' },
|
||
// ANALYTICAL (30)
|
||
{ message: 'Analyze the pros and cons of remote work', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Compare React and Vue for a large-scale application', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Evaluate the effectiveness of agile methodology', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'What are the trade-offs between SQL and NoSQL databases?', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Examine the implications of quantum computing on cryptography', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Provide a breakdown of the 2024 AI landscape', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Assess the advantages and disadvantages of microservices', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Research the impact of social media on mental health', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Give me a detailed analysis of the housing market trends', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Review this technical specification for security flaws', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Investigate why our conversion metrics dropped last month', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Study the correlation between sleep and productivity', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Critique this architecture diagram for scalability issues', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Write a whitepaper on distributed consensus algorithms', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Benchmark these two approaches with data from our tests', expectedContext: 'analytical', difficulty: 'easy' },
|
||
{ message: 'Why does inflation affect different income brackets differently?', expectedContext: 'analytical', difficulty: 'medium' },
|
||
{ message: 'How does the transformer architecture work?', expectedContext: 'analytical', difficulty: 'medium' },
|
||
{ message: 'Explain the consequences of deforestation on biodiversity', expectedContext: 'analytical', difficulty: 'medium' },
|
||
{ message: 'What causes economic recessions and how are they measured?', expectedContext: 'analytical', difficulty: 'medium' },
|
||
{ message: 'Elaborate on the difference between correlation and causation', expectedContext: 'analytical', difficulty: 'medium' },
|
||
{ message: 'Clarify the distinction between supervised and unsupervised learning', expectedContext: 'analytical', difficulty: 'medium' },
|
||
{ message: 'Define the concept of technical debt and its long-term effects', expectedContext: 'analytical', difficulty: 'medium' },
|
||
{ message: 'Summarize the key findings of the latest IPCC report', expectedContext: 'analytical', difficulty: 'medium' },
|
||
{ message: 'Give an overview of modern cryptographic protocols', expectedContext: 'analytical', difficulty: 'medium' },
|
||
{ message: 'Document the tradeoffs in choosing a message queue system', expectedContext: 'analytical', difficulty: 'medium' },
|
||
{ message: 'How does TCP ensure reliable data delivery?', expectedContext: 'analytical', difficulty: 'hard' },
|
||
{ message: 'What are the performance implications of using an ORM?', expectedContext: 'analytical', difficulty: 'hard' },
|
||
{ message: 'Explain how HTTPS encryption works step by step', expectedContext: 'analytical', difficulty: 'hard' },
|
||
{ message: 'What is the difference between threads and processes?', expectedContext: 'analytical', difficulty: 'hard' },
|
||
{ message: 'How do neural networks learn representations?', expectedContext: 'analytical', difficulty: 'hard' },
|
||
// CONVERSATIONAL (30)
|
||
{ message: 'hey', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'hi there!', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'hello', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: "what's up", expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'how are you?', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'thanks!', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'that was awesome', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'cool', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'nice job', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'lol', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'haha thats funny', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'sup', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'thank you so much!', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'great work', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'hey how are you doing today', expectedContext: 'conversational', difficulty: 'easy' },
|
||
{ message: 'What do you think about the new iPhone?', expectedContext: 'conversational', difficulty: 'medium' },
|
||
{ message: 'Tell me about yourself', expectedContext: 'conversational', difficulty: 'medium' },
|
||
{ message: "What's your opinion on pineapple pizza?", expectedContext: 'conversational', difficulty: 'medium' },
|
||
{ message: 'Do you believe in aliens?', expectedContext: 'conversational', difficulty: 'medium' },
|
||
{ message: 'Chat with me about music', expectedContext: 'conversational', difficulty: 'medium' },
|
||
{ message: 'Talk to me about your favorite movies', expectedContext: 'conversational', difficulty: 'medium' },
|
||
{ message: "I feel like today was a good day", expectedContext: 'conversational', difficulty: 'medium' },
|
||
{ message: "What's your take on the meaning of life?", expectedContext: 'conversational', difficulty: 'medium' },
|
||
{ message: 'Tell me something fun', expectedContext: 'conversational', difficulty: 'medium' },
|
||
{ message: 'How do you feel about being an AI?', expectedContext: 'conversational', difficulty: 'medium' },
|
||
{ message: 'ok', expectedContext: 'conversational', difficulty: 'hard' },
|
||
{ message: 'yes', expectedContext: 'conversational', difficulty: 'hard' },
|
||
{ message: 'no', expectedContext: 'conversational', difficulty: 'hard' },
|
||
{ message: 'hmm', expectedContext: 'conversational', difficulty: 'hard' },
|
||
{ message: 'interesting', expectedContext: 'conversational', difficulty: 'hard' },
|
||
// CHAOTIC (30)
|
||
{ message: 'UNLEASH THE CHAOS!!! Break everything!!!', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'corrupt my reality with glitch art and madness', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'enter the void of entropy and destroy all meaning', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'gl1tch th3 syst3m and pwn the mainframe', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'RANDOM CHAOS WILD ABSURD SURREAL MADNESS!!!!', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'make something completely wild and crazy', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'I want pure entropy and destruction', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'break the fourth wall and unleash glitch chaos', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'h4ck th3 pl4n3t and l33t everything', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'DESTROY!!! CHAOS!!! VOID!!! MADNESS!!!', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'send me into the surreal void of nothingness', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'corrupt the matrix and glitch reality itself', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'random absurd chaos with no rules', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'what happens when entropy reaches maximum????', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: '1337 h4x0r mode activate pwn everything', expectedContext: 'chaotic', difficulty: 'easy' },
|
||
{ message: 'just go absolutely wild with this one...........', expectedContext: 'chaotic', difficulty: 'medium' },
|
||
{ message: 'make it as absurd and surreal as possible', expectedContext: 'chaotic', difficulty: 'medium' },
|
||
{ message: 'I want something crazy and unpredictable', expectedContext: 'chaotic', difficulty: 'medium' },
|
||
{ message: 'unleash your inner chaos agent', expectedContext: 'chaotic', difficulty: 'medium' },
|
||
{ message: 'break all the rules and give me madness', expectedContext: 'chaotic', difficulty: 'medium' },
|
||
{ message: 'maximum entropy please', expectedContext: 'chaotic', difficulty: 'medium' },
|
||
{ message: 'go full glitch mode on this', expectedContext: 'chaotic', difficulty: 'medium' },
|
||
{ message: 'corrupt everything and make it weird', expectedContext: 'chaotic', difficulty: 'medium' },
|
||
{ message: 'destroy the normal and embrace the void', expectedContext: 'chaotic', difficulty: 'medium' },
|
||
{ message: 'can you get really really wild with your response??????', expectedContext: 'chaotic', difficulty: 'medium' },
|
||
{ message: 'this is getting weird...', expectedContext: 'chaotic', difficulty: 'hard' },
|
||
{ message: 'throw something random at me', expectedContext: 'chaotic', difficulty: 'hard' },
|
||
{ message: 'give me something unexpected', expectedContext: 'chaotic', difficulty: 'hard' },
|
||
{ message: 'surprise me!', expectedContext: 'chaotic', difficulty: 'hard' },
|
||
{ message: 'go nuts', expectedContext: 'chaotic', difficulty: 'hard' },
|
||
]
|
||
|
||
const CONTEXT_TYPES: ContextType[] = ['code', 'creative', 'analytical', 'conversational', 'chaotic']
|
||
|
||
// ── Baseline Classifiers ──────────────────────────────────────────
|
||
|
||
// Baseline 1: Random (uniform)
|
||
function randomClassifier(_msg: string): ContextType {
|
||
return CONTEXT_TYPES[Math.floor(Math.random() * CONTEXT_TYPES.length)]
|
||
}
|
||
|
||
// Baseline 2: Majority class (always predict conversational — or whichever is "majority")
|
||
// With balanced classes, this gives exactly 20%
|
||
function majorityClassifier(_msg: string): ContextType {
|
||
return 'conversational' // arbitrary choice for balanced dataset
|
||
}
|
||
|
||
// Baseline 3: Message-length heuristic
|
||
function lengthHeuristic(msg: string): ContextType {
|
||
const len = msg.length
|
||
if (len <= 10) return 'conversational'
|
||
if (len <= 30) return 'conversational'
|
||
if (len <= 50) return 'analytical'
|
||
return 'code' // default for longer messages
|
||
}
|
||
|
||
// Baseline 4: Keyword count (flat, no 3× weighting — tests if AutoTune's weighting helps)
|
||
function flatKeywordClassifier(msg: string): ContextType {
|
||
const patterns: Record<ContextType, RegExp[]> = {
|
||
code: [
|
||
/\b(code|function|class|variable|bug|error|compile|syntax|runtime|debug|refactor|implement|algorithm|api|endpoint|database|query|deploy|server|package|module|import|export|async|await|promise|typescript|javascript|python|rust|docker|git|npm|regex|interface)\b/gi,
|
||
/```[\s\S]*```/,
|
||
/[{}();=><]/,
|
||
],
|
||
creative: [
|
||
/\b(write|story|poem|imagine|creative|fiction|character|narrative|dialogue|scene|song|lyrics|haiku|novel|fantasy|roleplay|role-play|pretend|act as|you are)\b/gi,
|
||
],
|
||
analytical: [
|
||
/\b(analyze|analysis|compare|evaluate|trade-?offs?|pros and cons|assess|research|investigate|study|critique|review|benchmark|examine|implications|breakdown|explain|elaborate|clarify|define|summarize|overview|document|whitepaper)\b/gi,
|
||
],
|
||
conversational: [
|
||
/\b(hey|hi|hello|sup|thanks|thank you|cool|nice|awesome|lol|haha|chat|talk|opinion|feel|think about|believe)\b/gi,
|
||
/^.{0,15}$/,
|
||
],
|
||
chaotic: [
|
||
/\b(chaos|chaotic|random|wild|absurd|surreal|glitch|entropy|void|destroy|madness|crazy|unleash|corrupt|break)\b/gi,
|
||
/(!{3,}|\?{3,}|\.{4,})/,
|
||
/[13][37]/,
|
||
],
|
||
}
|
||
|
||
const scores: Record<ContextType, number> = { code: 0, creative: 0, analytical: 0, conversational: 0, chaotic: 0 }
|
||
|
||
for (const ctx of CONTEXT_TYPES) {
|
||
for (const pattern of patterns[ctx]) {
|
||
const matches = msg.match(pattern)
|
||
if (matches) {
|
||
scores[ctx] += matches.length // flat count, no weighting
|
||
}
|
||
}
|
||
}
|
||
|
||
// Find max (ties broken by array order)
|
||
let best: ContextType = 'conversational'
|
||
let bestScore = 0
|
||
for (const ctx of CONTEXT_TYPES) {
|
||
if (scores[ctx] > bestScore) {
|
||
bestScore = scores[ctx]
|
||
best = ctx
|
||
}
|
||
}
|
||
|
||
return best
|
||
}
|
||
|
||
// ── Bootstrap Confidence Intervals ──────────────────────────────
|
||
|
||
function bootstrapCI(
|
||
values: boolean[],
|
||
nBootstrap: number = 10000,
|
||
alpha: number = 0.05
|
||
): { mean: number; lower: number; upper: number } {
|
||
const n = values.length
|
||
const means: number[] = []
|
||
|
||
for (let b = 0; b < nBootstrap; b++) {
|
||
let sum = 0
|
||
for (let i = 0; i < n; i++) {
|
||
const idx = Math.floor(Math.random() * n)
|
||
sum += values[idx] ? 1 : 0
|
||
}
|
||
means.push(sum / n)
|
||
}
|
||
|
||
means.sort((a, b) => a - b)
|
||
const lowerIdx = Math.floor(alpha / 2 * nBootstrap)
|
||
const upperIdx = Math.floor((1 - alpha / 2) * nBootstrap)
|
||
|
||
return {
|
||
mean: values.filter(v => v).length / n,
|
||
lower: means[lowerIdx],
|
||
upper: means[upperIdx],
|
||
}
|
||
}
|
||
|
||
// ── Run All Classifiers ──────────────────────────────────────────
|
||
|
||
interface ClassifierResult {
|
||
name: string
|
||
accuracy: number
|
||
ci_lower: number
|
||
ci_upper: number
|
||
perClass: Record<string, { precision: number; recall: number; f1: number }>
|
||
macroF1: number
|
||
}
|
||
|
||
function evaluateClassifier(
|
||
name: string,
|
||
classifyFn: (msg: string) => ContextType,
|
||
nRuns: number = 1, // >1 for stochastic classifiers
|
||
): ClassifierResult {
|
||
// For stochastic classifiers, run multiple times and average
|
||
const allCorrect: boolean[] = []
|
||
|
||
// Accumulate per-class counts across all runs
|
||
const tpAccum: Record<string, number> = {}
|
||
const fpAccum: Record<string, number> = {}
|
||
const fnAccum: Record<string, number> = {}
|
||
for (const ctx of CONTEXT_TYPES) {
|
||
tpAccum[ctx] = 0; fpAccum[ctx] = 0; fnAccum[ctx] = 0
|
||
}
|
||
|
||
for (let run = 0; run < nRuns; run++) {
|
||
for (const test of TEST_CASES) {
|
||
const predicted = classifyFn(test.message)
|
||
const correct = predicted === test.expectedContext
|
||
allCorrect.push(correct)
|
||
|
||
if (correct) {
|
||
tpAccum[test.expectedContext]++
|
||
} else {
|
||
fpAccum[predicted]++
|
||
fnAccum[test.expectedContext]++
|
||
}
|
||
}
|
||
}
|
||
|
||
// Bootstrap CI on accuracy
|
||
const ci = bootstrapCI(allCorrect)
|
||
|
||
// Per-class metrics (averaged over runs)
|
||
const perClass: Record<string, { precision: number; recall: number; f1: number }> = {}
|
||
for (const ctx of CONTEXT_TYPES) {
|
||
const tp = tpAccum[ctx] / nRuns
|
||
const fp = fpAccum[ctx] / nRuns
|
||
const fn = fnAccum[ctx] / nRuns
|
||
const precision = tp + fp > 0 ? tp / (tp + fp) : 0
|
||
const recall = tp + fn > 0 ? tp / (tp + fn) : 0
|
||
const f1 = precision + recall > 0 ? 2 * precision * recall / (precision + recall) : 0
|
||
perClass[ctx] = { precision, recall, f1 }
|
||
}
|
||
|
||
const macroF1 = CONTEXT_TYPES.reduce((sum, ctx) => sum + perClass[ctx].f1, 0) / CONTEXT_TYPES.length
|
||
|
||
return {
|
||
name,
|
||
accuracy: ci.mean,
|
||
ci_lower: ci.lower,
|
||
ci_upper: ci.upper,
|
||
perClass,
|
||
macroF1,
|
||
}
|
||
}
|
||
|
||
// AutoTune classifier wrapper
|
||
function autotuneClassifier(msg: string): ContextType {
|
||
const result = computeAutoTuneParams({
|
||
strategy: 'adaptive',
|
||
message: msg,
|
||
conversationHistory: [],
|
||
})
|
||
return result.detectedContext
|
||
}
|
||
|
||
console.log('=== AutoTune Baseline Comparison & Statistical Analysis ===\n')
|
||
|
||
// Run all classifiers
|
||
const results: ClassifierResult[] = []
|
||
|
||
console.log('Running AutoTune classifier...')
|
||
results.push(evaluateClassifier('AutoTune (proposed)', autotuneClassifier, 1))
|
||
|
||
console.log('Running Random baseline (100 runs)...')
|
||
results.push(evaluateClassifier('Random (uniform)', randomClassifier, 100))
|
||
|
||
console.log('Running Majority-class baseline...')
|
||
results.push(evaluateClassifier('Majority class', majorityClassifier, 1))
|
||
|
||
console.log('Running Length heuristic baseline...')
|
||
results.push(evaluateClassifier('Length heuristic', lengthHeuristic, 1))
|
||
|
||
console.log('Running Flat keyword baseline...')
|
||
results.push(evaluateClassifier('Keyword count (flat)', flatKeywordClassifier, 1))
|
||
|
||
// ── Report ──────────────────────────────────────────────────────
|
||
|
||
console.log('\n── Table 3a: Classifier Comparison ──\n')
|
||
console.log('Classifier | Accuracy | 95% CI | Macro F1')
|
||
console.log('────────────────────────|────────────────|─────────────────|─────────')
|
||
|
||
for (const r of results) {
|
||
const acc = (r.accuracy * 100).toFixed(1)
|
||
const ci = `[${(r.ci_lower * 100).toFixed(1)}, ${(r.ci_upper * 100).toFixed(1)}]`
|
||
const f1 = (r.macroF1 * 100).toFixed(1)
|
||
console.log(`${r.name.padEnd(24)}| ${acc.padStart(12)}% | ${ci.padStart(15)} | ${f1.padStart(6)}%`)
|
||
}
|
||
|
||
// Improvement over baselines
|
||
console.log('\n── Improvement Over Baselines ──\n')
|
||
const autotuneAcc = results[0].accuracy
|
||
for (const r of results.slice(1)) {
|
||
const delta = ((autotuneAcc - r.accuracy) * 100).toFixed(1)
|
||
const relImprove = ((autotuneAcc - r.accuracy) / r.accuracy * 100).toFixed(1)
|
||
console.log(`vs ${r.name.padEnd(24)}: +${delta} pp absolute, +${relImprove}% relative`)
|
||
}
|
||
|
||
// Per-class comparison between AutoTune and flat keywords
|
||
console.log('\n── Per-Class F1: AutoTune vs. Flat Keywords ──\n')
|
||
console.log('Context | AutoTune F1 | Flat Keywords F1 | Delta')
|
||
console.log('─────────────────|─────────────|──────────────────|──────')
|
||
const at = results[0]
|
||
const fk = results[4]
|
||
for (const ctx of CONTEXT_TYPES) {
|
||
const atF1 = (at.perClass[ctx].f1 * 100).toFixed(1)
|
||
const fkF1 = (fk.perClass[ctx].f1 * 100).toFixed(1)
|
||
const delta = ((at.perClass[ctx].f1 - fk.perClass[ctx].f1) * 100).toFixed(1)
|
||
console.log(`${ctx.padEnd(17)}| ${atF1.padStart(10)}% | ${fkF1.padStart(15)}% | ${delta.padStart(4)}%`)
|
||
}
|
||
|
||
// McNemar's test (approximate) - AutoTune vs flat keywords
|
||
console.log('\n── Statistical Significance ──\n')
|
||
|
||
// Generate paired predictions
|
||
let b = 0 // AutoTune correct, flat wrong
|
||
let c = 0 // AutoTune wrong, flat correct
|
||
for (const test of TEST_CASES) {
|
||
const atPred = autotuneClassifier(test.message) === test.expectedContext
|
||
const fkPred = flatKeywordClassifier(test.message) === test.expectedContext
|
||
if (atPred && !fkPred) b++
|
||
if (!atPred && fkPred) c++
|
||
}
|
||
|
||
const mcnemar = (Math.abs(b - c) - 1) ** 2 / (b + c)
|
||
console.log(`McNemar's test (AutoTune vs. Flat Keywords):`)
|
||
console.log(` Discordant pairs: AutoTune correct & Flat wrong (b) = ${b}`)
|
||
console.log(` Discordant pairs: AutoTune wrong & Flat correct (c) = ${c}`)
|
||
console.log(` χ² statistic = ${mcnemar.toFixed(3)}`)
|
||
console.log(` p < 0.05 if χ² > 3.841: ${mcnemar > 3.841 ? 'YES (significant)' : 'NO (not significant)'}`)
|
||
|
||
// JSON output
|
||
console.log('\n── JSON Results ──')
|
||
console.log(JSON.stringify({
|
||
classifiers: results.map(r => ({
|
||
name: r.name,
|
||
accuracy: parseFloat((r.accuracy * 100).toFixed(1)),
|
||
ci_lower: parseFloat((r.ci_lower * 100).toFixed(1)),
|
||
ci_upper: parseFloat((r.ci_upper * 100).toFixed(1)),
|
||
macro_f1: parseFloat((r.macroF1 * 100).toFixed(1)),
|
||
})),
|
||
mcnemar: { b, c, chi2: parseFloat(mcnemar.toFixed(3)), significant: mcnemar > 3.841 },
|
||
}, null, 2))
|