mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-18 07:58:27 +00:00
fix(combo): trust finish_reason stop over the reasoning-ratio heuristic
The 90% reasoning-ratio check rejects responses where reasoning fills the budget and visible content is empty. That is correct for models that optionally think and run out of budget mid-reasoning, but it misfires on models that cannot disable thinking: Claude Opus 4.7+ and GLM-5.3 both spend most of their output on reasoning and still finish normally with finish_reason "stop". OpenAI, Anthropic, and Z.ai all document "stop" as a natural end of turn and "length"/"max_tokens" as truncation. Trust the stop signal and keep the ratio check only as the fallback when the provider reports no finish_reason. Signed-off-by: Minxi Hou <houminxi@gmail.com>
This commit is contained in:
@@ -950,10 +950,12 @@ export async function validateResponseQuality(
|
||||
reason: `reasoning truncated at token limit (finish_reason: ${finishReason}) — no content output`,
|
||||
};
|
||||
}
|
||||
if (usage) {
|
||||
if (usage && finishReason !== "stop") {
|
||||
const reasoningTokens = getReasoningTokens(usage);
|
||||
// If reasoning consumed 90%+ of completion tokens, the model ran out of
|
||||
// budget before producing any content output.
|
||||
// finish_reason "stop" means the model ended on its own (OpenAI, Anthropic,
|
||||
// Z.ai all document this). A high reasoning ratio on a clean stop is normal
|
||||
// for models that cannot disable thinking (Claude Opus 4.7+, GLM-5.3). The
|
||||
// 90% check stays as the fallback when the provider reports no finish_reason.
|
||||
if (completionTokens > 0 && reasoningTokens >= completionTokens * 0.9) {
|
||||
if (isTinyBudgetTruncation(completionTokens)) return { valid: true };
|
||||
return {
|
||||
|
||||
@@ -249,7 +249,7 @@ test("client-audit-2026-09-01: no finish_reason + reasoning <90% of tokens → s
|
||||
assert.equal(out.valid, true, "should stay valid: no finish_reason signal, ratio under 90%");
|
||||
});
|
||||
|
||||
test("client-audit-2026-09-01: finish_reason:stop + empty content + reasoning → ratio heuristic still applies unchanged", async () => {
|
||||
test("finish_reason:stop + reasoning over 90% is a clean stop, not exhaustion", async () => {
|
||||
const res = makeResponse({
|
||||
choices: [
|
||||
{
|
||||
@@ -260,7 +260,20 @@ test("client-audit-2026-09-01: finish_reason:stop + empty content + reasoning
|
||||
usage: { completion_tokens: 4096, reasoning_tokens: 3800 },
|
||||
});
|
||||
const out = await validateResponseQuality(res, false, silentLog);
|
||||
assert.equal(out.valid, false, "finish_reason:stop doesn't short-circuit — ratio (>90%) still applies");
|
||||
assert.equal(out.valid, true, "finish_reason:stop means the model ended on its own");
|
||||
});
|
||||
|
||||
test("no finish_reason + reasoning over 90% stays invalid (fallback heuristic)", async () => {
|
||||
const res = makeResponse({
|
||||
choices: [
|
||||
{
|
||||
message: { content: null, reasoning_content: "Deep reasoning" },
|
||||
},
|
||||
],
|
||||
usage: { completion_tokens: 4096, reasoning_tokens: 3800 },
|
||||
});
|
||||
const out = await validateResponseQuality(res, false, silentLog);
|
||||
assert.equal(out.valid, false, "no finish_reason still falls back to the 90% check");
|
||||
assert.match(out.reason ?? "", /reasoning consumed/i);
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user