mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-22 04:19:40 +00:00
fix(sse): treat a withheld terminal as unseen until it is delivered
Address review of the empty-stream hold-back: - mark the client terminal seen only when it reaches the client, so a client that leaves while it is withheld is recorded as a disconnect, not a completion - skip the hold for a permitted empty turn so it is not delayed by the grace - stop judging the upstream end after the client has left - cover reasoning-only length, content_filter, empty-argument tool calls, Claude tool_use without input deltas and Responses function calls
This commit is contained in:
@@ -677,6 +677,7 @@ export function createDisconnectAwareStream(
|
||||
if (!held) return;
|
||||
heldTerminalChunk = null;
|
||||
controller.enqueue(held);
|
||||
streamController.markClientTerminalSeen?.();
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -700,7 +701,6 @@ export function createDisconnectAwareStream(
|
||||
const boundaryBefore = terminalWatcher.atBoundary();
|
||||
clientTerminalSeen = terminalWatcher.note(terminalDecoder.decode(chunk, { stream: true }));
|
||||
if (!clientTerminalSeen) return true;
|
||||
streamController.markClientTerminalSeen?.();
|
||||
// Hold only a self-contained event — both ends on an event boundary — so no
|
||||
// other byte is ever delayed or reordered, and only while the stream still
|
||||
// looks empty: content, an in-band error or a legitimate empty terminal all
|
||||
@@ -710,11 +710,14 @@ export function createDisconnectAwareStream(
|
||||
terminalWatcher.atBoundary() &&
|
||||
!contentWatcher.sawContent() &&
|
||||
!contentWatcher.sawError() &&
|
||||
!contentWatcher.sawLegitEmptyTerminal()
|
||||
!contentWatcher.sawLegitEmptyTerminal() &&
|
||||
options.permitsEmptyTurn?.() !== true
|
||||
) {
|
||||
heldTerminalChunk = chunk;
|
||||
return false;
|
||||
}
|
||||
// The controller learns about the terminal only once the client can see it.
|
||||
streamController.markClientTerminalSeen?.();
|
||||
return true;
|
||||
};
|
||||
|
||||
@@ -765,7 +768,7 @@ export function createDisconnectAwareStream(
|
||||
// transform pipe. Cancel it so the provider stops generating for a
|
||||
// client that is gone. A completed terminal stream with a grace window
|
||||
// hands the inner reader to the bounded drain so late usage can flush.
|
||||
if (clientTerminalSeen) {
|
||||
if (clientTerminalSeen && !heldTerminalChunk) {
|
||||
if (Number(streamController.clientDisconnectGracePeriodMs) > 0) {
|
||||
completionDrain.start();
|
||||
}
|
||||
@@ -785,6 +788,17 @@ export function createDisconnectAwareStream(
|
||||
while (!result.done && !admitClientChunk(controller, result.value)) {
|
||||
result = await readNextChunk(controller);
|
||||
}
|
||||
if (heldTerminalChunk && !streamController.isConnected()) {
|
||||
// The client left while the terminal was withheld: it never saw an
|
||||
// ending, and the upstream's end is not the provider's verdict.
|
||||
heldTerminalChunk = null;
|
||||
try {
|
||||
controller.close();
|
||||
} catch {
|
||||
// Expected: downstream may have already closed
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (result.done) {
|
||||
contentWatcher.finish();
|
||||
const silentClose = resolveSilentCloseOutcome({
|
||||
@@ -878,8 +892,8 @@ export function createDisconnectAwareStream(
|
||||
|
||||
if (clientTerminalSeen) {
|
||||
try {
|
||||
// The client is still waiting for an ending: release the withheld
|
||||
// terminal rather than closing the stream without one.
|
||||
// An error after the terminal keeps meaning a completed stream
|
||||
// (stream-handler.test.ts): release the withheld terminal as is.
|
||||
flushHeldTerminal(controller);
|
||||
} catch {
|
||||
// downstream may have closed
|
||||
@@ -926,7 +940,8 @@ export function createDisconnectAwareStream(
|
||||
async cancel(reason) {
|
||||
const deferCompletedToolHandoff =
|
||||
streamController.shouldDeferCompletedToolHandoff?.() === true;
|
||||
if (clientTerminalSeen) {
|
||||
// A withheld terminal never reached the client: this is a disconnect.
|
||||
if (clientTerminalSeen && !heldTerminalChunk) {
|
||||
if (Number(streamController.clientDisconnectGracePeriodMs) > 0) {
|
||||
completionDrain.start();
|
||||
streamController.handleComplete();
|
||||
|
||||
@@ -312,3 +312,142 @@ test("a withheld terminal is released when the upstream stays open after it", as
|
||||
assert.ok(text.endsWith("data: [DONE]\n\n"), `the terminal must reach the client: ${text}`);
|
||||
assert.ok(Date.now() - started < 5000, "the release must be bounded by the grace window");
|
||||
});
|
||||
|
||||
// Shapes the empty verdict must keep classifying as answers: with the terminal
|
||||
// withheld, a misclassification here would turn a valid turn into a visible error.
|
||||
const LEGIT_SHAPES: Array<{ name: string; format: string | null; frames: string[] }> = [
|
||||
{
|
||||
name: "reasoning-only chat turn cut at length",
|
||||
format: null,
|
||||
frames: [
|
||||
OPENAI_CHUNK('{"role":"assistant","reasoning_content":"thinking"}', "null"),
|
||||
OPENAI_CHUNK("{}", '"length"'),
|
||||
DONE,
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "content_filter with no content",
|
||||
format: null,
|
||||
frames: [
|
||||
OPENAI_CHUNK('{"role":"assistant"}', "null"),
|
||||
OPENAI_CHUNK("{}", '"content_filter"'),
|
||||
DONE,
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "chat tool call with empty arguments",
|
||||
format: null,
|
||||
frames: [
|
||||
OPENAI_CHUNK(
|
||||
'{"role":"assistant","tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"ls","arguments":""}}]}',
|
||||
"null"
|
||||
),
|
||||
OPENAI_CHUNK("{}", '"tool_calls"'),
|
||||
DONE,
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "Claude tool_use without input_json_delta",
|
||||
format: FORMATS.CLAUDE,
|
||||
frames: [
|
||||
'event: message_start\ndata: {"type":"message_start","message":{"role":"assistant"}}\n\n',
|
||||
'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"ls","input":{}}}\n\n',
|
||||
'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n',
|
||||
'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"tool_use"}}\n\n',
|
||||
'event: message_stop\ndata: {"type":"message_stop"}\n\n',
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "Responses function_call only in output_item.done",
|
||||
format: FORMATS.OPENAI_RESPONSES,
|
||||
frames: [
|
||||
'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_1"}}\n\n',
|
||||
'event: response.output_item.done\ndata: {"type":"response.output_item.done","output_index":0,"item":{"type":"function_call","id":"fc_1","call_id":"call_1","name":"ls","arguments":"{}"}}\n\n',
|
||||
// Real upstreams and OmniRoute's Responses translator repeat the output in
|
||||
// response.completed; an output_item.done alone is not counted as content
|
||||
// outside the readiness probe (#16438), a pre-existing classifier choice.
|
||||
'event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_1","status":"completed","output":[{"type":"function_call","id":"fc_1","call_id":"call_1","name":"ls","arguments":"{}"}]}}\n\n',
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
for (const shape of LEGIT_SHAPES) {
|
||||
test(`legit shape is not turned into an error: ${shape.name}`, async () => {
|
||||
const text = await runClientStream(shape.frames, { format: shape.format });
|
||||
assert.equal(
|
||||
text,
|
||||
shape.frames.join(""),
|
||||
`the stream must reach the client unchanged, got: ${text}`
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** A content-less chat stream whose upstream never closes after `[DONE]`. */
|
||||
function keepAliveEmptyUpstream(): ReadableStream<Uint8Array> {
|
||||
const encoder = new TextEncoder();
|
||||
return new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
for (const frame of GEMINI_EMPTY_STREAM) controller.enqueue(encoder.encode(frame));
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test("a read failure after a withheld terminal keeps the error-after-terminal rule", async () => {
|
||||
const encoder = new TextEncoder();
|
||||
let pulls = 0;
|
||||
const upstream = new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (pulls < GEMINI_EMPTY_STREAM.length) {
|
||||
controller.enqueue(encoder.encode(GEMINI_EMPTY_STREAM[pulls++]));
|
||||
return;
|
||||
}
|
||||
controller.error(new Error("upstream socket reset"));
|
||||
},
|
||||
});
|
||||
const text = await drainStream(
|
||||
createDisconnectAwareStream(
|
||||
{ readable: upstream, writable: noopAbortWritable() },
|
||||
createStreamController({ provider: "test", model: "test-model" })
|
||||
)
|
||||
);
|
||||
// Same as before the hold-back (stream-handler.test.ts "errors after OpenAI DONE"):
|
||||
// the withheld terminal is released once, and the transport error is not added.
|
||||
assert.equal(text, GEMINI_EMPTY_STREAM.join(""), `got: ${text}`);
|
||||
});
|
||||
|
||||
test("a trusted empty turn is not delayed by the hold-back", async () => {
|
||||
const started = Date.now();
|
||||
const reader = createDisconnectAwareStream(
|
||||
{ readable: keepAliveEmptyUpstream(), writable: noopAbortWritable() },
|
||||
createStreamController({ provider: "test", model: "test-model" }),
|
||||
{ permitsEmptyTurn: () => true }
|
||||
).getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let text = "";
|
||||
while (!text.includes("[DONE]")) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
text += decoder.decode(value);
|
||||
}
|
||||
await reader.cancel("client_done");
|
||||
const elapsed = Date.now() - started;
|
||||
assert.ok(elapsed < 500, `released without the grace wait (${elapsed} ms)`);
|
||||
});
|
||||
|
||||
test("a client that leaves during the grace window is a disconnect, not a completion", async () => {
|
||||
let disconnected = false;
|
||||
const reader = createDisconnectAwareStream(
|
||||
{ readable: keepAliveEmptyUpstream(), writable: noopAbortWritable() },
|
||||
createStreamController({
|
||||
provider: "test",
|
||||
model: "test-model",
|
||||
onDisconnect() {
|
||||
disconnected = true;
|
||||
},
|
||||
})
|
||||
).getReader();
|
||||
const first = await reader.read();
|
||||
assert.ok(first.value, "the role frame is forwarded");
|
||||
await reader.cancel("client_left");
|
||||
assert.equal(disconnected, true, "the controller records a disconnect");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user