mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-21 15:05:56 +00:00
fix(sse): reuse the upstream part identity on synthesized Responses text deltas
- stream.ts: remember the item_id/output_index/content_index of the delta that started a buffered textual tool-call candidate and reuse them when the tail is flushed at EOF, instead of a fabricated msg id at 0/0. - streamRecovery.ts: track the last Responses text delta identity and stitch the continuation suffix onto that open part instead of msg_0. - Move the replayed-tail delta builder and the id backfill shared by the credential and PII transforms into sseTextBuffers.ts, which keeps flushResponsesFormat under the complexity ratchet. - Test isolates DATA_DIR and closes the DB handle after the PII case.
This commit is contained in:
committed by
diegosouzapw
parent
21fdb04ae5
commit
2684763dd5
@@ -12,6 +12,7 @@
|
||||
*/
|
||||
import { STREAM_RECOVERY } from "../config/constants.ts";
|
||||
import { isRegisteredFeatureFlagEnabled } from "../utils/registeredFeatureFlag.ts";
|
||||
import { pickResponsesTextPart, type ResponsesTextPart } from "../utils/responsesTextPart.ts";
|
||||
import {
|
||||
createThroughputWatchdog,
|
||||
ThroughputWatchdogError,
|
||||
@@ -268,6 +269,8 @@ export interface OpenAiSseScan {
|
||||
* signals, or a terminal lifecycle event.
|
||||
*/
|
||||
parsedResponses: boolean;
|
||||
/** Identity of the last Responses text delta, or null if none was seen (#20257). */
|
||||
textPart: ResponsesTextPart | null;
|
||||
}
|
||||
|
||||
/** Responses-API `type` values the scanner understands. Anything else is ignored. */
|
||||
@@ -297,6 +300,7 @@ export function scanOpenAiSseText(sse: string): OpenAiSseScan {
|
||||
finishReason: null,
|
||||
parsedOpenAi: false,
|
||||
parsedResponses: false,
|
||||
textPart: null,
|
||||
};
|
||||
if (typeof sse !== "string" || sse.length === 0) {
|
||||
return {
|
||||
@@ -308,6 +312,7 @@ export function scanOpenAiSseText(sse: string): OpenAiSseScan {
|
||||
finishReason: fold.finishReason,
|
||||
parsedOpenAi: fold.parsedOpenAi,
|
||||
parsedResponses: fold.parsedResponses,
|
||||
textPart: fold.textPart,
|
||||
};
|
||||
}
|
||||
for (const line of sse.split("\n")) {
|
||||
@@ -323,6 +328,7 @@ export function scanOpenAiSseText(sse: string): OpenAiSseScan {
|
||||
finishReason: fold.finishReason,
|
||||
parsedOpenAi: fold.parsedOpenAi,
|
||||
parsedResponses: fold.parsedResponses,
|
||||
textPart: fold.textPart,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -335,6 +341,7 @@ interface SseScanFold {
|
||||
finishReason: string | null;
|
||||
parsedOpenAi: boolean;
|
||||
parsedResponses: boolean;
|
||||
textPart: ResponsesTextPart | null;
|
||||
}
|
||||
|
||||
/** Parse one raw SSE line into the running scan fold. */
|
||||
@@ -366,6 +373,9 @@ function scanSseLine(line: string, fold: SseScanFold): void {
|
||||
addText: (delta) => {
|
||||
fold.text += delta;
|
||||
},
|
||||
setTextPart: (part) => {
|
||||
fold.textPart = part;
|
||||
},
|
||||
addReasoning: (delta) => {
|
||||
fold.reasoningText += delta;
|
||||
},
|
||||
@@ -415,6 +425,7 @@ function scanChatFinishReason(rawFinishReason: unknown, fold: SseScanFold): void
|
||||
|
||||
interface ResponsesScanSink {
|
||||
addText: (delta: string) => void;
|
||||
setTextPart: (part: ResponsesTextPart) => void;
|
||||
addReasoning: (delta: string) => void;
|
||||
markToolCall: () => void;
|
||||
markToolCallFinished: () => void;
|
||||
@@ -468,6 +479,7 @@ function scanResponsesTextDelta(
|
||||
eventType: string,
|
||||
sink: ResponsesScanSink
|
||||
): void {
|
||||
if (eventType === RESPONSES_TEXT_DELTA) sink.setTextPart(pickResponsesTextPart(json));
|
||||
const delta = (json as { delta?: unknown }).delta;
|
||||
if (typeof delta !== "string" || delta.length === 0) {
|
||||
sink.markParsed();
|
||||
@@ -677,6 +689,7 @@ export function createRecoverableStream(
|
||||
let emittedToolCallFinish = false; // any finish_reason "tool_calls" seen
|
||||
let emittedParsedOpenAi = false;
|
||||
let emittedParsedResponses = false;
|
||||
let emittedTextPart: ResponsesTextPart = {}; // open Responses text part a suffix extends
|
||||
// STREAM_RECOVERY_TOOLCALL_ORDER_FIX, resolved lazily at most once per stream and only
|
||||
// on a recovery decision, so the flag costs nothing on streams that end cleanly.
|
||||
let toolCallOrderFix: boolean | undefined;
|
||||
@@ -705,6 +718,7 @@ export function createRecoverableStream(
|
||||
if (scan.finishReason === "tool_calls") emittedToolCallFinish = true;
|
||||
if (scan.parsedOpenAi) emittedParsedOpenAi = true;
|
||||
if (scan.parsedResponses) emittedParsedResponses = true;
|
||||
if (scan.textPart) emittedTextPart = scan.textPart;
|
||||
};
|
||||
|
||||
const flushHeld = (controller: ReadableStreamDefaultController<Uint8Array>) => {
|
||||
@@ -963,7 +977,7 @@ export function createRecoverableStream(
|
||||
emit(
|
||||
controller,
|
||||
encoder.encode(
|
||||
`data: ${JSON.stringify({ type: "response.output_text.delta", item_id: "msg_0", output_index: 0, content_index: 0, delta: suffix })}\n\n`
|
||||
`data: ${JSON.stringify({ type: "response.output_text.delta", item_id: "msg_0", output_index: 0, content_index: 0, ...emittedTextPart, delta: suffix })}\n\n`
|
||||
)
|
||||
);
|
||||
} else {
|
||||
|
||||
22
open-sse/utils/responsesTextPart.ts
Normal file
22
open-sse/utils/responsesTextPart.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* Wire identity of a Responses-API `output_text` content part. Strict Responses clients
|
||||
* correlate `response.output_text.delta` events by these fields, so a delta OmniRoute
|
||||
* synthesizes later (a held-back tail, a recovery suffix) must reuse the identity of the
|
||||
* upstream part it extends instead of inventing one (#20257).
|
||||
*/
|
||||
export interface ResponsesTextPart {
|
||||
item_id?: string;
|
||||
output_index?: number;
|
||||
content_index?: number;
|
||||
}
|
||||
|
||||
/** Copy only the well-typed identity fields of a Responses text event. */
|
||||
export function pickResponsesTextPart(event: unknown): ResponsesTextPart {
|
||||
const part: ResponsesTextPart = {};
|
||||
if (!event || typeof event !== "object") return part;
|
||||
const record = event as Record<string, unknown>;
|
||||
if (typeof record.item_id === "string" && record.item_id) part.item_id = record.item_id;
|
||||
if (typeof record.output_index === "number") part.output_index = record.output_index;
|
||||
if (typeof record.content_index === "number") part.content_index = record.content_index;
|
||||
return part;
|
||||
}
|
||||
@@ -67,6 +67,7 @@ import {
|
||||
} from "./responsesCommentaryDrop.ts";
|
||||
import { buildErrorBody } from "./error.ts";
|
||||
import { parseTextualToolCallCandidate, isValidToolCallHeaderPrefix } from "./textualToolCall.ts";
|
||||
import { pickResponsesTextPart, type ResponsesTextPart } from "./responsesTextPart.ts";
|
||||
import { stripObfuscationZeroWidth } from "./zeroWidth.ts";
|
||||
import {
|
||||
formatTranslatedStreamError,
|
||||
@@ -1031,6 +1032,7 @@ export function createSSEStream(options: StreamOptions = {}) {
|
||||
let passthroughAccumulatedContent = "";
|
||||
let passthroughAccumulatedReasoning = "";
|
||||
let passthroughBufferedTextualToolCallContent = "";
|
||||
let passthroughBufferedTextPart: ResponsesTextPart = {}; // #20257: identity of the buffered part
|
||||
/** Passthrough: whether a usage block was already forwarded to the client (prevents double). */
|
||||
let passthroughForwardedUsage = false;
|
||||
let passthroughForwardedUsageSummary = false;
|
||||
@@ -1939,6 +1941,8 @@ export function createSSEStream(options: StreamOptions = {}) {
|
||||
parsed.delta = "";
|
||||
if (collectedToolCall) continue;
|
||||
} else if (parsedCandidate?.kind === "partial") {
|
||||
if (!passthroughBufferedTextualToolCallContent)
|
||||
passthroughBufferedTextPart = pickResponsesTextPart(parsed);
|
||||
passthroughBufferedTextualToolCallContent = appendBoundedText(
|
||||
passthroughBufferedTextualToolCallContent,
|
||||
incomingDelta
|
||||
@@ -3220,6 +3224,7 @@ export function createSSEStream(options: StreamOptions = {}) {
|
||||
item_id: passthroughResponsesId ? `msg_${passthroughResponsesId}_0` : "msg_0",
|
||||
output_index: 0,
|
||||
content_index: 0,
|
||||
...passthroughBufferedTextPart,
|
||||
delta: passthroughBufferedTextualToolCallContent,
|
||||
};
|
||||
flushOutput = `data: ${JSON.stringify(syntheticChunk)}\n\n`;
|
||||
|
||||
@@ -54,3 +54,35 @@ export function flushOpenAiTextBuffers(
|
||||
for (const choice of choices) drainChoice(store, choice);
|
||||
return { ...finalJson, choices };
|
||||
}
|
||||
|
||||
/** Item id for a replayed Responses text tail: the event's own, its item's, else positional. */
|
||||
function responsesTailItemId(lastJson: Record<string, unknown>, idx: number): string {
|
||||
if (typeof lastJson.item_id === "string") return lastJson.item_id;
|
||||
const item = lastJson.item as { id?: unknown } | undefined;
|
||||
return typeof item?.id === "string" ? item.id : `msg_${idx}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* A fresh `output_text.delta` carrying held-back text after a non-delta frame. Strict
|
||||
* Responses clients reject a delta without `item_id`/`content_index` (#20257).
|
||||
*/
|
||||
export function responsesTextTailDelta(
|
||||
lastJson: Record<string, unknown>,
|
||||
idx: number,
|
||||
delta: string
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
type: "response.output_text.delta",
|
||||
item_id: responsesTailItemId(lastJson, idx),
|
||||
output_index: idx,
|
||||
content_index: typeof lastJson.content_index === "number" ? lastJson.content_index : 0,
|
||||
delta,
|
||||
};
|
||||
}
|
||||
|
||||
/** Fill the ids a strict Responses client requires on an `output_text.delta` (#20257). */
|
||||
export function ensureResponsesTextDeltaIds(json: Record<string, unknown>, idx: number): void {
|
||||
if (json.type !== "response.output_text.delta") return;
|
||||
if (json.item_id === undefined) json.item_id = `msg_${idx}`;
|
||||
if (json.content_index === undefined) json.content_index = 0;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { createSseTextTransform, type FieldCategory } from "./sseTextTransform";
|
||||
import {
|
||||
bufferField,
|
||||
ensureResponsesTextDeltaIds,
|
||||
flushOpenAiTextBuffers,
|
||||
responsesTextTailDelta,
|
||||
type BufferedField,
|
||||
type TextBuffers,
|
||||
} from "./sseTextBuffers";
|
||||
@@ -379,19 +381,7 @@ function flushResponsesFormat(
|
||||
buffers.reasoning = "";
|
||||
}
|
||||
if (!parts.length) return null;
|
||||
const itemId =
|
||||
(typeof lastJson.item_id === "string" ? lastJson.item_id : undefined) ??
|
||||
(typeof (lastJson.item as Record<string, unknown> | undefined)?.id === "string"
|
||||
? (lastJson.item as Record<string, unknown>).id
|
||||
: undefined) ??
|
||||
`msg_${idx}`;
|
||||
return {
|
||||
type: "response.output_text.delta",
|
||||
item_id: itemId,
|
||||
output_index: idx,
|
||||
content_index: typeof lastJson.content_index === "number" ? lastJson.content_index : 0,
|
||||
delta: parts.join(""),
|
||||
};
|
||||
return responsesTextTailDelta(lastJson, idx, parts.join(""));
|
||||
}
|
||||
const finalJson = JSON.parse(JSON.stringify(lastJson)) as Record<string, unknown>;
|
||||
if (buffers.content) {
|
||||
@@ -406,14 +396,7 @@ function flushResponsesFormat(
|
||||
finalJson.reasoning = buffers.reasoning;
|
||||
buffers.reasoning = "";
|
||||
}
|
||||
if (finalJson.type === "response.output_text.delta") {
|
||||
if (finalJson.item_id === undefined) {
|
||||
finalJson.item_id = `msg_${idx}`;
|
||||
}
|
||||
if (finalJson.content_index === undefined) {
|
||||
finalJson.content_index = 0;
|
||||
}
|
||||
}
|
||||
ensureResponsesTextDeltaIds(finalJson, idx);
|
||||
return finalJson;
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,9 @@ import { createSseTextTransform, FieldCategory, classifyField } from "./sseTextT
|
||||
import { sanitizePII, getPiiResponseMode } from "./piiSanitizer";
|
||||
import {
|
||||
bufferField,
|
||||
ensureResponsesTextDeltaIds,
|
||||
flushOpenAiTextBuffers,
|
||||
responsesTextTailDelta,
|
||||
type BufferedField,
|
||||
type TextBuffers,
|
||||
} from "./sseTextBuffers";
|
||||
@@ -207,19 +209,7 @@ export function createPiiSseTransform(options?: PiiTransformOptions): TransformS
|
||||
if (!buffers.content) return null;
|
||||
const delta = buffers.content;
|
||||
buffers.content = "";
|
||||
const itemId =
|
||||
(typeof lastJson.item_id === "string" ? lastJson.item_id : undefined) ??
|
||||
(typeof (lastJson.item as Record<string, unknown> | undefined)?.id === "string"
|
||||
? (lastJson.item as Record<string, unknown>).id
|
||||
: undefined) ??
|
||||
`msg_${idx}`;
|
||||
return {
|
||||
type: "response.output_text.delta",
|
||||
item_id: itemId,
|
||||
output_index: idx,
|
||||
content_index: typeof lastJson.content_index === "number" ? lastJson.content_index : 0,
|
||||
delta,
|
||||
};
|
||||
return responsesTextTailDelta(lastJson, idx, delta);
|
||||
}
|
||||
const finalJson = JSON.parse(JSON.stringify(lastJson));
|
||||
if (buffers.content) {
|
||||
@@ -232,14 +222,7 @@ export function createPiiSseTransform(options?: PiiTransformOptions): TransformS
|
||||
};
|
||||
buffers.toolArgs = "";
|
||||
}
|
||||
if (finalJson.type === "response.output_text.delta") {
|
||||
if (finalJson.item_id === undefined) {
|
||||
finalJson.item_id = `msg_${idx}`;
|
||||
}
|
||||
if (finalJson.content_index === undefined) {
|
||||
finalJson.content_index = 0;
|
||||
}
|
||||
}
|
||||
ensureResponsesTextDeltaIds(finalJson, idx);
|
||||
return finalJson;
|
||||
}
|
||||
|
||||
|
||||
@@ -8,8 +8,20 @@
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createCredentialSseTransform } from "../../src/lib/streamingCredentialTransform.ts";
|
||||
import { createPiiSseTransform } from "../../src/lib/streamingPiiTransform.ts";
|
||||
import { createTempDataDir } from "../_setup/tempDataDir.ts";
|
||||
|
||||
// The PII transform resolves its mode through feature-flag overrides, which open the DB
|
||||
// singleton: isolate DATA_DIR before the imports and close the handle afterwards.
|
||||
const { cleanup } = createTempDataDir("omniroute-responses-delta-item-id-");
|
||||
test.after(cleanup);
|
||||
|
||||
const { createCredentialSseTransform } =
|
||||
await import("../../src/lib/streamingCredentialTransform.ts");
|
||||
const { createPiiSseTransform } = await import("../../src/lib/streamingPiiTransform.ts");
|
||||
const { createSSEStream } = await import("../../open-sse/utils/stream.ts");
|
||||
const { createRecoverableStream, TruncatedStreamError } =
|
||||
await import("../../open-sse/services/streamRecovery.ts");
|
||||
const { FORMATS } = await import("../../open-sse/translator/formats.ts");
|
||||
|
||||
type SseEvent = Record<string, unknown>;
|
||||
|
||||
@@ -168,3 +180,96 @@ test("PII transform: response.output_text.delta always includes item_id", async
|
||||
assert.equal(delta.item_id, "msg_pii_123");
|
||||
}
|
||||
});
|
||||
|
||||
function dataEvents(sse: string): SseEvent[] {
|
||||
return sse
|
||||
.split("\n")
|
||||
.filter((line) => line.startsWith("data: ") && line.slice(6).trim() !== "[DONE]")
|
||||
.map((line) => JSON.parse(line.slice(6)) as SseEvent);
|
||||
}
|
||||
|
||||
function sseSource(chunks: string[], cut = false): ReadableStream<Uint8Array> {
|
||||
const encoder = new TextEncoder();
|
||||
let i = 0;
|
||||
return new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (i < chunks.length) controller.enqueue(encoder.encode(chunks[i++]));
|
||||
else if (cut) controller.error(new TruncatedStreamError());
|
||||
else controller.close();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test("passthrough flush of a buffered textual tool-call keeps the source part identity", async () => {
|
||||
const partial = {
|
||||
type: "response.output_text.delta",
|
||||
item_id: "msg_real",
|
||||
output_index: 1,
|
||||
content_index: 2,
|
||||
delta: '[Tool call: terminal]\nArguments: {"command":"ls',
|
||||
};
|
||||
const source = sseSource([`data: ${JSON.stringify(partial)}\n\n`]);
|
||||
const out = await new Response(
|
||||
source.pipeThrough(
|
||||
createSSEStream({
|
||||
mode: "passthrough",
|
||||
sourceFormat: FORMATS.OPENAI_RESPONSES,
|
||||
clientResponseFormat: FORMATS.OPENAI_RESPONSES,
|
||||
provider: "openai",
|
||||
model: "gpt-test",
|
||||
body: {
|
||||
input: "run ls",
|
||||
tools: [{ type: "function", name: "terminal", parameters: { type: "object" } }],
|
||||
},
|
||||
})
|
||||
)
|
||||
).text();
|
||||
|
||||
const flushed = dataEvents(out).find(
|
||||
(e) =>
|
||||
e.type === "response.output_text.delta" &&
|
||||
typeof e.delta === "string" &&
|
||||
e.delta.includes("[Tool call: terminal]")
|
||||
);
|
||||
assert.ok(flushed, `buffered text must be flushed at EOF: ${out}`);
|
||||
assert.equal(flushed.item_id, "msg_real");
|
||||
assert.equal(flushed.output_index, 1);
|
||||
assert.equal(flushed.content_index, 2);
|
||||
});
|
||||
|
||||
test("recovery suffix extends the open upstream text part instead of msg_0", async () => {
|
||||
const delta = (text: string) =>
|
||||
`data: ${JSON.stringify({
|
||||
type: "response.output_text.delta",
|
||||
item_id: "msg_provider_abc",
|
||||
output_index: 1,
|
||||
content_index: 0,
|
||||
delta: text,
|
||||
})}\n\n`;
|
||||
let t = 0;
|
||||
const stream = createRecoverableStream(
|
||||
sseSource([delta("Hello there world")], true),
|
||||
async () => null,
|
||||
{
|
||||
finalize: () => {},
|
||||
now: () => (t += 1000),
|
||||
continueStream: async () =>
|
||||
sseSource([
|
||||
delta("there world, nice to meet you!") +
|
||||
`data: ${JSON.stringify({ type: "response.completed", response: { status: "completed", output: [] } })}\n\n`,
|
||||
]),
|
||||
}
|
||||
);
|
||||
const out = await new Response(stream).text();
|
||||
|
||||
const suffix = dataEvents(out).find(
|
||||
(e) =>
|
||||
e.type === "response.output_text.delta" &&
|
||||
typeof e.delta === "string" &&
|
||||
e.delta.includes("nice to meet you")
|
||||
);
|
||||
assert.ok(suffix, `stitched suffix must be emitted: ${out}`);
|
||||
assert.equal(suffix.item_id, "msg_provider_abc");
|
||||
assert.equal(suffix.output_index, 1);
|
||||
assert.equal(suffix.content_index, 0);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user