fix(sse): reuse the upstream part identity on synthesized Responses text deltas

- stream.ts: remember the item_id/output_index/content_index of the delta
  that started a buffered textual tool-call candidate and reuse them when
  the tail is flushed at EOF, instead of a fabricated msg id at 0/0.
- streamRecovery.ts: track the last Responses text delta identity and
  stitch the continuation suffix onto that open part instead of msg_0.
- Move the replayed-tail delta builder and the id backfill shared by the
  credential and PII transforms into sseTextBuffers.ts, which keeps
  flushResponsesFormat under the complexity ratchet.
- Test isolates DATA_DIR and closes the DB handle after the PII case.
This commit is contained in:
Rafa Martins
2026-11-18 21:50:14 -03:00
committed by diegosouzapw
parent 21fdb04ae5
commit 2684763dd5
7 changed files with 189 additions and 45 deletions

View File

@@ -12,6 +12,7 @@
*/
import { STREAM_RECOVERY } from "../config/constants.ts";
import { isRegisteredFeatureFlagEnabled } from "../utils/registeredFeatureFlag.ts";
import { pickResponsesTextPart, type ResponsesTextPart } from "../utils/responsesTextPart.ts";
import {
createThroughputWatchdog,
ThroughputWatchdogError,
@@ -268,6 +269,8 @@ export interface OpenAiSseScan {
* signals, or a terminal lifecycle event.
*/
parsedResponses: boolean;
/** Identity of the last Responses text delta, or null if none was seen (#20257). */
textPart: ResponsesTextPart | null;
}
/** Responses-API `type` values the scanner understands. Anything else is ignored. */
@@ -297,6 +300,7 @@ export function scanOpenAiSseText(sse: string): OpenAiSseScan {
finishReason: null,
parsedOpenAi: false,
parsedResponses: false,
textPart: null,
};
if (typeof sse !== "string" || sse.length === 0) {
return {
@@ -308,6 +312,7 @@ export function scanOpenAiSseText(sse: string): OpenAiSseScan {
finishReason: fold.finishReason,
parsedOpenAi: fold.parsedOpenAi,
parsedResponses: fold.parsedResponses,
textPart: fold.textPart,
};
}
for (const line of sse.split("\n")) {
@@ -323,6 +328,7 @@ export function scanOpenAiSseText(sse: string): OpenAiSseScan {
finishReason: fold.finishReason,
parsedOpenAi: fold.parsedOpenAi,
parsedResponses: fold.parsedResponses,
textPart: fold.textPart,
};
}
@@ -335,6 +341,7 @@ interface SseScanFold {
finishReason: string | null;
parsedOpenAi: boolean;
parsedResponses: boolean;
textPart: ResponsesTextPart | null;
}
/** Parse one raw SSE line into the running scan fold. */
@@ -366,6 +373,9 @@ function scanSseLine(line: string, fold: SseScanFold): void {
addText: (delta) => {
fold.text += delta;
},
setTextPart: (part) => {
fold.textPart = part;
},
addReasoning: (delta) => {
fold.reasoningText += delta;
},
@@ -415,6 +425,7 @@ function scanChatFinishReason(rawFinishReason: unknown, fold: SseScanFold): void
interface ResponsesScanSink {
addText: (delta: string) => void;
setTextPart: (part: ResponsesTextPart) => void;
addReasoning: (delta: string) => void;
markToolCall: () => void;
markToolCallFinished: () => void;
@@ -468,6 +479,7 @@ function scanResponsesTextDelta(
eventType: string,
sink: ResponsesScanSink
): void {
if (eventType === RESPONSES_TEXT_DELTA) sink.setTextPart(pickResponsesTextPart(json));
const delta = (json as { delta?: unknown }).delta;
if (typeof delta !== "string" || delta.length === 0) {
sink.markParsed();
@@ -677,6 +689,7 @@ export function createRecoverableStream(
let emittedToolCallFinish = false; // any finish_reason "tool_calls" seen
let emittedParsedOpenAi = false;
let emittedParsedResponses = false;
let emittedTextPart: ResponsesTextPart = {}; // open Responses text part a suffix extends
// STREAM_RECOVERY_TOOLCALL_ORDER_FIX, resolved lazily at most once per stream and only
// on a recovery decision, so the flag costs nothing on streams that end cleanly.
let toolCallOrderFix: boolean | undefined;
@@ -705,6 +718,7 @@ export function createRecoverableStream(
if (scan.finishReason === "tool_calls") emittedToolCallFinish = true;
if (scan.parsedOpenAi) emittedParsedOpenAi = true;
if (scan.parsedResponses) emittedParsedResponses = true;
if (scan.textPart) emittedTextPart = scan.textPart;
};
const flushHeld = (controller: ReadableStreamDefaultController<Uint8Array>) => {
@@ -963,7 +977,7 @@ export function createRecoverableStream(
emit(
controller,
encoder.encode(
`data: ${JSON.stringify({ type: "response.output_text.delta", item_id: "msg_0", output_index: 0, content_index: 0, delta: suffix })}\n\n`
`data: ${JSON.stringify({ type: "response.output_text.delta", item_id: "msg_0", output_index: 0, content_index: 0, ...emittedTextPart, delta: suffix })}\n\n`
)
);
} else {

View File

@@ -0,0 +1,22 @@
/**
* Wire identity of a Responses-API `output_text` content part. Strict Responses clients
* correlate `response.output_text.delta` events by these fields, so a delta OmniRoute
* synthesizes later (a held-back tail, a recovery suffix) must reuse the identity of the
* upstream part it extends instead of inventing one (#20257).
*/
export interface ResponsesTextPart {
item_id?: string;
output_index?: number;
content_index?: number;
}
/** Copy only the well-typed identity fields of a Responses text event. */
export function pickResponsesTextPart(event: unknown): ResponsesTextPart {
const part: ResponsesTextPart = {};
if (!event || typeof event !== "object") return part;
const record = event as Record<string, unknown>;
if (typeof record.item_id === "string" && record.item_id) part.item_id = record.item_id;
if (typeof record.output_index === "number") part.output_index = record.output_index;
if (typeof record.content_index === "number") part.content_index = record.content_index;
return part;
}

View File

@@ -67,6 +67,7 @@ import {
} from "./responsesCommentaryDrop.ts";
import { buildErrorBody } from "./error.ts";
import { parseTextualToolCallCandidate, isValidToolCallHeaderPrefix } from "./textualToolCall.ts";
import { pickResponsesTextPart, type ResponsesTextPart } from "./responsesTextPart.ts";
import { stripObfuscationZeroWidth } from "./zeroWidth.ts";
import {
formatTranslatedStreamError,
@@ -1031,6 +1032,7 @@ export function createSSEStream(options: StreamOptions = {}) {
let passthroughAccumulatedContent = "";
let passthroughAccumulatedReasoning = "";
let passthroughBufferedTextualToolCallContent = "";
let passthroughBufferedTextPart: ResponsesTextPart = {}; // #20257: identity of the buffered part
/** Passthrough: whether a usage block was already forwarded to the client (prevents double). */
let passthroughForwardedUsage = false;
let passthroughForwardedUsageSummary = false;
@@ -1939,6 +1941,8 @@ export function createSSEStream(options: StreamOptions = {}) {
parsed.delta = "";
if (collectedToolCall) continue;
} else if (parsedCandidate?.kind === "partial") {
if (!passthroughBufferedTextualToolCallContent)
passthroughBufferedTextPart = pickResponsesTextPart(parsed);
passthroughBufferedTextualToolCallContent = appendBoundedText(
passthroughBufferedTextualToolCallContent,
incomingDelta
@@ -3220,6 +3224,7 @@ export function createSSEStream(options: StreamOptions = {}) {
item_id: passthroughResponsesId ? `msg_${passthroughResponsesId}_0` : "msg_0",
output_index: 0,
content_index: 0,
...passthroughBufferedTextPart,
delta: passthroughBufferedTextualToolCallContent,
};
flushOutput = `data: ${JSON.stringify(syntheticChunk)}\n\n`;

View File

@@ -54,3 +54,35 @@ export function flushOpenAiTextBuffers(
for (const choice of choices) drainChoice(store, choice);
return { ...finalJson, choices };
}
/** Item id for a replayed Responses text tail: the event's own, its item's, else positional. */
function responsesTailItemId(lastJson: Record<string, unknown>, idx: number): string {
if (typeof lastJson.item_id === "string") return lastJson.item_id;
const item = lastJson.item as { id?: unknown } | undefined;
return typeof item?.id === "string" ? item.id : `msg_${idx}`;
}
/**
* A fresh `output_text.delta` carrying held-back text after a non-delta frame. Strict
* Responses clients reject a delta without `item_id`/`content_index` (#20257).
*/
export function responsesTextTailDelta(
lastJson: Record<string, unknown>,
idx: number,
delta: string
): Record<string, unknown> {
return {
type: "response.output_text.delta",
item_id: responsesTailItemId(lastJson, idx),
output_index: idx,
content_index: typeof lastJson.content_index === "number" ? lastJson.content_index : 0,
delta,
};
}
/** Fill the ids a strict Responses client requires on an `output_text.delta` (#20257). */
export function ensureResponsesTextDeltaIds(json: Record<string, unknown>, idx: number): void {
if (json.type !== "response.output_text.delta") return;
if (json.item_id === undefined) json.item_id = `msg_${idx}`;
if (json.content_index === undefined) json.content_index = 0;
}

View File

@@ -1,7 +1,9 @@
import { createSseTextTransform, type FieldCategory } from "./sseTextTransform";
import {
bufferField,
ensureResponsesTextDeltaIds,
flushOpenAiTextBuffers,
responsesTextTailDelta,
type BufferedField,
type TextBuffers,
} from "./sseTextBuffers";
@@ -379,19 +381,7 @@ function flushResponsesFormat(
buffers.reasoning = "";
}
if (!parts.length) return null;
const itemId =
(typeof lastJson.item_id === "string" ? lastJson.item_id : undefined) ??
(typeof (lastJson.item as Record<string, unknown> | undefined)?.id === "string"
? (lastJson.item as Record<string, unknown>).id
: undefined) ??
`msg_${idx}`;
return {
type: "response.output_text.delta",
item_id: itemId,
output_index: idx,
content_index: typeof lastJson.content_index === "number" ? lastJson.content_index : 0,
delta: parts.join(""),
};
return responsesTextTailDelta(lastJson, idx, parts.join(""));
}
const finalJson = JSON.parse(JSON.stringify(lastJson)) as Record<string, unknown>;
if (buffers.content) {
@@ -406,14 +396,7 @@ function flushResponsesFormat(
finalJson.reasoning = buffers.reasoning;
buffers.reasoning = "";
}
if (finalJson.type === "response.output_text.delta") {
if (finalJson.item_id === undefined) {
finalJson.item_id = `msg_${idx}`;
}
if (finalJson.content_index === undefined) {
finalJson.content_index = 0;
}
}
ensureResponsesTextDeltaIds(finalJson, idx);
return finalJson;
}

View File

@@ -3,7 +3,9 @@ import { createSseTextTransform, FieldCategory, classifyField } from "./sseTextT
import { sanitizePII, getPiiResponseMode } from "./piiSanitizer";
import {
bufferField,
ensureResponsesTextDeltaIds,
flushOpenAiTextBuffers,
responsesTextTailDelta,
type BufferedField,
type TextBuffers,
} from "./sseTextBuffers";
@@ -207,19 +209,7 @@ export function createPiiSseTransform(options?: PiiTransformOptions): TransformS
if (!buffers.content) return null;
const delta = buffers.content;
buffers.content = "";
const itemId =
(typeof lastJson.item_id === "string" ? lastJson.item_id : undefined) ??
(typeof (lastJson.item as Record<string, unknown> | undefined)?.id === "string"
? (lastJson.item as Record<string, unknown>).id
: undefined) ??
`msg_${idx}`;
return {
type: "response.output_text.delta",
item_id: itemId,
output_index: idx,
content_index: typeof lastJson.content_index === "number" ? lastJson.content_index : 0,
delta,
};
return responsesTextTailDelta(lastJson, idx, delta);
}
const finalJson = JSON.parse(JSON.stringify(lastJson));
if (buffers.content) {
@@ -232,14 +222,7 @@ export function createPiiSseTransform(options?: PiiTransformOptions): TransformS
};
buffers.toolArgs = "";
}
if (finalJson.type === "response.output_text.delta") {
if (finalJson.item_id === undefined) {
finalJson.item_id = `msg_${idx}`;
}
if (finalJson.content_index === undefined) {
finalJson.content_index = 0;
}
}
ensureResponsesTextDeltaIds(finalJson, idx);
return finalJson;
}

View File

@@ -8,8 +8,20 @@
import test from "node:test";
import assert from "node:assert/strict";
import { createCredentialSseTransform } from "../../src/lib/streamingCredentialTransform.ts";
import { createPiiSseTransform } from "../../src/lib/streamingPiiTransform.ts";
import { createTempDataDir } from "../_setup/tempDataDir.ts";
// The PII transform resolves its mode through feature-flag overrides, which open the DB
// singleton: isolate DATA_DIR before the imports and close the handle afterwards.
const { cleanup } = createTempDataDir("omniroute-responses-delta-item-id-");
test.after(cleanup);
const { createCredentialSseTransform } =
await import("../../src/lib/streamingCredentialTransform.ts");
const { createPiiSseTransform } = await import("../../src/lib/streamingPiiTransform.ts");
const { createSSEStream } = await import("../../open-sse/utils/stream.ts");
const { createRecoverableStream, TruncatedStreamError } =
await import("../../open-sse/services/streamRecovery.ts");
const { FORMATS } = await import("../../open-sse/translator/formats.ts");
type SseEvent = Record<string, unknown>;
@@ -168,3 +180,96 @@ test("PII transform: response.output_text.delta always includes item_id", async
assert.equal(delta.item_id, "msg_pii_123");
}
});
function dataEvents(sse: string): SseEvent[] {
return sse
.split("\n")
.filter((line) => line.startsWith("data: ") && line.slice(6).trim() !== "[DONE]")
.map((line) => JSON.parse(line.slice(6)) as SseEvent);
}
function sseSource(chunks: string[], cut = false): ReadableStream<Uint8Array> {
const encoder = new TextEncoder();
let i = 0;
return new ReadableStream<Uint8Array>({
pull(controller) {
if (i < chunks.length) controller.enqueue(encoder.encode(chunks[i++]));
else if (cut) controller.error(new TruncatedStreamError());
else controller.close();
},
});
}
test("passthrough flush of a buffered textual tool-call keeps the source part identity", async () => {
const partial = {
type: "response.output_text.delta",
item_id: "msg_real",
output_index: 1,
content_index: 2,
delta: '[Tool call: terminal]\nArguments: {"command":"ls',
};
const source = sseSource([`data: ${JSON.stringify(partial)}\n\n`]);
const out = await new Response(
source.pipeThrough(
createSSEStream({
mode: "passthrough",
sourceFormat: FORMATS.OPENAI_RESPONSES,
clientResponseFormat: FORMATS.OPENAI_RESPONSES,
provider: "openai",
model: "gpt-test",
body: {
input: "run ls",
tools: [{ type: "function", name: "terminal", parameters: { type: "object" } }],
},
})
)
).text();
const flushed = dataEvents(out).find(
(e) =>
e.type === "response.output_text.delta" &&
typeof e.delta === "string" &&
e.delta.includes("[Tool call: terminal]")
);
assert.ok(flushed, `buffered text must be flushed at EOF: ${out}`);
assert.equal(flushed.item_id, "msg_real");
assert.equal(flushed.output_index, 1);
assert.equal(flushed.content_index, 2);
});
test("recovery suffix extends the open upstream text part instead of msg_0", async () => {
const delta = (text: string) =>
`data: ${JSON.stringify({
type: "response.output_text.delta",
item_id: "msg_provider_abc",
output_index: 1,
content_index: 0,
delta: text,
})}\n\n`;
let t = 0;
const stream = createRecoverableStream(
sseSource([delta("Hello there world")], true),
async () => null,
{
finalize: () => {},
now: () => (t += 1000),
continueStream: async () =>
sseSource([
delta("there world, nice to meet you!") +
`data: ${JSON.stringify({ type: "response.completed", response: { status: "completed", output: [] } })}\n\n`,
]),
}
);
const out = await new Response(stream).text();
const suffix = dataEvents(out).find(
(e) =>
e.type === "response.output_text.delta" &&
typeof e.delta === "string" &&
e.delta.includes("nice to meet you")
);
assert.ok(suffix, `stitched suffix must be emitted: ${out}`);
assert.equal(suffix.item_id, "msg_provider_abc");
assert.equal(suffix.output_index, 1);
assert.equal(suffix.content_index, 0);
});