mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-18 07:58:27 +00:00
fix(providers): devin-desktop forward image_url parts to the upstream
messageText() only extracted text content parts, so client images were silently dropped and vision-capable models answered text-only. ChatMessagePrompt supports repeated ImageData (field 10: base64_data=1, mime_type=2, uri=4) in exa.api_server_pb — encode data: URIs inline and pass http(s) URLs as uri. Adds a regression test asserting the base64 payload and mime type appear in the encoded request body.
This commit is contained in:
@@ -99,6 +99,7 @@ export type DevinDesktopPromptInput = {
|
||||
prompt: string;
|
||||
toolCalls?: DevinDesktopToolCallInput[];
|
||||
toolCallId?: string;
|
||||
images?: DevinDesktopImageInput[];
|
||||
};
|
||||
|
||||
export type DevinDesktopToolInput = {
|
||||
@@ -168,6 +169,13 @@ function encodeChatMessagePrompt(prompt: DevinDesktopPromptInput): Uint8Array {
|
||||
fields.push(encodeField(6, encodeChatToolCall(toolCall)));
|
||||
}
|
||||
fields.push(encodeString(7, prompt.toolCallId ?? ""));
|
||||
for (const image of prompt.images ?? []) {
|
||||
const imageFields: Uint8Array[] = [];
|
||||
if (image.base64Data) imageFields.push(encodeString(1, image.base64Data));
|
||||
if (image.mimeType) imageFields.push(encodeString(2, image.mimeType));
|
||||
if (image.uri) imageFields.push(encodeString(4, image.uri));
|
||||
fields.push(encodeField(10, concatBytes(imageFields)));
|
||||
}
|
||||
return concatBytes(fields);
|
||||
}
|
||||
|
||||
@@ -227,6 +235,34 @@ type OpenAIMessage = {
|
||||
}>;
|
||||
};
|
||||
|
||||
export type DevinDesktopImageInput = {
|
||||
base64Data?: string;
|
||||
mimeType?: string;
|
||||
uri?: string;
|
||||
};
|
||||
|
||||
function messageImages(content: unknown): DevinDesktopImageInput[] {
|
||||
if (!Array.isArray(content)) return [];
|
||||
const images: DevinDesktopImageInput[] = [];
|
||||
for (const part of content) {
|
||||
if (!part || typeof part !== "object") continue;
|
||||
const p = part as Record<string, unknown>;
|
||||
if (p.type !== "image_url") continue;
|
||||
const imageUrl =
|
||||
p.image_url && typeof p.image_url === "object"
|
||||
? (p.image_url as Record<string, unknown>).url
|
||||
: p.image_url;
|
||||
if (typeof imageUrl !== "string" || !imageUrl) continue;
|
||||
const dataUri = /^data:([^;,]+);base64,(.+)$/s.exec(imageUrl);
|
||||
if (dataUri) {
|
||||
images.push({ base64Data: dataUri[2], mimeType: dataUri[1] });
|
||||
} else if (/^https?:\/\//.test(imageUrl)) {
|
||||
images.push({ uri: imageUrl });
|
||||
}
|
||||
}
|
||||
return images;
|
||||
}
|
||||
|
||||
function messageText(content: unknown): string {
|
||||
if (typeof content === "string") return content;
|
||||
if (!Array.isArray(content)) return "";
|
||||
@@ -282,6 +318,10 @@ function convertMessages(messages: OpenAIMessage[]): {
|
||||
prompt,
|
||||
...(source === 2 ? { toolCalls: convertHistoryToolCalls(message.tool_calls) } : {}),
|
||||
...(source === 4 && message.tool_call_id ? { toolCallId: message.tool_call_id } : {}),
|
||||
...(() => {
|
||||
const images = messageImages(message.content);
|
||||
return images.length ? { images } : {};
|
||||
})(),
|
||||
});
|
||||
}
|
||||
return { systemPrompt: systemParts.join("\n\n"), prompts };
|
||||
|
||||
@@ -283,3 +283,52 @@ test("Devin Desktop reassembles fragmented tool call arguments", async () => {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
// ─── Devin Desktop: image parts are forwarded as ImageData (field 10) ───────
|
||||
// Previously messageText() dropped every non-text content part, so client
|
||||
// images never reached the upstream and vision prompts returned text-only
|
||||
// answers ("无法确定" for images that were never sent).
|
||||
|
||||
test("Devin Desktop encodes image_url parts into the request", async () => {
|
||||
const executor = await getExecutor("devin-desktop");
|
||||
const originalFetch = globalThis.fetch;
|
||||
let requestBody: Uint8Array | null = null;
|
||||
globalThis.fetch = async (url, init) => {
|
||||
if (String(url).endsWith("/exa.auth_pb.AuthService/GetUserJwt")) {
|
||||
const jwt = new TextEncoder().encode("test-jwt");
|
||||
return new Response(new Uint8Array([0x0a, jwt.length, ...jwt]));
|
||||
}
|
||||
requestBody = new Uint8Array(init?.body as ArrayBuffer);
|
||||
return new Response("expected test stop", { status: 418 });
|
||||
};
|
||||
|
||||
const imageB64 = "aGVsbG8tdmlzaW9uLWJsb2I="; // stable marker blob
|
||||
try {
|
||||
await executor.execute({
|
||||
model: "swe-2-medium",
|
||||
body: {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "what is in this image?" },
|
||||
{ type: "image_url", image_url: { url: `data:image/png;base64,${imageB64}` } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
stream: true,
|
||||
credentials: { accessToken: "test-devin-desktop-token" },
|
||||
});
|
||||
|
||||
assert.ok(requestBody);
|
||||
const bodyText = new TextDecoder().decode(requestBody);
|
||||
assert.ok(
|
||||
bodyText.includes(imageB64),
|
||||
"image base64 payload must be present in the upstream request"
|
||||
);
|
||||
assert.ok(bodyText.includes("image/png"), "image mime type must be present");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user