fix(providers): devin-desktop forward image_url parts to the upstream

messageText() only extracted text content parts, so client images were
silently dropped and vision-capable models answered text-only. ChatMessagePrompt
supports repeated ImageData (field 10: base64_data=1, mime_type=2, uri=4) in
exa.api_server_pb — encode data: URIs inline and pass http(s) URLs as uri.

Adds a regression test asserting the base64 payload and mime type appear in
the encoded request body.
This commit is contained in:
wusaki0723
2026-10-15 19:00:45 +08:00
parent 03a6595e99
commit a1429bd466
2 changed files with 89 additions and 0 deletions

View File

@@ -99,6 +99,7 @@ export type DevinDesktopPromptInput = {
prompt: string;
toolCalls?: DevinDesktopToolCallInput[];
toolCallId?: string;
images?: DevinDesktopImageInput[];
};
export type DevinDesktopToolInput = {
@@ -168,6 +169,13 @@ function encodeChatMessagePrompt(prompt: DevinDesktopPromptInput): Uint8Array {
fields.push(encodeField(6, encodeChatToolCall(toolCall)));
}
fields.push(encodeString(7, prompt.toolCallId ?? ""));
for (const image of prompt.images ?? []) {
const imageFields: Uint8Array[] = [];
if (image.base64Data) imageFields.push(encodeString(1, image.base64Data));
if (image.mimeType) imageFields.push(encodeString(2, image.mimeType));
if (image.uri) imageFields.push(encodeString(4, image.uri));
fields.push(encodeField(10, concatBytes(imageFields)));
}
return concatBytes(fields);
}
@@ -227,6 +235,34 @@ type OpenAIMessage = {
}>;
};
export type DevinDesktopImageInput = {
base64Data?: string;
mimeType?: string;
uri?: string;
};
function messageImages(content: unknown): DevinDesktopImageInput[] {
if (!Array.isArray(content)) return [];
const images: DevinDesktopImageInput[] = [];
for (const part of content) {
if (!part || typeof part !== "object") continue;
const p = part as Record<string, unknown>;
if (p.type !== "image_url") continue;
const imageUrl =
p.image_url && typeof p.image_url === "object"
? (p.image_url as Record<string, unknown>).url
: p.image_url;
if (typeof imageUrl !== "string" || !imageUrl) continue;
const dataUri = /^data:([^;,]+);base64,(.+)$/s.exec(imageUrl);
if (dataUri) {
images.push({ base64Data: dataUri[2], mimeType: dataUri[1] });
} else if (/^https?:\/\//.test(imageUrl)) {
images.push({ uri: imageUrl });
}
}
return images;
}
function messageText(content: unknown): string {
if (typeof content === "string") return content;
if (!Array.isArray(content)) return "";
@@ -282,6 +318,10 @@ function convertMessages(messages: OpenAIMessage[]): {
prompt,
...(source === 2 ? { toolCalls: convertHistoryToolCalls(message.tool_calls) } : {}),
...(source === 4 && message.tool_call_id ? { toolCallId: message.tool_call_id } : {}),
...(() => {
const images = messageImages(message.content);
return images.length ? { images } : {};
})(),
});
}
return { systemPrompt: systemParts.join("\n\n"), prompts };

View File

@@ -283,3 +283,52 @@ test("Devin Desktop reassembles fragmented tool call arguments", async () => {
globalThis.fetch = originalFetch;
}
});
// ─── Devin Desktop: image parts are forwarded as ImageData (field 10) ───────
// Previously messageText() dropped every non-text content part, so client
// images never reached the upstream and vision prompts returned text-only
// answers ("无法确定" for images that were never sent).
test("Devin Desktop encodes image_url parts into the request", async () => {
const executor = await getExecutor("devin-desktop");
const originalFetch = globalThis.fetch;
let requestBody: Uint8Array | null = null;
globalThis.fetch = async (url, init) => {
if (String(url).endsWith("/exa.auth_pb.AuthService/GetUserJwt")) {
const jwt = new TextEncoder().encode("test-jwt");
return new Response(new Uint8Array([0x0a, jwt.length, ...jwt]));
}
requestBody = new Uint8Array(init?.body as ArrayBuffer);
return new Response("expected test stop", { status: 418 });
};
const imageB64 = "aGVsbG8tdmlzaW9uLWJsb2I="; // stable marker blob
try {
await executor.execute({
model: "swe-2-medium",
body: {
messages: [
{
role: "user",
content: [
{ type: "text", text: "what is in this image?" },
{ type: "image_url", image_url: { url: `data:image/png;base64,${imageB64}` } },
],
},
],
},
stream: true,
credentials: { accessToken: "test-devin-desktop-token" },
});
assert.ok(requestBody);
const bodyText = new TextDecoder().decode(requestBody);
assert.ok(
bodyText.includes(imageB64),
"image base64 payload must be present in the upstream request"
);
assert.ok(bodyText.includes("image/png"), "image mime type must be present");
} finally {
globalThis.fetch = originalFetch;
}
});