Merge PR #16783 (feat/16670-dmr-discovery-1015) into merge train

This commit is contained in:
diegosouzapw
2026-11-13 10:01:04 -03:00
10 changed files with 347 additions and 8 deletions

View File

@@ -0,0 +1 @@
- Use registered local-provider URLs for model discovery and synchronized embedding routes, and align Docker Model Runner's default with its engine-neutral OpenAI endpoint. Docker Model Runner embedding requests now accept keyless connections while preserving configured credentials and per-model capability checks. (#16670)

View File

@@ -0,0 +1,59 @@
---
title: "Docker Model Runner"
version: 3.8.52
lastUpdated: 2026-10-15
---
# Docker Model Runner
Select the existing **Docker Model Runner** provider when adding a local connection.
Its provider ID is `docker-model-runner` and its model alias is `dmr`.
An API key is optional.
## Connection address
The default base URL is `http://localhost:12434/engines/v1`. Discovery, chat,
and capability-enabled embeddings use this registered default when the connection
has no explicit base URL. A configured base URL takes precedence.
Docker documents an engine-neutral OpenAI endpoint and an optional engine-specific
path, such as `http://localhost:12434/engines/llama.cpp/v1`. For Docker Desktop
containers, Docker documents `http://model-runner.docker.internal/engines/v1`.
Choose the address reachable from the process running OmniRoute; `localhost`
inside a container refers to that container. Host TCP access must be enabled when
required by your Docker installation. See the
[Docker Model Runner API reference](https://docs.docker.com/ai/model-runner/api-reference/)
for Docker Desktop and Docker Engine networking requirements.
OmniRoute uses the configured address; it does not scan hosts or start a Docker
daemon automatically.
## Models and requests
Model discovery probes the OpenAI models endpoint and stores the returned model
IDs per connection. The public model catalog preserves names and tags: an upstream
ID such as `ai/smollm2:latest` is exposed as `dmr/ai/smollm2:latest` in alias mode.
The existing OpenAI executor dispatches both JSON and streaming chat requests.
Embeddings require a synchronized model that declares the `embeddings` endpoint,
or an explicit supported-endpoint override for that model. A model listed without
capability metadata is not assumed to support embeddings. The embedding request
uses the same connection's base URL and forwards a configured key, if present.
Connection failures can fall back to the available cached or local catalog with a
warning. A stalled discovery request returns a bounded timeout error; it does not
erase the synchronized model cache. Cached models are not evidence that the daemon
is currently reachable.
## Implementation and validation
The implementation reuses the local-provider discovery route, synchronized model
store, public catalog, and OpenAI executor. Relevant source paths are
[`local.ts`](../../src/shared/constants/providers/local.ts),
[`localProviderBaseUrl.ts`](../../src/shared/utils/localProviderBaseUrl.ts), and
[`service.ts`](../../src/lib/embeddings/service.ts).
Automated regression tests exercise real routing, persistence, capability checks,
and executor transport with controlled upstream responses. They do not establish
compatibility with every Docker version or replace a smoke test against the Docker
Model Runner installation you intend to use.

View File

@@ -399,7 +399,7 @@ Use the dashboard at `/dashboard/providers` to enable, configure, and test each
| ID | Alias | Name | Tags | Website | Notes |
|----|-------|------|------|---------|-------|
| `comfyui` | `comfyui` | ComfyUI | Local | [link](https://github.com/comfyanonymous/ComfyUI) | No API key required. Configure the local ComfyUI base URL (default: http://localhost:8188). |
| `docker-model-runner` | `dmr` | Docker Model Runner | Local, self-hosted | [link](https://docs.docker.com/ai/model-runner/) | API key optional. Configure the local Docker Model Runner OpenAI-compatible base URL (default: http://localhost:12434/v1). |
| `docker-model-runner` | `dmr` | Docker Model Runner | Local, self-hosted | [link](https://docs.docker.com/ai/model-runner/) | API key optional. Configure the local Docker Model Runner OpenAI-compatible base URL (default: http://localhost:12434/engines/v1). |
| `lemonade` | `lemonade` | Lemonade Server | Local, self-hosted | [link](https://lemonade-server.ai) | API key optional. Configure the local Lemonade OpenAI-compatible base URL (default: http://localhost:13305/api/v1). |
| `llama-cpp` | `llamacpp` | llama.cpp | Local, self-hosted | [link](https://github.com/ggml-org/llama.cpp) | API key optional (use any value, e.g. sk-no-key-required). Configure the llama-server OpenAI-compatible base URL (default: http://127.0.0.1:8080/v1). Note: if Llamafile is also installed, both default to port 8080 — run only one at a time or override the port. |
| `llamafile` | `llamafile` | Llamafile | Local, self-hosted | [link](https://github.com/Mozilla-Ocho/llamafile) | API key optional. Configure the local Llamafile OpenAI-compatible base URL (default: http://127.0.0.1:8080/v1). |

View File

@@ -1,3 +1,4 @@
import { resolveLocalProviderBaseUrl } from "@/shared/utils/localProviderBaseUrl";
import { isSelfHostedChatProvider } from "@/shared/constants/providers";
import { getStaticModelsForProvider, type LocalCatalogModel } from "@/lib/providers/staticModels";
import { SAFE_OUTBOUND_FETCH_PRESETS, safeOutboundFetch } from "@/shared/network/safeOutboundFetch";
@@ -19,10 +20,10 @@ export function toNonEmptyString(value: unknown): string | null {
return trimmed.length > 0 ? trimmed : null;
}
export function getProviderBaseUrl(providerSpecificData: unknown): string | null {
export function getProviderBaseUrl(providerSpecificData: unknown, provider = ""): string | null {
const data = asRecord(providerSpecificData);
const baseUrl = data.baseUrl;
return typeof baseUrl === "string" && baseUrl.trim().length > 0 ? baseUrl : null;
return resolveLocalProviderBaseUrl(provider, baseUrl);
}
export function normalizeAzureOpenAIBaseUrl(baseUrl: string) {

View File

@@ -856,7 +856,7 @@ export async function GET(
? getRegistryEntry(provider)
: null;
const rawBaseUrl =
getProviderBaseUrl(connection.providerSpecificData) ||
getProviderBaseUrl(connection.providerSpecificData, provider) ||
(typeof registryEntry?.baseUrl === "string" ? registryEntry.baseUrl : null);
const baseUrl = rawBaseUrl;
if (!baseUrl) {

View File

@@ -1,3 +1,4 @@
import { resolveLocalProviderBaseUrl } from "@/shared/utils/localProviderBaseUrl";
import { handleEmbedding } from "@omniroute/open-sse/handlers/embeddings.ts";
import {
parseEmbeddingModel,
@@ -285,7 +286,7 @@ export async function createEmbeddingResponse(
const providerSpecificData = (credentials as { providerSpecificData?: Record<string, unknown> })
.providerSpecificData;
const configuredBaseUrl = providerSpecificData?.baseUrl;
const configuredBaseUrl = resolveLocalProviderBaseUrl(provider, providerSpecificData?.baseUrl);
if (typeof configuredBaseUrl !== "string" || configuredBaseUrl.trim().length === 0) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,
@@ -300,7 +301,7 @@ export async function createEmbeddingResponse(
providerConfig = {
id: provider,
baseUrl: baseUrl.endsWith("/embeddings") ? baseUrl : `${baseUrl}/embeddings`,
authType: "apikey",
authType: provider === "docker-model-runner" ? "none" : "apikey",
authHeader: "bearer",
models: [],
structuredInputProtocol: registryConfig?.structuredInputProtocol,

View File

@@ -153,8 +153,8 @@ export const LOCAL_PROVIDERS = {
textIcon: "DM",
website: "https://docs.docker.com/ai/model-runner/",
authHint:
"API key optional. Configure the local Docker Model Runner OpenAI-compatible base URL (default: http://localhost:12434/v1).",
localDefault: "http://localhost:12434/v1",
"API key optional. Configure the local Docker Model Runner OpenAI-compatible base URL (default: http://localhost:12434/engines/v1).",
localDefault: "http://localhost:12434/engines/v1",
passthroughModels: true,
},
xinference: {

View File

@@ -0,0 +1,13 @@
import { LOCAL_PROVIDERS } from "@/shared/constants/providers/local";
/** Explicit connection URLs win; registered local defaults need no duplicate configuration. */
export function resolveLocalProviderBaseUrl(
provider: string,
configuredBaseUrl: unknown
): string | null {
if (typeof configuredBaseUrl === "string" && configuredBaseUrl.trim()) {
return configuredBaseUrl;
}
if (!Object.prototype.hasOwnProperty.call(LOCAL_PROVIDERS, provider)) return null;
return LOCAL_PROVIDERS[provider as keyof typeof LOCAL_PROVIDERS].localDefault;
}

View File

@@ -0,0 +1,241 @@
import test from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, mkdirSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
const dataDir = mkdtempSync(join(tmpdir(), "omniroute-dmr-16670-"));
process.env.DATA_DIR = dataDir;
process.env.APP_LOG_TO_FILE = "false";
process.env.OMNIROUTE_PROVIDER_PROBE_TIMEOUT_MS = "1000";
const core = await import("../../src/lib/db/core.ts");
const providers = await import("../../src/lib/db/providers.ts");
const models = await import("../../src/lib/db/models.ts");
const { GET } = await import("../../src/app/api/providers/[id]/models/route.ts");
const { getUnifiedModelsResponse } = await import("../../src/app/api/v1/models/catalog.ts");
const { createEmbeddingResponse } = await import("../../src/lib/embeddings/service.ts");
const { DefaultExecutor } = await import("../../open-sse/executors/default.ts");
const { resolveLocalSyncedEndpointRoute } =
await import("../../src/lib/providerModels/syncedEndpointRouting.ts");
const originalFetch = globalThis.fetch;
const provider = "docker-model-runner";
const defaultBase = "http://localhost:12434/engines/v1";
async function seed(baseUrl?: string) {
return providers.createProviderConnection({
provider,
authType: "apikey",
name: "Docker test",
isActive: true,
testStatus: "active",
providerSpecificData: { autoFetchModels: true, ...(baseUrl ? { baseUrl } : {}) },
});
}
function discover(id: string) {
return GET(new Request(`http://localhost/api/providers/${id}/models?refresh=true`), {
params: { id },
});
}
test.beforeEach(() => {
globalThis.fetch = originalFetch;
core.resetDbInstance();
rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
mkdirSync(dataDir, { recursive: true });
});
test.after(() => {
globalThis.fetch = originalFetch;
core.resetDbInstance();
rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
});
test("DMR discovers its registered default and exposes exact tagged IDs in the public catalog", async () => {
const connection = await seed();
const urls: string[] = [];
globalThis.fetch = async (url) => {
urls.push(String(url));
return Response.json({ data: [{ id: "ai/smollm2:latest", object: "model" }] });
};
const response = await discover(connection.id);
const body = await response.json();
assert.equal(response.status, 200, JSON.stringify(body));
assert.equal(body.source, "api");
assert.deepEqual(urls, [`${defaultBase}/models`]);
const synced = await models.getSyncedAvailableModelsByConnection(provider);
assert.equal(synced[connection.id][0].id, "ai/smollm2:latest");
const catalog = await getUnifiedModelsResponse(
new Request("http://localhost/v1/models?prefix=alias")
);
const publicBody = await catalog.json();
assert.equal(catalog.status, 200);
assert.ok(publicBody.data.some((model: { id: string }) => model.id === "dmr/ai/smollm2:latest"));
});
test("DMR discovery preserves an explicit container engine URL", async () => {
const base = "http://model-runner.docker.internal/engines/llama.cpp/v1";
const connection = await seed(base);
const urls: string[] = [];
globalThis.fetch = async (url) => {
urls.push(String(url));
return Response.json({ data: [{ id: "ai/smollm2:q4" }] });
};
const response = await discover(connection.id);
assert.equal(response.status, 200);
assert.equal((await response.json()).source, "api");
assert.deepEqual(urls, [`${base}/models`]);
});
for (const base of [undefined, "http://127.0.0.1:12435/engines/llama.cpp/v1"]) {
test(`DMR embedding dispatch respects ${base ? "explicit" : "default"} URL after capability discovery`, async () => {
const connection = await seed(base);
await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [
{ id: "ai/embeddinggemma:latest", supportedEndpoints: ["embeddings"] },
]);
const calls: Array<{
url: string;
body: Record<string, unknown>;
authorization: string | null;
}> = [];
globalThis.fetch = async (url, options) => {
calls.push({
url: String(url),
body: JSON.parse(String(options?.body)),
authorization: new Headers(options?.headers).get("authorization"),
});
return Response.json({
data: [{ object: "embedding", index: 0, embedding: [0.1, 0.2] }],
usage: { prompt_tokens: 1, total_tokens: 1 },
});
};
const response = await createEmbeddingResponse({
model: "dmr/ai/embeddinggemma:latest",
input: "hello",
});
const body = await response.json();
assert.equal(response.status, 200, JSON.stringify(body));
assert.deepEqual(body.data[0].embedding, [0.1, 0.2]);
assert.equal(calls.length, 1);
assert.equal(calls[0].url, `${base || defaultBase}/embeddings`);
assert.equal(calls[0].body.model, "ai/embeddinggemma:latest");
assert.equal(calls[0].authorization, null);
});
}
test("DMR does not infer embeddings capability from a chat-only model listing", async () => {
const connection = await seed();
await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [
{ id: "ai/smollm2:latest" },
]);
assert.equal(await resolveLocalSyncedEndpointRoute("dmr/ai/smollm2:latest", "embeddings"), null);
});
test("DMR chat dispatch preserves JSON and SSE bodies through default and engine URLs", async () => {
const executor = new DefaultExecutor(provider);
for (const stream of [false, true]) {
for (const baseUrl of [undefined, "http://127.0.0.1:12435/engines/llama.cpp/v1"]) {
const expectedBase = baseUrl || defaultBase;
const responseBody = stream
? 'data: {"choices":[{"delta":{"content":"hello"}}]}\n\ndata: [DONE]\n\n'
: JSON.stringify({ choices: [{ message: { role: "assistant", content: "hello" } }] });
const calls: Array<{ url: string; body: Record<string, unknown> }> = [];
globalThis.fetch = async (url, options) => {
calls.push({ url: String(url), body: JSON.parse(String(options?.body)) });
return new Response(responseBody, {
headers: { "content-type": stream ? "text/event-stream" : "application/json" },
});
};
const result = await executor.execute({
model: "ai/smollm2:latest",
stream,
credentials: { providerSpecificData: baseUrl ? { baseUrl } : {} },
body: {
model: "ai/smollm2:latest",
stream,
messages: [{ role: "user", content: "hello" }],
},
});
assert.equal(result.response.status, 200);
assert.equal(await result.response.text(), responseBody);
assert.equal(calls.length, 1);
assert.equal(calls[0].url, `${expectedBase}/chat/completions`);
assert.equal(calls[0].body.model, "ai/smollm2:latest");
assert.equal(calls[0].body.stream, stream);
}
}
});
test("DMR optional embedding credentials are forwarded when configured", async () => {
const connection = await providers.createProviderConnection({
provider,
authType: "apikey",
apiKey: "dmr-test-key",
name: "Keyed DMR",
isActive: true,
providerSpecificData: { baseUrl: defaultBase },
});
await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [
{ id: "ai/embeddinggemma:latest", supportedEndpoints: ["embeddings"] },
]);
let authorization: string | null = null;
globalThis.fetch = async (_url, options) => {
authorization = new Headers(options?.headers).get("authorization");
return Response.json({ data: [{ object: "embedding", index: 0, embedding: [0.1] }] });
};
const response = await createEmbeddingResponse({
model: "dmr/ai/embeddinggemma:latest",
input: "hello",
});
assert.equal(response.status, 200, JSON.stringify(await response.json()));
assert.equal(authorization, "Bearer dmr-test-key");
});
test("DMR unavailable discovery retains cached models and terminates after bounded attempts", async () => {
const connection = await seed(defaultBase);
await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [
{ id: "ai/smollm2:cached" },
]);
let calls = 0;
globalThis.fetch = async (_url, options) => {
calls += 1;
assert.ok(options?.signal, "existing probe must carry a timeout signal");
throw new TypeError("fetch failed");
};
const response = await discover(connection.id);
const body = await response.json();
assert.equal(response.status, 200);
assert.notEqual(body.source, "api");
assert.ok(body.models.some((model: { id: string }) => model.id === "ai/smollm2:cached"));
assert.ok(calls > 0 && calls <= 6, `bounded probe attempts: ${calls}`);
assert.equal(
(await models.getSyncedAvailableModelsByConnection(provider))[connection.id][0].id,
"ai/smollm2:cached"
);
});
test("DMR stalled discovery returns a bounded timeout error without erasing cached models", async () => {
const connection = await seed(defaultBase);
await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [
{ id: "ai/smollm2:cached" },
]);
let calls = 0;
globalThis.fetch = async (_url, options) => {
calls += 1;
assert.ok(options?.signal);
await new Promise<void>((_resolve, reject) => {
options.signal?.addEventListener("abort", () => reject(options.signal?.reason), {
once: true,
});
});
throw new Error("timeout signal must reject");
};
const response = await discover(connection.id);
const body = await response.json();
assert.equal(response.status, 504);
assert.equal(typeof body.error, "string");
assert.ok(!body.error.includes("at /"));
assert.ok(calls > 0 && calls <= 2, `bounded timeout attempts: ${calls}`);
assert.equal(
(await models.getSyncedAvailableModelsByConnection(provider))[connection.id][0].id,
"ai/smollm2:cached"
);
});

View File

@@ -0,0 +1,23 @@
import test from "node:test";
import assert from "node:assert/strict";
import { resolveLocalProviderBaseUrl } from "../../src/shared/utils/localProviderBaseUrl.ts";
test("local endpoint resolution keeps explicit URLs authoritative", () => {
for (const provider of ["docker-model-runner", "llama-cpp", "openai", "custom"]) {
assert.equal(
resolveLocalProviderBaseUrl(provider, "http://127.0.0.1:9000/custom/v1"),
"http://127.0.0.1:9000/custom/v1"
);
}
});
test("only own registered local providers receive default endpoints", () => {
assert.equal(
resolveLocalProviderBaseUrl("docker-model-runner", undefined),
"http://localhost:12434/engines/v1"
);
assert.equal(resolveLocalProviderBaseUrl("llama-cpp", " "), "http://127.0.0.1:8080/v1");
for (const provider of ["openai", "unknown", "toString", "__proto__", "constructor"]) {
assert.equal(resolveLocalProviderBaseUrl(provider, undefined), null);
}
});