diff --git a/changelog.d/features/16670-docker-model-runner-discovery.md b/changelog.d/features/16670-docker-model-runner-discovery.md new file mode 100644 index 00000000000..b826466d6e4 --- /dev/null +++ b/changelog.d/features/16670-docker-model-runner-discovery.md @@ -0,0 +1 @@ +- Use registered local-provider URLs for model discovery and synchronized embedding routes, and align Docker Model Runner's default with its engine-neutral OpenAI endpoint. Docker Model Runner embedding requests now accept keyless connections while preserving configured credentials and per-model capability checks. (#16670) diff --git a/docs/guides/DOCKER_MODEL_RUNNER.md b/docs/guides/DOCKER_MODEL_RUNNER.md new file mode 100644 index 00000000000..7df3c566b1d --- /dev/null +++ b/docs/guides/DOCKER_MODEL_RUNNER.md @@ -0,0 +1,59 @@ +--- +title: "Docker Model Runner" +version: 3.8.52 +lastUpdated: 2026-10-15 +--- + +# Docker Model Runner + +Select the existing **Docker Model Runner** provider when adding a local connection. +Its provider ID is `docker-model-runner` and its model alias is `dmr`. +An API key is optional. + +## Connection address + +The default base URL is `http://localhost:12434/engines/v1`. Discovery, chat, +and capability-enabled embeddings use this registered default when the connection +has no explicit base URL. A configured base URL takes precedence. + +Docker documents an engine-neutral OpenAI endpoint and an optional engine-specific +path, such as `http://localhost:12434/engines/llama.cpp/v1`. For Docker Desktop +containers, Docker documents `http://model-runner.docker.internal/engines/v1`. +Choose the address reachable from the process running OmniRoute; `localhost` +inside a container refers to that container. Host TCP access must be enabled when +required by your Docker installation. See the +[Docker Model Runner API reference](https://docs.docker.com/ai/model-runner/api-reference/) +for Docker Desktop and Docker Engine networking requirements. + +OmniRoute uses the configured address; it does not scan hosts or start a Docker +daemon automatically. + +## Models and requests + +Model discovery probes the OpenAI models endpoint and stores the returned model +IDs per connection. The public model catalog preserves names and tags: an upstream +ID such as `ai/smollm2:latest` is exposed as `dmr/ai/smollm2:latest` in alias mode. +The existing OpenAI executor dispatches both JSON and streaming chat requests. + +Embeddings require a synchronized model that declares the `embeddings` endpoint, +or an explicit supported-endpoint override for that model. A model listed without +capability metadata is not assumed to support embeddings. The embedding request +uses the same connection's base URL and forwards a configured key, if present. + +Connection failures can fall back to the available cached or local catalog with a +warning. A stalled discovery request returns a bounded timeout error; it does not +erase the synchronized model cache. Cached models are not evidence that the daemon +is currently reachable. + +## Implementation and validation + +The implementation reuses the local-provider discovery route, synchronized model +store, public catalog, and OpenAI executor. Relevant source paths are +[`local.ts`](../../src/shared/constants/providers/local.ts), +[`localProviderBaseUrl.ts`](../../src/shared/utils/localProviderBaseUrl.ts), and +[`service.ts`](../../src/lib/embeddings/service.ts). + +Automated regression tests exercise real routing, persistence, capability checks, +and executor transport with controlled upstream responses. They do not establish +compatibility with every Docker version or replace a smoke test against the Docker +Model Runner installation you intend to use. diff --git a/docs/reference/PROVIDER_REFERENCE.md b/docs/reference/PROVIDER_REFERENCE.md index b00ec51811b..519fc99af14 100644 --- a/docs/reference/PROVIDER_REFERENCE.md +++ b/docs/reference/PROVIDER_REFERENCE.md @@ -399,7 +399,7 @@ Use the dashboard at `/dashboard/providers` to enable, configure, and test each | ID | Alias | Name | Tags | Website | Notes | |----|-------|------|------|---------|-------| | `comfyui` | `comfyui` | ComfyUI | Local | [link](https://github.com/comfyanonymous/ComfyUI) | No API key required. Configure the local ComfyUI base URL (default: http://localhost:8188). | -| `docker-model-runner` | `dmr` | Docker Model Runner | Local, self-hosted | [link](https://docs.docker.com/ai/model-runner/) | API key optional. Configure the local Docker Model Runner OpenAI-compatible base URL (default: http://localhost:12434/v1). | +| `docker-model-runner` | `dmr` | Docker Model Runner | Local, self-hosted | [link](https://docs.docker.com/ai/model-runner/) | API key optional. Configure the local Docker Model Runner OpenAI-compatible base URL (default: http://localhost:12434/engines/v1). | | `lemonade` | `lemonade` | Lemonade Server | Local, self-hosted | [link](https://lemonade-server.ai) | API key optional. Configure the local Lemonade OpenAI-compatible base URL (default: http://localhost:13305/api/v1). | | `llama-cpp` | `llamacpp` | llama.cpp | Local, self-hosted | [link](https://github.com/ggml-org/llama.cpp) | API key optional (use any value, e.g. sk-no-key-required). Configure the llama-server OpenAI-compatible base URL (default: http://127.0.0.1:8080/v1). Note: if Llamafile is also installed, both default to port 8080 — run only one at a time or override the port. | | `llamafile` | `llamafile` | Llamafile | Local, self-hosted | [link](https://github.com/Mozilla-Ocho/llamafile) | API key optional. Configure the local Llamafile OpenAI-compatible base URL (default: http://127.0.0.1:8080/v1). | diff --git a/src/app/api/providers/[id]/models/discovery/helpers.ts b/src/app/api/providers/[id]/models/discovery/helpers.ts index 7f33dc80af0..124430043a5 100644 --- a/src/app/api/providers/[id]/models/discovery/helpers.ts +++ b/src/app/api/providers/[id]/models/discovery/helpers.ts @@ -1,3 +1,4 @@ +import { resolveLocalProviderBaseUrl } from "@/shared/utils/localProviderBaseUrl"; import { isSelfHostedChatProvider } from "@/shared/constants/providers"; import { getStaticModelsForProvider, type LocalCatalogModel } from "@/lib/providers/staticModels"; import { SAFE_OUTBOUND_FETCH_PRESETS, safeOutboundFetch } from "@/shared/network/safeOutboundFetch"; @@ -19,10 +20,10 @@ export function toNonEmptyString(value: unknown): string | null { return trimmed.length > 0 ? trimmed : null; } -export function getProviderBaseUrl(providerSpecificData: unknown): string | null { +export function getProviderBaseUrl(providerSpecificData: unknown, provider = ""): string | null { const data = asRecord(providerSpecificData); const baseUrl = data.baseUrl; - return typeof baseUrl === "string" && baseUrl.trim().length > 0 ? baseUrl : null; + return resolveLocalProviderBaseUrl(provider, baseUrl); } export function normalizeAzureOpenAIBaseUrl(baseUrl: string) { diff --git a/src/app/api/providers/[id]/models/route.ts b/src/app/api/providers/[id]/models/route.ts index 14ccc0c4a0b..658fb3ba341 100755 --- a/src/app/api/providers/[id]/models/route.ts +++ b/src/app/api/providers/[id]/models/route.ts @@ -856,7 +856,7 @@ export async function GET( ? getRegistryEntry(provider) : null; const rawBaseUrl = - getProviderBaseUrl(connection.providerSpecificData) || + getProviderBaseUrl(connection.providerSpecificData, provider) || (typeof registryEntry?.baseUrl === "string" ? registryEntry.baseUrl : null); const baseUrl = rawBaseUrl; if (!baseUrl) { diff --git a/src/lib/embeddings/service.ts b/src/lib/embeddings/service.ts index e51de56635b..6db93121287 100644 --- a/src/lib/embeddings/service.ts +++ b/src/lib/embeddings/service.ts @@ -1,3 +1,4 @@ +import { resolveLocalProviderBaseUrl } from "@/shared/utils/localProviderBaseUrl"; import { handleEmbedding } from "@omniroute/open-sse/handlers/embeddings.ts"; import { parseEmbeddingModel, @@ -285,7 +286,7 @@ export async function createEmbeddingResponse( const providerSpecificData = (credentials as { providerSpecificData?: Record }) .providerSpecificData; - const configuredBaseUrl = providerSpecificData?.baseUrl; + const configuredBaseUrl = resolveLocalProviderBaseUrl(provider, providerSpecificData?.baseUrl); if (typeof configuredBaseUrl !== "string" || configuredBaseUrl.trim().length === 0) { return errorResponse( HTTP_STATUS.BAD_REQUEST, @@ -300,7 +301,7 @@ export async function createEmbeddingResponse( providerConfig = { id: provider, baseUrl: baseUrl.endsWith("/embeddings") ? baseUrl : `${baseUrl}/embeddings`, - authType: "apikey", + authType: provider === "docker-model-runner" ? "none" : "apikey", authHeader: "bearer", models: [], structuredInputProtocol: registryConfig?.structuredInputProtocol, diff --git a/src/shared/constants/providers/local.ts b/src/shared/constants/providers/local.ts index 64bb04af8e3..ca114a2f12e 100644 --- a/src/shared/constants/providers/local.ts +++ b/src/shared/constants/providers/local.ts @@ -153,8 +153,8 @@ export const LOCAL_PROVIDERS = { textIcon: "DM", website: "https://docs.docker.com/ai/model-runner/", authHint: - "API key optional. Configure the local Docker Model Runner OpenAI-compatible base URL (default: http://localhost:12434/v1).", - localDefault: "http://localhost:12434/v1", + "API key optional. Configure the local Docker Model Runner OpenAI-compatible base URL (default: http://localhost:12434/engines/v1).", + localDefault: "http://localhost:12434/engines/v1", passthroughModels: true, }, xinference: { diff --git a/src/shared/utils/localProviderBaseUrl.ts b/src/shared/utils/localProviderBaseUrl.ts new file mode 100644 index 00000000000..b089c74e5e7 --- /dev/null +++ b/src/shared/utils/localProviderBaseUrl.ts @@ -0,0 +1,13 @@ +import { LOCAL_PROVIDERS } from "@/shared/constants/providers/local"; + +/** Explicit connection URLs win; registered local defaults need no duplicate configuration. */ +export function resolveLocalProviderBaseUrl( + provider: string, + configuredBaseUrl: unknown +): string | null { + if (typeof configuredBaseUrl === "string" && configuredBaseUrl.trim()) { + return configuredBaseUrl; + } + if (!Object.prototype.hasOwnProperty.call(LOCAL_PROVIDERS, provider)) return null; + return LOCAL_PROVIDERS[provider as keyof typeof LOCAL_PROVIDERS].localDefault; +} diff --git a/tests/unit/docker-model-runner-discovery-16670.test.ts b/tests/unit/docker-model-runner-discovery-16670.test.ts new file mode 100644 index 00000000000..3b5306cfa76 --- /dev/null +++ b/tests/unit/docker-model-runner-discovery-16670.test.ts @@ -0,0 +1,241 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const dataDir = mkdtempSync(join(tmpdir(), "omniroute-dmr-16670-")); +process.env.DATA_DIR = dataDir; +process.env.APP_LOG_TO_FILE = "false"; +process.env.OMNIROUTE_PROVIDER_PROBE_TIMEOUT_MS = "1000"; +const core = await import("../../src/lib/db/core.ts"); +const providers = await import("../../src/lib/db/providers.ts"); +const models = await import("../../src/lib/db/models.ts"); +const { GET } = await import("../../src/app/api/providers/[id]/models/route.ts"); +const { getUnifiedModelsResponse } = await import("../../src/app/api/v1/models/catalog.ts"); +const { createEmbeddingResponse } = await import("../../src/lib/embeddings/service.ts"); +const { DefaultExecutor } = await import("../../open-sse/executors/default.ts"); +const { resolveLocalSyncedEndpointRoute } = + await import("../../src/lib/providerModels/syncedEndpointRouting.ts"); +const originalFetch = globalThis.fetch; +const provider = "docker-model-runner"; +const defaultBase = "http://localhost:12434/engines/v1"; + +async function seed(baseUrl?: string) { + return providers.createProviderConnection({ + provider, + authType: "apikey", + name: "Docker test", + isActive: true, + testStatus: "active", + providerSpecificData: { autoFetchModels: true, ...(baseUrl ? { baseUrl } : {}) }, + }); +} +function discover(id: string) { + return GET(new Request(`http://localhost/api/providers/${id}/models?refresh=true`), { + params: { id }, + }); +} + +test.beforeEach(() => { + globalThis.fetch = originalFetch; + core.resetDbInstance(); + rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + mkdirSync(dataDir, { recursive: true }); +}); +test.after(() => { + globalThis.fetch = originalFetch; + core.resetDbInstance(); + rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("DMR discovers its registered default and exposes exact tagged IDs in the public catalog", async () => { + const connection = await seed(); + const urls: string[] = []; + globalThis.fetch = async (url) => { + urls.push(String(url)); + return Response.json({ data: [{ id: "ai/smollm2:latest", object: "model" }] }); + }; + const response = await discover(connection.id); + const body = await response.json(); + assert.equal(response.status, 200, JSON.stringify(body)); + assert.equal(body.source, "api"); + assert.deepEqual(urls, [`${defaultBase}/models`]); + const synced = await models.getSyncedAvailableModelsByConnection(provider); + assert.equal(synced[connection.id][0].id, "ai/smollm2:latest"); + const catalog = await getUnifiedModelsResponse( + new Request("http://localhost/v1/models?prefix=alias") + ); + const publicBody = await catalog.json(); + assert.equal(catalog.status, 200); + assert.ok(publicBody.data.some((model: { id: string }) => model.id === "dmr/ai/smollm2:latest")); +}); + +test("DMR discovery preserves an explicit container engine URL", async () => { + const base = "http://model-runner.docker.internal/engines/llama.cpp/v1"; + const connection = await seed(base); + const urls: string[] = []; + globalThis.fetch = async (url) => { + urls.push(String(url)); + return Response.json({ data: [{ id: "ai/smollm2:q4" }] }); + }; + const response = await discover(connection.id); + assert.equal(response.status, 200); + assert.equal((await response.json()).source, "api"); + assert.deepEqual(urls, [`${base}/models`]); +}); + +for (const base of [undefined, "http://127.0.0.1:12435/engines/llama.cpp/v1"]) { + test(`DMR embedding dispatch respects ${base ? "explicit" : "default"} URL after capability discovery`, async () => { + const connection = await seed(base); + await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [ + { id: "ai/embeddinggemma:latest", supportedEndpoints: ["embeddings"] }, + ]); + const calls: Array<{ + url: string; + body: Record; + authorization: string | null; + }> = []; + globalThis.fetch = async (url, options) => { + calls.push({ + url: String(url), + body: JSON.parse(String(options?.body)), + authorization: new Headers(options?.headers).get("authorization"), + }); + return Response.json({ + data: [{ object: "embedding", index: 0, embedding: [0.1, 0.2] }], + usage: { prompt_tokens: 1, total_tokens: 1 }, + }); + }; + const response = await createEmbeddingResponse({ + model: "dmr/ai/embeddinggemma:latest", + input: "hello", + }); + const body = await response.json(); + assert.equal(response.status, 200, JSON.stringify(body)); + assert.deepEqual(body.data[0].embedding, [0.1, 0.2]); + assert.equal(calls.length, 1); + assert.equal(calls[0].url, `${base || defaultBase}/embeddings`); + assert.equal(calls[0].body.model, "ai/embeddinggemma:latest"); + assert.equal(calls[0].authorization, null); + }); +} + +test("DMR does not infer embeddings capability from a chat-only model listing", async () => { + const connection = await seed(); + await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [ + { id: "ai/smollm2:latest" }, + ]); + assert.equal(await resolveLocalSyncedEndpointRoute("dmr/ai/smollm2:latest", "embeddings"), null); +}); + +test("DMR chat dispatch preserves JSON and SSE bodies through default and engine URLs", async () => { + const executor = new DefaultExecutor(provider); + for (const stream of [false, true]) { + for (const baseUrl of [undefined, "http://127.0.0.1:12435/engines/llama.cpp/v1"]) { + const expectedBase = baseUrl || defaultBase; + const responseBody = stream + ? 'data: {"choices":[{"delta":{"content":"hello"}}]}\n\ndata: [DONE]\n\n' + : JSON.stringify({ choices: [{ message: { role: "assistant", content: "hello" } }] }); + const calls: Array<{ url: string; body: Record }> = []; + globalThis.fetch = async (url, options) => { + calls.push({ url: String(url), body: JSON.parse(String(options?.body)) }); + return new Response(responseBody, { + headers: { "content-type": stream ? "text/event-stream" : "application/json" }, + }); + }; + const result = await executor.execute({ + model: "ai/smollm2:latest", + stream, + credentials: { providerSpecificData: baseUrl ? { baseUrl } : {} }, + body: { + model: "ai/smollm2:latest", + stream, + messages: [{ role: "user", content: "hello" }], + }, + }); + assert.equal(result.response.status, 200); + assert.equal(await result.response.text(), responseBody); + assert.equal(calls.length, 1); + assert.equal(calls[0].url, `${expectedBase}/chat/completions`); + assert.equal(calls[0].body.model, "ai/smollm2:latest"); + assert.equal(calls[0].body.stream, stream); + } + } +}); + +test("DMR optional embedding credentials are forwarded when configured", async () => { + const connection = await providers.createProviderConnection({ + provider, + authType: "apikey", + apiKey: "dmr-test-key", + name: "Keyed DMR", + isActive: true, + providerSpecificData: { baseUrl: defaultBase }, + }); + await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [ + { id: "ai/embeddinggemma:latest", supportedEndpoints: ["embeddings"] }, + ]); + let authorization: string | null = null; + globalThis.fetch = async (_url, options) => { + authorization = new Headers(options?.headers).get("authorization"); + return Response.json({ data: [{ object: "embedding", index: 0, embedding: [0.1] }] }); + }; + const response = await createEmbeddingResponse({ + model: "dmr/ai/embeddinggemma:latest", + input: "hello", + }); + assert.equal(response.status, 200, JSON.stringify(await response.json())); + assert.equal(authorization, "Bearer dmr-test-key"); +}); + +test("DMR unavailable discovery retains cached models and terminates after bounded attempts", async () => { + const connection = await seed(defaultBase); + await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [ + { id: "ai/smollm2:cached" }, + ]); + let calls = 0; + globalThis.fetch = async (_url, options) => { + calls += 1; + assert.ok(options?.signal, "existing probe must carry a timeout signal"); + throw new TypeError("fetch failed"); + }; + const response = await discover(connection.id); + const body = await response.json(); + assert.equal(response.status, 200); + assert.notEqual(body.source, "api"); + assert.ok(body.models.some((model: { id: string }) => model.id === "ai/smollm2:cached")); + assert.ok(calls > 0 && calls <= 6, `bounded probe attempts: ${calls}`); + assert.equal( + (await models.getSyncedAvailableModelsByConnection(provider))[connection.id][0].id, + "ai/smollm2:cached" + ); +}); + +test("DMR stalled discovery returns a bounded timeout error without erasing cached models", async () => { + const connection = await seed(defaultBase); + await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [ + { id: "ai/smollm2:cached" }, + ]); + let calls = 0; + globalThis.fetch = async (_url, options) => { + calls += 1; + assert.ok(options?.signal); + await new Promise((_resolve, reject) => { + options.signal?.addEventListener("abort", () => reject(options.signal?.reason), { + once: true, + }); + }); + throw new Error("timeout signal must reject"); + }; + const response = await discover(connection.id); + const body = await response.json(); + assert.equal(response.status, 504); + assert.equal(typeof body.error, "string"); + assert.ok(!body.error.includes("at /")); + assert.ok(calls > 0 && calls <= 2, `bounded timeout attempts: ${calls}`); + assert.equal( + (await models.getSyncedAvailableModelsByConnection(provider))[connection.id][0].id, + "ai/smollm2:cached" + ); +}); diff --git a/tests/unit/local-provider-base-url.test.ts b/tests/unit/local-provider-base-url.test.ts new file mode 100644 index 00000000000..de338a70234 --- /dev/null +++ b/tests/unit/local-provider-base-url.test.ts @@ -0,0 +1,23 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { resolveLocalProviderBaseUrl } from "../../src/shared/utils/localProviderBaseUrl.ts"; + +test("local endpoint resolution keeps explicit URLs authoritative", () => { + for (const provider of ["docker-model-runner", "llama-cpp", "openai", "custom"]) { + assert.equal( + resolveLocalProviderBaseUrl(provider, "http://127.0.0.1:9000/custom/v1"), + "http://127.0.0.1:9000/custom/v1" + ); + } +}); + +test("only own registered local providers receive default endpoints", () => { + assert.equal( + resolveLocalProviderBaseUrl("docker-model-runner", undefined), + "http://localhost:12434/engines/v1" + ); + assert.equal(resolveLocalProviderBaseUrl("llama-cpp", " "), "http://127.0.0.1:8080/v1"); + for (const provider of ["openai", "unknown", "toString", "__proto__", "constructor"]) { + assert.equal(resolveLocalProviderBaseUrl(provider, undefined), null); + } +});