mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-16 17:09:09 +00:00
Merge PR #16783 (feat/16670-dmr-discovery-1015) into merge train
This commit is contained in:
@@ -0,0 +1 @@
|
||||
- Use registered local-provider URLs for model discovery and synchronized embedding routes, and align Docker Model Runner's default with its engine-neutral OpenAI endpoint. Docker Model Runner embedding requests now accept keyless connections while preserving configured credentials and per-model capability checks. (#16670)
|
||||
59
docs/guides/DOCKER_MODEL_RUNNER.md
Normal file
59
docs/guides/DOCKER_MODEL_RUNNER.md
Normal file
@@ -0,0 +1,59 @@
|
||||
---
|
||||
title: "Docker Model Runner"
|
||||
version: 3.8.52
|
||||
lastUpdated: 2026-10-15
|
||||
---
|
||||
|
||||
# Docker Model Runner
|
||||
|
||||
Select the existing **Docker Model Runner** provider when adding a local connection.
|
||||
Its provider ID is `docker-model-runner` and its model alias is `dmr`.
|
||||
An API key is optional.
|
||||
|
||||
## Connection address
|
||||
|
||||
The default base URL is `http://localhost:12434/engines/v1`. Discovery, chat,
|
||||
and capability-enabled embeddings use this registered default when the connection
|
||||
has no explicit base URL. A configured base URL takes precedence.
|
||||
|
||||
Docker documents an engine-neutral OpenAI endpoint and an optional engine-specific
|
||||
path, such as `http://localhost:12434/engines/llama.cpp/v1`. For Docker Desktop
|
||||
containers, Docker documents `http://model-runner.docker.internal/engines/v1`.
|
||||
Choose the address reachable from the process running OmniRoute; `localhost`
|
||||
inside a container refers to that container. Host TCP access must be enabled when
|
||||
required by your Docker installation. See the
|
||||
[Docker Model Runner API reference](https://docs.docker.com/ai/model-runner/api-reference/)
|
||||
for Docker Desktop and Docker Engine networking requirements.
|
||||
|
||||
OmniRoute uses the configured address; it does not scan hosts or start a Docker
|
||||
daemon automatically.
|
||||
|
||||
## Models and requests
|
||||
|
||||
Model discovery probes the OpenAI models endpoint and stores the returned model
|
||||
IDs per connection. The public model catalog preserves names and tags: an upstream
|
||||
ID such as `ai/smollm2:latest` is exposed as `dmr/ai/smollm2:latest` in alias mode.
|
||||
The existing OpenAI executor dispatches both JSON and streaming chat requests.
|
||||
|
||||
Embeddings require a synchronized model that declares the `embeddings` endpoint,
|
||||
or an explicit supported-endpoint override for that model. A model listed without
|
||||
capability metadata is not assumed to support embeddings. The embedding request
|
||||
uses the same connection's base URL and forwards a configured key, if present.
|
||||
|
||||
Connection failures can fall back to the available cached or local catalog with a
|
||||
warning. A stalled discovery request returns a bounded timeout error; it does not
|
||||
erase the synchronized model cache. Cached models are not evidence that the daemon
|
||||
is currently reachable.
|
||||
|
||||
## Implementation and validation
|
||||
|
||||
The implementation reuses the local-provider discovery route, synchronized model
|
||||
store, public catalog, and OpenAI executor. Relevant source paths are
|
||||
[`local.ts`](../../src/shared/constants/providers/local.ts),
|
||||
[`localProviderBaseUrl.ts`](../../src/shared/utils/localProviderBaseUrl.ts), and
|
||||
[`service.ts`](../../src/lib/embeddings/service.ts).
|
||||
|
||||
Automated regression tests exercise real routing, persistence, capability checks,
|
||||
and executor transport with controlled upstream responses. They do not establish
|
||||
compatibility with every Docker version or replace a smoke test against the Docker
|
||||
Model Runner installation you intend to use.
|
||||
@@ -399,7 +399,7 @@ Use the dashboard at `/dashboard/providers` to enable, configure, and test each
|
||||
| ID | Alias | Name | Tags | Website | Notes |
|
||||
|----|-------|------|------|---------|-------|
|
||||
| `comfyui` | `comfyui` | ComfyUI | Local | [link](https://github.com/comfyanonymous/ComfyUI) | No API key required. Configure the local ComfyUI base URL (default: http://localhost:8188). |
|
||||
| `docker-model-runner` | `dmr` | Docker Model Runner | Local, self-hosted | [link](https://docs.docker.com/ai/model-runner/) | API key optional. Configure the local Docker Model Runner OpenAI-compatible base URL (default: http://localhost:12434/v1). |
|
||||
| `docker-model-runner` | `dmr` | Docker Model Runner | Local, self-hosted | [link](https://docs.docker.com/ai/model-runner/) | API key optional. Configure the local Docker Model Runner OpenAI-compatible base URL (default: http://localhost:12434/engines/v1). |
|
||||
| `lemonade` | `lemonade` | Lemonade Server | Local, self-hosted | [link](https://lemonade-server.ai) | API key optional. Configure the local Lemonade OpenAI-compatible base URL (default: http://localhost:13305/api/v1). |
|
||||
| `llama-cpp` | `llamacpp` | llama.cpp | Local, self-hosted | [link](https://github.com/ggml-org/llama.cpp) | API key optional (use any value, e.g. sk-no-key-required). Configure the llama-server OpenAI-compatible base URL (default: http://127.0.0.1:8080/v1). Note: if Llamafile is also installed, both default to port 8080 — run only one at a time or override the port. |
|
||||
| `llamafile` | `llamafile` | Llamafile | Local, self-hosted | [link](https://github.com/Mozilla-Ocho/llamafile) | API key optional. Configure the local Llamafile OpenAI-compatible base URL (default: http://127.0.0.1:8080/v1). |
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveLocalProviderBaseUrl } from "@/shared/utils/localProviderBaseUrl";
|
||||
import { isSelfHostedChatProvider } from "@/shared/constants/providers";
|
||||
import { getStaticModelsForProvider, type LocalCatalogModel } from "@/lib/providers/staticModels";
|
||||
import { SAFE_OUTBOUND_FETCH_PRESETS, safeOutboundFetch } from "@/shared/network/safeOutboundFetch";
|
||||
@@ -19,10 +20,10 @@ export function toNonEmptyString(value: unknown): string | null {
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
|
||||
export function getProviderBaseUrl(providerSpecificData: unknown): string | null {
|
||||
export function getProviderBaseUrl(providerSpecificData: unknown, provider = ""): string | null {
|
||||
const data = asRecord(providerSpecificData);
|
||||
const baseUrl = data.baseUrl;
|
||||
return typeof baseUrl === "string" && baseUrl.trim().length > 0 ? baseUrl : null;
|
||||
return resolveLocalProviderBaseUrl(provider, baseUrl);
|
||||
}
|
||||
|
||||
export function normalizeAzureOpenAIBaseUrl(baseUrl: string) {
|
||||
|
||||
@@ -856,7 +856,7 @@ export async function GET(
|
||||
? getRegistryEntry(provider)
|
||||
: null;
|
||||
const rawBaseUrl =
|
||||
getProviderBaseUrl(connection.providerSpecificData) ||
|
||||
getProviderBaseUrl(connection.providerSpecificData, provider) ||
|
||||
(typeof registryEntry?.baseUrl === "string" ? registryEntry.baseUrl : null);
|
||||
const baseUrl = rawBaseUrl;
|
||||
if (!baseUrl) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveLocalProviderBaseUrl } from "@/shared/utils/localProviderBaseUrl";
|
||||
import { handleEmbedding } from "@omniroute/open-sse/handlers/embeddings.ts";
|
||||
import {
|
||||
parseEmbeddingModel,
|
||||
@@ -285,7 +286,7 @@ export async function createEmbeddingResponse(
|
||||
|
||||
const providerSpecificData = (credentials as { providerSpecificData?: Record<string, unknown> })
|
||||
.providerSpecificData;
|
||||
const configuredBaseUrl = providerSpecificData?.baseUrl;
|
||||
const configuredBaseUrl = resolveLocalProviderBaseUrl(provider, providerSpecificData?.baseUrl);
|
||||
if (typeof configuredBaseUrl !== "string" || configuredBaseUrl.trim().length === 0) {
|
||||
return errorResponse(
|
||||
HTTP_STATUS.BAD_REQUEST,
|
||||
@@ -300,7 +301,7 @@ export async function createEmbeddingResponse(
|
||||
providerConfig = {
|
||||
id: provider,
|
||||
baseUrl: baseUrl.endsWith("/embeddings") ? baseUrl : `${baseUrl}/embeddings`,
|
||||
authType: "apikey",
|
||||
authType: provider === "docker-model-runner" ? "none" : "apikey",
|
||||
authHeader: "bearer",
|
||||
models: [],
|
||||
structuredInputProtocol: registryConfig?.structuredInputProtocol,
|
||||
|
||||
@@ -153,8 +153,8 @@ export const LOCAL_PROVIDERS = {
|
||||
textIcon: "DM",
|
||||
website: "https://docs.docker.com/ai/model-runner/",
|
||||
authHint:
|
||||
"API key optional. Configure the local Docker Model Runner OpenAI-compatible base URL (default: http://localhost:12434/v1).",
|
||||
localDefault: "http://localhost:12434/v1",
|
||||
"API key optional. Configure the local Docker Model Runner OpenAI-compatible base URL (default: http://localhost:12434/engines/v1).",
|
||||
localDefault: "http://localhost:12434/engines/v1",
|
||||
passthroughModels: true,
|
||||
},
|
||||
xinference: {
|
||||
|
||||
13
src/shared/utils/localProviderBaseUrl.ts
Normal file
13
src/shared/utils/localProviderBaseUrl.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { LOCAL_PROVIDERS } from "@/shared/constants/providers/local";
|
||||
|
||||
/** Explicit connection URLs win; registered local defaults need no duplicate configuration. */
|
||||
export function resolveLocalProviderBaseUrl(
|
||||
provider: string,
|
||||
configuredBaseUrl: unknown
|
||||
): string | null {
|
||||
if (typeof configuredBaseUrl === "string" && configuredBaseUrl.trim()) {
|
||||
return configuredBaseUrl;
|
||||
}
|
||||
if (!Object.prototype.hasOwnProperty.call(LOCAL_PROVIDERS, provider)) return null;
|
||||
return LOCAL_PROVIDERS[provider as keyof typeof LOCAL_PROVIDERS].localDefault;
|
||||
}
|
||||
241
tests/unit/docker-model-runner-discovery-16670.test.ts
Normal file
241
tests/unit/docker-model-runner-discovery-16670.test.ts
Normal file
@@ -0,0 +1,241 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const dataDir = mkdtempSync(join(tmpdir(), "omniroute-dmr-16670-"));
|
||||
process.env.DATA_DIR = dataDir;
|
||||
process.env.APP_LOG_TO_FILE = "false";
|
||||
process.env.OMNIROUTE_PROVIDER_PROBE_TIMEOUT_MS = "1000";
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const providers = await import("../../src/lib/db/providers.ts");
|
||||
const models = await import("../../src/lib/db/models.ts");
|
||||
const { GET } = await import("../../src/app/api/providers/[id]/models/route.ts");
|
||||
const { getUnifiedModelsResponse } = await import("../../src/app/api/v1/models/catalog.ts");
|
||||
const { createEmbeddingResponse } = await import("../../src/lib/embeddings/service.ts");
|
||||
const { DefaultExecutor } = await import("../../open-sse/executors/default.ts");
|
||||
const { resolveLocalSyncedEndpointRoute } =
|
||||
await import("../../src/lib/providerModels/syncedEndpointRouting.ts");
|
||||
const originalFetch = globalThis.fetch;
|
||||
const provider = "docker-model-runner";
|
||||
const defaultBase = "http://localhost:12434/engines/v1";
|
||||
|
||||
async function seed(baseUrl?: string) {
|
||||
return providers.createProviderConnection({
|
||||
provider,
|
||||
authType: "apikey",
|
||||
name: "Docker test",
|
||||
isActive: true,
|
||||
testStatus: "active",
|
||||
providerSpecificData: { autoFetchModels: true, ...(baseUrl ? { baseUrl } : {}) },
|
||||
});
|
||||
}
|
||||
function discover(id: string) {
|
||||
return GET(new Request(`http://localhost/api/providers/${id}/models?refresh=true`), {
|
||||
params: { id },
|
||||
});
|
||||
}
|
||||
|
||||
test.beforeEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
core.resetDbInstance();
|
||||
rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
mkdirSync(dataDir, { recursive: true });
|
||||
});
|
||||
test.after(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
core.resetDbInstance();
|
||||
rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
});
|
||||
|
||||
test("DMR discovers its registered default and exposes exact tagged IDs in the public catalog", async () => {
|
||||
const connection = await seed();
|
||||
const urls: string[] = [];
|
||||
globalThis.fetch = async (url) => {
|
||||
urls.push(String(url));
|
||||
return Response.json({ data: [{ id: "ai/smollm2:latest", object: "model" }] });
|
||||
};
|
||||
const response = await discover(connection.id);
|
||||
const body = await response.json();
|
||||
assert.equal(response.status, 200, JSON.stringify(body));
|
||||
assert.equal(body.source, "api");
|
||||
assert.deepEqual(urls, [`${defaultBase}/models`]);
|
||||
const synced = await models.getSyncedAvailableModelsByConnection(provider);
|
||||
assert.equal(synced[connection.id][0].id, "ai/smollm2:latest");
|
||||
const catalog = await getUnifiedModelsResponse(
|
||||
new Request("http://localhost/v1/models?prefix=alias")
|
||||
);
|
||||
const publicBody = await catalog.json();
|
||||
assert.equal(catalog.status, 200);
|
||||
assert.ok(publicBody.data.some((model: { id: string }) => model.id === "dmr/ai/smollm2:latest"));
|
||||
});
|
||||
|
||||
test("DMR discovery preserves an explicit container engine URL", async () => {
|
||||
const base = "http://model-runner.docker.internal/engines/llama.cpp/v1";
|
||||
const connection = await seed(base);
|
||||
const urls: string[] = [];
|
||||
globalThis.fetch = async (url) => {
|
||||
urls.push(String(url));
|
||||
return Response.json({ data: [{ id: "ai/smollm2:q4" }] });
|
||||
};
|
||||
const response = await discover(connection.id);
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal((await response.json()).source, "api");
|
||||
assert.deepEqual(urls, [`${base}/models`]);
|
||||
});
|
||||
|
||||
for (const base of [undefined, "http://127.0.0.1:12435/engines/llama.cpp/v1"]) {
|
||||
test(`DMR embedding dispatch respects ${base ? "explicit" : "default"} URL after capability discovery`, async () => {
|
||||
const connection = await seed(base);
|
||||
await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [
|
||||
{ id: "ai/embeddinggemma:latest", supportedEndpoints: ["embeddings"] },
|
||||
]);
|
||||
const calls: Array<{
|
||||
url: string;
|
||||
body: Record<string, unknown>;
|
||||
authorization: string | null;
|
||||
}> = [];
|
||||
globalThis.fetch = async (url, options) => {
|
||||
calls.push({
|
||||
url: String(url),
|
||||
body: JSON.parse(String(options?.body)),
|
||||
authorization: new Headers(options?.headers).get("authorization"),
|
||||
});
|
||||
return Response.json({
|
||||
data: [{ object: "embedding", index: 0, embedding: [0.1, 0.2] }],
|
||||
usage: { prompt_tokens: 1, total_tokens: 1 },
|
||||
});
|
||||
};
|
||||
const response = await createEmbeddingResponse({
|
||||
model: "dmr/ai/embeddinggemma:latest",
|
||||
input: "hello",
|
||||
});
|
||||
const body = await response.json();
|
||||
assert.equal(response.status, 200, JSON.stringify(body));
|
||||
assert.deepEqual(body.data[0].embedding, [0.1, 0.2]);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].url, `${base || defaultBase}/embeddings`);
|
||||
assert.equal(calls[0].body.model, "ai/embeddinggemma:latest");
|
||||
assert.equal(calls[0].authorization, null);
|
||||
});
|
||||
}
|
||||
|
||||
test("DMR does not infer embeddings capability from a chat-only model listing", async () => {
|
||||
const connection = await seed();
|
||||
await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [
|
||||
{ id: "ai/smollm2:latest" },
|
||||
]);
|
||||
assert.equal(await resolveLocalSyncedEndpointRoute("dmr/ai/smollm2:latest", "embeddings"), null);
|
||||
});
|
||||
|
||||
test("DMR chat dispatch preserves JSON and SSE bodies through default and engine URLs", async () => {
|
||||
const executor = new DefaultExecutor(provider);
|
||||
for (const stream of [false, true]) {
|
||||
for (const baseUrl of [undefined, "http://127.0.0.1:12435/engines/llama.cpp/v1"]) {
|
||||
const expectedBase = baseUrl || defaultBase;
|
||||
const responseBody = stream
|
||||
? 'data: {"choices":[{"delta":{"content":"hello"}}]}\n\ndata: [DONE]\n\n'
|
||||
: JSON.stringify({ choices: [{ message: { role: "assistant", content: "hello" } }] });
|
||||
const calls: Array<{ url: string; body: Record<string, unknown> }> = [];
|
||||
globalThis.fetch = async (url, options) => {
|
||||
calls.push({ url: String(url), body: JSON.parse(String(options?.body)) });
|
||||
return new Response(responseBody, {
|
||||
headers: { "content-type": stream ? "text/event-stream" : "application/json" },
|
||||
});
|
||||
};
|
||||
const result = await executor.execute({
|
||||
model: "ai/smollm2:latest",
|
||||
stream,
|
||||
credentials: { providerSpecificData: baseUrl ? { baseUrl } : {} },
|
||||
body: {
|
||||
model: "ai/smollm2:latest",
|
||||
stream,
|
||||
messages: [{ role: "user", content: "hello" }],
|
||||
},
|
||||
});
|
||||
assert.equal(result.response.status, 200);
|
||||
assert.equal(await result.response.text(), responseBody);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].url, `${expectedBase}/chat/completions`);
|
||||
assert.equal(calls[0].body.model, "ai/smollm2:latest");
|
||||
assert.equal(calls[0].body.stream, stream);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("DMR optional embedding credentials are forwarded when configured", async () => {
|
||||
const connection = await providers.createProviderConnection({
|
||||
provider,
|
||||
authType: "apikey",
|
||||
apiKey: "dmr-test-key",
|
||||
name: "Keyed DMR",
|
||||
isActive: true,
|
||||
providerSpecificData: { baseUrl: defaultBase },
|
||||
});
|
||||
await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [
|
||||
{ id: "ai/embeddinggemma:latest", supportedEndpoints: ["embeddings"] },
|
||||
]);
|
||||
let authorization: string | null = null;
|
||||
globalThis.fetch = async (_url, options) => {
|
||||
authorization = new Headers(options?.headers).get("authorization");
|
||||
return Response.json({ data: [{ object: "embedding", index: 0, embedding: [0.1] }] });
|
||||
};
|
||||
const response = await createEmbeddingResponse({
|
||||
model: "dmr/ai/embeddinggemma:latest",
|
||||
input: "hello",
|
||||
});
|
||||
assert.equal(response.status, 200, JSON.stringify(await response.json()));
|
||||
assert.equal(authorization, "Bearer dmr-test-key");
|
||||
});
|
||||
|
||||
test("DMR unavailable discovery retains cached models and terminates after bounded attempts", async () => {
|
||||
const connection = await seed(defaultBase);
|
||||
await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [
|
||||
{ id: "ai/smollm2:cached" },
|
||||
]);
|
||||
let calls = 0;
|
||||
globalThis.fetch = async (_url, options) => {
|
||||
calls += 1;
|
||||
assert.ok(options?.signal, "existing probe must carry a timeout signal");
|
||||
throw new TypeError("fetch failed");
|
||||
};
|
||||
const response = await discover(connection.id);
|
||||
const body = await response.json();
|
||||
assert.equal(response.status, 200);
|
||||
assert.notEqual(body.source, "api");
|
||||
assert.ok(body.models.some((model: { id: string }) => model.id === "ai/smollm2:cached"));
|
||||
assert.ok(calls > 0 && calls <= 6, `bounded probe attempts: ${calls}`);
|
||||
assert.equal(
|
||||
(await models.getSyncedAvailableModelsByConnection(provider))[connection.id][0].id,
|
||||
"ai/smollm2:cached"
|
||||
);
|
||||
});
|
||||
|
||||
test("DMR stalled discovery returns a bounded timeout error without erasing cached models", async () => {
|
||||
const connection = await seed(defaultBase);
|
||||
await models.replaceSyncedAvailableModelsForConnection(provider, connection.id, [
|
||||
{ id: "ai/smollm2:cached" },
|
||||
]);
|
||||
let calls = 0;
|
||||
globalThis.fetch = async (_url, options) => {
|
||||
calls += 1;
|
||||
assert.ok(options?.signal);
|
||||
await new Promise<void>((_resolve, reject) => {
|
||||
options.signal?.addEventListener("abort", () => reject(options.signal?.reason), {
|
||||
once: true,
|
||||
});
|
||||
});
|
||||
throw new Error("timeout signal must reject");
|
||||
};
|
||||
const response = await discover(connection.id);
|
||||
const body = await response.json();
|
||||
assert.equal(response.status, 504);
|
||||
assert.equal(typeof body.error, "string");
|
||||
assert.ok(!body.error.includes("at /"));
|
||||
assert.ok(calls > 0 && calls <= 2, `bounded timeout attempts: ${calls}`);
|
||||
assert.equal(
|
||||
(await models.getSyncedAvailableModelsByConnection(provider))[connection.id][0].id,
|
||||
"ai/smollm2:cached"
|
||||
);
|
||||
});
|
||||
23
tests/unit/local-provider-base-url.test.ts
Normal file
23
tests/unit/local-provider-base-url.test.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { resolveLocalProviderBaseUrl } from "../../src/shared/utils/localProviderBaseUrl.ts";
|
||||
|
||||
test("local endpoint resolution keeps explicit URLs authoritative", () => {
|
||||
for (const provider of ["docker-model-runner", "llama-cpp", "openai", "custom"]) {
|
||||
assert.equal(
|
||||
resolveLocalProviderBaseUrl(provider, "http://127.0.0.1:9000/custom/v1"),
|
||||
"http://127.0.0.1:9000/custom/v1"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("only own registered local providers receive default endpoints", () => {
|
||||
assert.equal(
|
||||
resolveLocalProviderBaseUrl("docker-model-runner", undefined),
|
||||
"http://localhost:12434/engines/v1"
|
||||
);
|
||||
assert.equal(resolveLocalProviderBaseUrl("llama-cpp", " "), "http://127.0.0.1:8080/v1");
|
||||
for (const provider of ["openai", "unknown", "toString", "__proto__", "constructor"]) {
|
||||
assert.equal(resolveLocalProviderBaseUrl(provider, undefined), null);
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user