mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-21 15:05:56 +00:00
fix(combo): keep the account dimension when mapping round-robin targets to their step (#20191)
findSameStepIndex() matched every target by its base step identity, so the per-connection targets produced by connectionAwareExpansion (step@c0, step@c1, step@c2) all collapsed onto index 0. Round-robin then read and advanced the counter at the same entry on every request and pinned one connection, breaking the connection-aware-expansion T6 contract. Match the exact execution key first and fall back to the base step only when the target itself is absent, which is the prompt-cache-affinity expansion case (step <-> step@account) that #20191 needs. Adds a direct unit test for the helper and a round-robin test with connection-aware expansion while affinity is engaged.
This commit is contained in:
committed by
diegosouzapw
parent
8fcc58e49e
commit
54f45f31ee
@@ -324,14 +324,21 @@ function getBaseModelIdentity(target: ResolvedComboTarget): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* #20191: index of the first entry in `targets` that belongs to the same combo step as `target`.
|
||||
* Affinity expansion turns a step into `step@account` targets, so steps are matched by their
|
||||
* base identity rather than by the (possibly expanded) execution key. -1 when none matches.
|
||||
* #20191: index of `target` in `targets`, or of the first entry of the same combo step.
|
||||
* The exact execution key wins, so already-per-connection targets (connection-aware expansion's
|
||||
* `step@c0`, `step@c1`, …) keep their account dimension and round-robin still rotates across
|
||||
* them. Only when the target itself is absent — prompt-cache affinity expanded its step into
|
||||
* `step@account` targets, or one of those maps back to its step — are steps matched by their
|
||||
* base identity. -1 when none matches.
|
||||
*/
|
||||
export function findSameStepIndex(
|
||||
targets: ResolvedComboTarget[],
|
||||
target: ResolvedComboTarget
|
||||
): number {
|
||||
const exactIndex = target.executionKey
|
||||
? targets.findIndex((entry) => entry.executionKey === target.executionKey)
|
||||
: -1;
|
||||
if (exactIndex >= 0) return exactIndex;
|
||||
const step = getBaseModelIdentity(target);
|
||||
return targets.findIndex((entry) => getBaseModelIdentity(entry) === step);
|
||||
}
|
||||
|
||||
@@ -1056,8 +1056,8 @@ export async function handleRoundRobinCombo({
|
||||
);
|
||||
}
|
||||
|
||||
// #948, #20191: advance past the step that ACTUALLY served, in the initialTargets
|
||||
// index space (an affinity-expanded step@account target maps back to its step).
|
||||
// #948, #20191: advance past the target that ACTUALLY served, in initialTargets space
|
||||
// (exact entry first; only an affinity-expanded step@account maps back to its step).
|
||||
// Read side applies `% initialTargets.length`, so servedIndex + 1 is correct.
|
||||
const servedIndex = findSameStepIndex(initialTargets, target);
|
||||
if (stickyRoundRobinEnabled) {
|
||||
|
||||
@@ -14,6 +14,8 @@ const stick = await import("../../open-sse/services/combo/sessionStickiness.ts")
|
||||
const dbCore = await import("../../src/lib/db/core.ts");
|
||||
const { getSettings } = await import("../../src/lib/db/settings.ts");
|
||||
const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
const { findSameStepIndex } = await import("../../open-sse/services/combo/promptCacheAffinity.ts");
|
||||
const { registerQuotaFetcher } = await import("../../open-sse/services/quotaPreflight.ts");
|
||||
|
||||
function makeLog() {
|
||||
return { info() {}, warn() {}, debug() {}, error() {} };
|
||||
@@ -171,6 +173,54 @@ test("round-robin rotates across steps when affinity expands them into per-accou
|
||||
assert.deepEqual(served, ["mock-a", "mock-b", "mock-c", "mock-a", "mock-b", "mock-c"]);
|
||||
});
|
||||
|
||||
test("findSameStepIndex keeps the account dimension of per-connection targets (#20191)", () => {
|
||||
const target = (executionKey: string) =>
|
||||
({ executionKey, modelStr: "p/m" }) as Parameters<typeof findSameStepIndex>[1];
|
||||
const expanded = [target("s@c0"), target("s@c1"), target("s@c2"), target("t")];
|
||||
|
||||
// An exact entry wins over the first entry of its step.
|
||||
assert.equal(findSameStepIndex(expanded, target("s@c2")), 2);
|
||||
assert.equal(findSameStepIndex(expanded, target("t")), 3);
|
||||
// Only an absent target falls back to its step (affinity expansion maps step <-> step@account).
|
||||
assert.equal(findSameStepIndex(expanded, target("s")), 0);
|
||||
assert.equal(findSameStepIndex([target("s"), target("t")], target("t@c9")), 1);
|
||||
assert.equal(findSameStepIndex(expanded, target("u")), -1);
|
||||
});
|
||||
|
||||
test("round-robin spans connection-expanded targets while prompt-cache affinity is engaged (#20191)", async () => {
|
||||
// connectionAwareExpansion turns the single step into step@account targets BEFORE the RR
|
||||
// counter is read, so the rotation unit is model x connection. A shared system prompt makes
|
||||
// affinity resolve a key on every request; it must not collapse the rotation onto one account.
|
||||
const provider = "mock-cae-affinity";
|
||||
registerQuotaFetcher(provider, async () => ({ used: 0, total: 100, percentUsed: 0 }));
|
||||
const ids: string[] = [];
|
||||
for (const suffix of ["1", "2", "3"]) {
|
||||
const connection = await providersDb.createProviderConnection({
|
||||
provider,
|
||||
authType: "apikey",
|
||||
name: `${provider}-${suffix}`,
|
||||
apiKey: `sk-${provider}-${suffix}`,
|
||||
isActive: true,
|
||||
});
|
||||
assert.ok(connection, `connection ${suffix} must be created`);
|
||||
ids.push(String(connection.id));
|
||||
}
|
||||
const combo = {
|
||||
name: "rr-cae-affinity-test",
|
||||
strategy: "round-robin",
|
||||
config: { maxRetries: 0, disableSessionStickiness: true, connectionAwareExpansion: true },
|
||||
models: [`${provider}/m-x`],
|
||||
};
|
||||
|
||||
const served: string[] = [];
|
||||
for (let i = 0; i < 3; i++) {
|
||||
served.push(await dispatchWithPrefix(combo, "Shared system prompt", `User query ${i}`));
|
||||
}
|
||||
|
||||
assert.equal(new Set(served).size, 3, `expected 3 distinct connections, got ${served}`);
|
||||
for (const id of ids) assert.ok(served.includes(id), `connection ${id} must be hit`);
|
||||
});
|
||||
|
||||
test("getSettings falls back to databaseSettings namespace for promptCacheAffinityEnabled (#20191)", async () => {
|
||||
const db = dbCore.getDbInstance();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user