Merge model tiers: fast, standard and code per space for bots and helpers
Conflicts were additive: duplicate and createBot carry both the tool policy and the tier, and the bot panel saves both. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude
commit
04b7513e57
@ -1,3 +1,4 @@
|
||||
import { serializeModelSecret } from "@rakazo/adapters";
|
||||
import type { Actor } from "@rakazo/contracts";
|
||||
import { AI_DISCLOSURE_VERSION } from "@rakazo/contracts";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
@ -15,6 +16,7 @@ function setup() {
|
||||
env: { agentRuntime: "scripted" },
|
||||
prisma: {
|
||||
spaceModelPreference: { findMany: vi.fn(async () => []), findFirst: vi.fn(async () => null) },
|
||||
spaceModelTier: { findMany: vi.fn(async () => []) },
|
||||
secret: { findMany: vi.fn(async () => []) },
|
||||
bot: { findMany: vi.fn(async () => []) },
|
||||
spaceVoicePreference: {
|
||||
@ -58,6 +60,46 @@ describe("consent grants", () => {
|
||||
expect.objectContaining({ where: expect.objectContaining({ id: { in: ["bot"] } }) }),
|
||||
);
|
||||
});
|
||||
it("discloses the model a bot's tier maps to on a router endpoint", async () => {
|
||||
const { deps, actor } = setup();
|
||||
deps.env.agentRuntime = "pi";
|
||||
deps.secrets = {
|
||||
load: vi.fn(() =>
|
||||
serializeModelSecret({ kind: "openai_compatible", baseUrl: "http://127.0.0.1:8080/v1" }),
|
||||
),
|
||||
} as unknown as RouterDeps["secrets"];
|
||||
vi.mocked(deps.prisma.spaceModelTier.findMany).mockResolvedValue([
|
||||
{
|
||||
tier: "code",
|
||||
credentialId: "router",
|
||||
modelId: "router-code",
|
||||
credential: {
|
||||
id: "router",
|
||||
userId: actor.userId,
|
||||
provider: "openai-compatible",
|
||||
label: "Router",
|
||||
secretId: "router-secret",
|
||||
supportsImages: false,
|
||||
createdAt: new Date(0),
|
||||
updatedAt: new Date(0),
|
||||
preferences: [],
|
||||
},
|
||||
},
|
||||
] as never);
|
||||
vi.mocked(deps.prisma.bot.findMany).mockResolvedValue([
|
||||
{ modelProvider: null, modelId: null, thinkingLevel: null, modelTier: "code" },
|
||||
] as never);
|
||||
vi.mocked(deps.prisma.secret.findMany).mockResolvedValue([
|
||||
{ id: "router-secret", ciphertext: "fake-ciphertext" },
|
||||
] as never);
|
||||
const status = await aiConsentStatus(deps, actor, { botId: "bot", uses: ["model"] });
|
||||
expect(status.recipients).toEqual([
|
||||
expect.objectContaining({
|
||||
name: expect.stringContaining("http://127.0.0.1:8080"),
|
||||
detail: "Model: router-code.",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
it("discloses only the selected voice provider before a voice action", async () => {
|
||||
const { deps, actor } = setup();
|
||||
vi.mocked(deps.prisma.spaceVoicePreference.findMany).mockResolvedValue([
|
||||
|
||||
@ -13,6 +13,7 @@ import {
|
||||
findDefaultModelCredential,
|
||||
findDefaultVoiceCredential,
|
||||
findModelCredential,
|
||||
listSpaceModelTiers,
|
||||
} from "@rakazo/db";
|
||||
import { isSpaceOwner } from "./memory-provider-config.js";
|
||||
import type { RouterDeps } from "./router.js";
|
||||
@ -44,7 +45,7 @@ export async function aiConsentStatus(
|
||||
: target.memberBotIds
|
||||
: undefined;
|
||||
const includesMemory = uses.includes("memory");
|
||||
const [preferences, defaultCredential, bots, voices, memoryOwner, settings, consents] =
|
||||
const [preferences, tiers, defaultCredential, bots, voices, memoryOwner, settings, consents] =
|
||||
await Promise.all([
|
||||
modelsEnabled
|
||||
? deps.prisma.spaceModelPreference.findMany({
|
||||
@ -52,6 +53,7 @@ export async function aiConsentStatus(
|
||||
include: { credential: true },
|
||||
})
|
||||
: [],
|
||||
modelsEnabled ? listSpaceModelTiers(deps.prisma, actor) : [],
|
||||
modelsEnabled ? findDefaultModelCredential(deps.prisma, actor) : null,
|
||||
modelsEnabled
|
||||
? deps.prisma.bot.findMany({
|
||||
@ -61,7 +63,7 @@ export async function aiConsentStatus(
|
||||
archivedAt: null,
|
||||
...(botIds ? { id: { in: botIds } } : {}),
|
||||
},
|
||||
select: { modelProvider: true, modelId: true, thinkingLevel: true },
|
||||
select: { modelProvider: true, modelId: true, thinkingLevel: true, modelTier: true },
|
||||
})
|
||||
: [],
|
||||
uses.includes("voice")
|
||||
@ -100,6 +102,7 @@ export async function aiConsentStatus(
|
||||
: null;
|
||||
return selectConfiguredModel({
|
||||
bot,
|
||||
tierModel: tiers.find((tier) => tier.tier === bot?.modelTier) ?? null,
|
||||
overrideCredential,
|
||||
defaultCredential,
|
||||
settings,
|
||||
@ -119,6 +122,14 @@ export async function aiConsentStatus(
|
||||
},
|
||||
thinkingLevel: null,
|
||||
});
|
||||
// So are the models mapped to tiers, which helpers can ask for.
|
||||
for (const tier of tiers)
|
||||
selected.push({
|
||||
provider: tier.credential.provider,
|
||||
id: tier.modelId,
|
||||
credential: tier.credential,
|
||||
thinkingLevel: null,
|
||||
});
|
||||
if (deps.env.teamChatJudgeProvider && deps.env.teamChatJudgeModel) {
|
||||
selected.push({
|
||||
provider: deps.env.teamChatJudgeProvider,
|
||||
|
||||
@ -70,6 +70,7 @@ describe("local settings authority", () => {
|
||||
for (const procedure of [
|
||||
"models/connect",
|
||||
"models/setDefault",
|
||||
"models/tiers/set",
|
||||
"integrationSetup/get",
|
||||
"integrationSetup/save",
|
||||
]) {
|
||||
|
||||
111
apps/api/src/model-tiers.test.ts
Normal file
111
apps/api/src/model-tiers.test.ts
Normal file
@ -0,0 +1,111 @@
|
||||
import { serializeModelSecret } from "@rakazo/adapters";
|
||||
import type { Actor } from "@rakazo/contracts";
|
||||
import type { PrismaClient } from "@rakazo/db";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { connectedOpenAiCompatibleServer, setModelTier } from "./model-tiers.js";
|
||||
|
||||
const actor = { userId: "user", spaceId: "space" } as Actor;
|
||||
|
||||
function setup(provider: string | null) {
|
||||
const upsert = vi.fn(async () => ({}));
|
||||
const deleteMany = vi.fn(async () => ({ count: 1 }));
|
||||
const credentialFindFirst = vi.fn(async () =>
|
||||
provider ? { provider, secretId: "router-secret" } : null,
|
||||
);
|
||||
const prisma = {
|
||||
userModelCredential: { findFirst: credentialFindFirst },
|
||||
spaceModelTier: { upsert, deleteMany, findMany: vi.fn(async () => []) },
|
||||
secret: {
|
||||
findFirst: vi.fn(async () => ({ id: "router-secret", ciphertext: "fake-ciphertext" })),
|
||||
},
|
||||
} as unknown as PrismaClient;
|
||||
return { prisma, upsert, deleteMany, credentialFindFirst };
|
||||
}
|
||||
|
||||
describe("model tier settings", () => {
|
||||
it("maps a tier to any model an OpenAI-compatible server serves", async () => {
|
||||
const { prisma, upsert, credentialFindFirst } = setup("openai-compatible");
|
||||
|
||||
await setModelTier(prisma, actor, {
|
||||
tier: "code",
|
||||
model: { credentialId: "router", modelId: "router-code" },
|
||||
});
|
||||
|
||||
expect(credentialFindFirst).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ where: { id: "router", userId: "user" } }),
|
||||
);
|
||||
expect(upsert).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
create: expect.objectContaining({
|
||||
tier: "code",
|
||||
credentialId: "router",
|
||||
modelId: "router-code",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses another user's credential and models a catalog provider does not run", async () => {
|
||||
const missing = setup(null);
|
||||
await expect(
|
||||
setModelTier(missing.prisma, actor, {
|
||||
tier: "fast",
|
||||
model: { credentialId: "someone-else", modelId: "router-fast" },
|
||||
}),
|
||||
).rejects.toMatchObject({ code: "NOT_FOUND" });
|
||||
expect(missing.upsert).not.toHaveBeenCalled();
|
||||
|
||||
const catalog = setup("xai");
|
||||
await expect(
|
||||
setModelTier(catalog.prisma, actor, {
|
||||
tier: "fast",
|
||||
model: { credentialId: "xai", modelId: "not-a-model" },
|
||||
}),
|
||||
).rejects.toMatchObject({ code: "BAD_REQUEST" });
|
||||
await setModelTier(catalog.prisma, actor, {
|
||||
tier: "fast",
|
||||
model: { credentialId: "xai", modelId: "grok-4.6" },
|
||||
});
|
||||
expect(catalog.upsert).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("clears a tier without looking up a credential", async () => {
|
||||
const { prisma, deleteMany, credentialFindFirst } = setup("openai-compatible");
|
||||
|
||||
await expect(setModelTier(prisma, actor, { tier: "standard", model: null })).resolves.toEqual(
|
||||
[],
|
||||
);
|
||||
|
||||
expect(credentialFindFirst).not.toHaveBeenCalled();
|
||||
expect(deleteMany).toHaveBeenCalledWith({
|
||||
where: { spaceId: "space", userId: "user", tier: "standard" },
|
||||
});
|
||||
});
|
||||
|
||||
it("probes a connected server with its stored address and key, for its owner only", async () => {
|
||||
const { prisma, credentialFindFirst } = setup("openai-compatible");
|
||||
const secrets = {
|
||||
load: vi.fn(() =>
|
||||
serializeModelSecret({
|
||||
kind: "openai_compatible",
|
||||
baseUrl: "https://router.example.test/v1",
|
||||
apiKey: "fake-router-key",
|
||||
}),
|
||||
),
|
||||
};
|
||||
|
||||
await expect(
|
||||
connectedOpenAiCompatibleServer({ prisma, secrets }, actor, "router"),
|
||||
).resolves.toEqual({ baseUrl: "https://router.example.test/v1", apiKey: "fake-router-key" });
|
||||
expect(credentialFindFirst).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
where: { id: "router", userId: "user", provider: "openai-compatible" },
|
||||
}),
|
||||
);
|
||||
|
||||
const missing = setup(null);
|
||||
await expect(
|
||||
connectedOpenAiCompatibleServer({ prisma: missing.prisma, secrets }, actor, "router"),
|
||||
).rejects.toMatchObject({ code: "NOT_FOUND" });
|
||||
});
|
||||
});
|
||||
77
apps/api/src/model-tiers.ts
Normal file
77
apps/api/src/model-tiers.ts
Normal file
@ -0,0 +1,77 @@
|
||||
import { ORPCError } from "@orpc/server";
|
||||
import type { EncryptedSecretStore } from "@rakazo/adapters";
|
||||
import { isCatalogModelChoice, parseModelSecret } from "@rakazo/adapters";
|
||||
import type { Actor, ModelTierMapping, SetModelTierInput } from "@rakazo/contracts";
|
||||
import { ModelTierSchema, OPENAI_COMPATIBLE_PROVIDER_ID } from "@rakazo/contracts";
|
||||
import type { PrismaClient } from "@rakazo/db";
|
||||
import { listSpaceModelTiers, setSpaceModelTier } from "@rakazo/db";
|
||||
|
||||
const NOT_CONNECTED = "Connect that model provider first";
|
||||
|
||||
export async function listModelTiers(
|
||||
prisma: PrismaClient,
|
||||
actor: Actor,
|
||||
): Promise<ModelTierMapping[]> {
|
||||
const rows = await listSpaceModelTiers(prisma, actor);
|
||||
return rows.flatMap((row) => {
|
||||
const tier = ModelTierSchema.safeParse(row.tier);
|
||||
return tier.success
|
||||
? [
|
||||
{
|
||||
tier: tier.data,
|
||||
credentialId: row.credentialId,
|
||||
provider: row.credential.provider,
|
||||
modelId: row.modelId,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
});
|
||||
}
|
||||
|
||||
export async function setModelTier(prisma: PrismaClient, actor: Actor, input: SetModelTierInput) {
|
||||
if (input.model) {
|
||||
const credential = await prisma.userModelCredential.findFirst({
|
||||
where: { id: input.model.credentialId, userId: actor.userId },
|
||||
select: { provider: true },
|
||||
});
|
||||
if (!credential) throw new ORPCError("NOT_FOUND", { message: NOT_CONNECTED });
|
||||
// An OpenAI-compatible server runs whatever ids it serves; other providers run catalog models.
|
||||
if (
|
||||
credential.provider !== OPENAI_COMPATIBLE_PROVIDER_ID &&
|
||||
!isCatalogModelChoice(credential.provider, input.model.modelId)
|
||||
) {
|
||||
throw new ORPCError("BAD_REQUEST", { message: "Unknown model for that provider" });
|
||||
}
|
||||
}
|
||||
await setSpaceModelTier(prisma, actor, input.tier, input.model);
|
||||
return listModelTiers(prisma, actor);
|
||||
}
|
||||
|
||||
/** The stored address and key of a connected OpenAI-compatible server, for listing its models. */
|
||||
export async function connectedOpenAiCompatibleServer(
|
||||
deps: { prisma: PrismaClient; secrets: Pick<EncryptedSecretStore, "load"> },
|
||||
actor: Actor,
|
||||
credentialId: string,
|
||||
) {
|
||||
const credential = await deps.prisma.userModelCredential.findFirst({
|
||||
where: { id: credentialId, userId: actor.userId, provider: OPENAI_COMPATIBLE_PROVIDER_ID },
|
||||
select: { secretId: true },
|
||||
});
|
||||
const secret = credential
|
||||
? await deps.prisma.secret.findFirst({
|
||||
where: { id: credential.secretId, userId: actor.userId, spaceId: null },
|
||||
select: { id: true, ciphertext: true },
|
||||
})
|
||||
: null;
|
||||
if (!secret) throw new ORPCError("NOT_FOUND", { message: NOT_CONNECTED });
|
||||
let stored: ReturnType<typeof parseModelSecret>;
|
||||
try {
|
||||
stored = parseModelSecret(deps.secrets.load(secret.ciphertext, secret.id));
|
||||
} catch {
|
||||
throw new ORPCError("NOT_FOUND", { message: NOT_CONNECTED });
|
||||
}
|
||||
if (stored.kind !== "openai_compatible") {
|
||||
throw new ORPCError("NOT_FOUND", { message: NOT_CONNECTED });
|
||||
}
|
||||
return { baseUrl: stored.baseUrl, apiKey: stored.apiKey };
|
||||
}
|
||||
@ -120,6 +120,7 @@ import {
|
||||
findDefaultModelCredential,
|
||||
findDefaultVoiceCredential,
|
||||
findModelCredential,
|
||||
findSpaceModelTier,
|
||||
formatMessagingLinkCode,
|
||||
InvalidProjectWorkspacePathError,
|
||||
InvalidSpaceNameError,
|
||||
@ -163,6 +164,7 @@ import {
|
||||
persistMemoryProviderConfig,
|
||||
updateMemoryProviderDefaultScope,
|
||||
} from "./memory-provider-config.js";
|
||||
import { connectedOpenAiCompatibleServer, listModelTiers, setModelTier } from "./model-tiers.js";
|
||||
import {
|
||||
chooseFocus,
|
||||
dismissFocus,
|
||||
@ -906,8 +908,12 @@ export function createRouter(deps: RouterDeps) {
|
||||
}),
|
||||
probeOpenAiCompatible: authed.models.probeOpenAiCompatible.handler(
|
||||
async ({ context, input }) => {
|
||||
const server =
|
||||
"credentialId" in input
|
||||
? await connectedOpenAiCompatibleServer(deps, context.actor, input.credentialId)
|
||||
: input;
|
||||
try {
|
||||
const models = await probeOpenAiCompatibleModels(input, undefined, context.signal);
|
||||
const models = await probeOpenAiCompatibleModels(server, undefined, context.signal);
|
||||
return { models };
|
||||
} catch (error) {
|
||||
throw new ORPCError("BAD_REQUEST", {
|
||||
@ -983,6 +989,14 @@ export function createRouter(deps: RouterDeps) {
|
||||
);
|
||||
return { ok: true as const };
|
||||
}),
|
||||
tiers: {
|
||||
list: authed.models.tiers.list.handler(({ context }) =>
|
||||
listModelTiers(deps.prisma, context.actor),
|
||||
),
|
||||
set: authed.models.tiers.set.handler(({ context, input }) =>
|
||||
setModelTier(deps.prisma, context.actor, input),
|
||||
),
|
||||
},
|
||||
},
|
||||
bots: {
|
||||
list: authed.bots.list.handler(async ({ context }) => repos.listBots(context.actor)),
|
||||
@ -1019,6 +1033,7 @@ export function createRouter(deps: RouterDeps) {
|
||||
tags: source.tags,
|
||||
restrictBuiltinTools: source.restrictBuiltinTools,
|
||||
allowedBuiltinTools: source.allowedBuiltinTools,
|
||||
modelTier: source.modelTier,
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
throw mapSpaceLifecycleError(error);
|
||||
@ -1081,12 +1096,28 @@ export function createRouter(deps: RouterDeps) {
|
||||
}
|
||||
const thinkingLevel = input.thinkingLevel;
|
||||
if (input.thinkingLevel) {
|
||||
// Mirror what this update writes: a new tier clears the pin and a new pin the tier.
|
||||
const provider =
|
||||
input.modelProvider !== undefined ? input.modelProvider : existing.modelProvider;
|
||||
const modelId = input.modelId !== undefined ? input.modelId : existing.modelId;
|
||||
input.modelProvider !== undefined
|
||||
? input.modelProvider
|
||||
: input.modelTier
|
||||
? null
|
||||
: existing.modelProvider;
|
||||
const modelId =
|
||||
input.modelId !== undefined ? input.modelId : input.modelTier ? null : existing.modelId;
|
||||
const modelTier =
|
||||
input.modelTier !== undefined
|
||||
? input.modelTier
|
||||
: input.modelProvider
|
||||
? null
|
||||
: existing.modelTier;
|
||||
const tierModel = modelTier
|
||||
? await findSpaceModelTier(deps.prisma, context.actor, modelTier)
|
||||
: null;
|
||||
const me = await meDto(deps, context.actor);
|
||||
const effectiveProvider = provider ?? me.defaultProvider;
|
||||
const effectiveModelId = modelId ?? me.defaultModel;
|
||||
const effectiveProvider =
|
||||
tierModel?.credential.provider ?? provider ?? me.defaultProvider;
|
||||
const effectiveModelId = tierModel?.modelId ?? modelId ?? me.defaultModel;
|
||||
if (effectiveProvider && effectiveModelId) {
|
||||
const entry = listPiCatalog().find(
|
||||
(item) => item.provider === effectiveProvider && item.id === effectiveModelId,
|
||||
@ -1094,12 +1125,11 @@ export function createRouter(deps: RouterDeps) {
|
||||
let allowed = entry?.thinkingLevels;
|
||||
if (effectiveProvider === OPENAI_COMPATIBLE_PROVIDER_ID) {
|
||||
allowed = ["off"];
|
||||
const credential = await findModelCredential(
|
||||
deps.prisma,
|
||||
context.actor,
|
||||
effectiveProvider,
|
||||
);
|
||||
if (credential && credential.defaultModel === effectiveModelId) {
|
||||
// A tier's server keeps one reasoning setting for every model it serves.
|
||||
const credential =
|
||||
tierModel?.credential ??
|
||||
(await findModelCredential(deps.prisma, context.actor, effectiveProvider));
|
||||
if (credential && (tierModel || credential.defaultModel === effectiveModelId)) {
|
||||
const secret = await deps.prisma.secret.findFirst({
|
||||
where: { id: credential.secretId, userId: context.actor.userId, spaceId: null },
|
||||
select: { ciphertext: true },
|
||||
@ -1148,6 +1178,10 @@ export function createRouter(deps: RouterDeps) {
|
||||
? { modelProvider: input.modelProvider, modelId: input.modelId ?? null }
|
||||
: {}),
|
||||
...(input.thinkingLevel !== undefined ? { thinkingLevel } : {}),
|
||||
...(input.modelTier !== undefined ? { modelTier: input.modelTier } : {}),
|
||||
// A bot runs on a pinned model or a tier, so choosing one clears the other.
|
||||
...(input.modelTier ? { modelProvider: null, modelId: null } : {}),
|
||||
...(input.modelProvider ? { modelTier: null } : {}),
|
||||
...(input.teamChatAmbientEnabled !== undefined
|
||||
? { teamChatAmbientEnabled: input.teamChatAmbientEnabled }
|
||||
: {}),
|
||||
|
||||
113
apps/web/e2e/model-tiers.spec.ts
Normal file
113
apps/web/e2e/model-tiers.spec.ts
Normal file
@ -0,0 +1,113 @@
|
||||
import { createServer } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { expect, type Page, test } from "@playwright/test";
|
||||
import { captureScreenshot, completeOnboarding, openUserSettings, rpc, signup } from "./helpers";
|
||||
|
||||
// One OpenAI-compatible router endpoint serving several local models.
|
||||
const ROUTER_MODELS = ["router-standard", "router-fast", "router-code"];
|
||||
|
||||
async function mappedTiers(page: Page) {
|
||||
const tiers = await rpc<Array<{ tier: string; modelId: string }>>(page, "models/tiers/list", {});
|
||||
return tiers.map((entry) => `${entry.tier}:${entry.modelId}`).sort();
|
||||
}
|
||||
|
||||
async function openTiers(page: Page) {
|
||||
const tiers = page.getByTestId("model-tiers");
|
||||
await tiers.getByText("Tiers", { exact: true }).click();
|
||||
return tiers;
|
||||
}
|
||||
|
||||
async function openBotModel(page: Page) {
|
||||
await page.locator("main").getByRole("button", { name: "Chief", exact: true }).click();
|
||||
const settings = page.getByTestId("bot-settings");
|
||||
await expect(settings).toBeVisible();
|
||||
await settings.getByTestId("bot-settings-advanced").evaluate((element) => {
|
||||
(element as HTMLDetailsElement).open = true;
|
||||
});
|
||||
return { settings, model: settings.getByRole("combobox", { name: "Model", exact: true }) };
|
||||
}
|
||||
|
||||
test("maps tiers to the models of one router endpoint and lets a bot ask for a tier", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.method === "GET" && request.url === "/v1/models") {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify({ object: "list", data: ROUTER_MODELS.map((id) => ({ id })) }));
|
||||
return;
|
||||
}
|
||||
response.writeHead(404);
|
||||
response.end();
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
|
||||
try {
|
||||
const baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1`;
|
||||
const stamp = Date.now();
|
||||
await signup(page, `model-tiers-${stamp}@rakazo.test`, "password12", `Tiers ${stamp}`);
|
||||
await completeOnboarding(page);
|
||||
|
||||
await openUserSettings(page, "models");
|
||||
// Nothing to map before a model is connected.
|
||||
await expect(page.getByTestId("model-tiers")).toBeHidden();
|
||||
await page.getByPlaceholder("Search providers").fill("openai-compatible");
|
||||
await page.getByRole("button", { name: /OpenAI-compatible/ }).click();
|
||||
await page.getByLabel("OpenAI-compatible server URL").fill(baseUrl);
|
||||
await page.getByLabel("Model id").fill("router-standard");
|
||||
await page.getByRole("button", { name: "Save", exact: true }).click();
|
||||
await expect(page.getByText("Saved.", { exact: true })).toBeVisible();
|
||||
const [router] = await rpc<Array<{ id: string }>>(page, "models/credentials", {});
|
||||
|
||||
const tiers = await openTiers(page);
|
||||
const fast = tiers.getByRole("combobox", { name: "Fast", exact: true });
|
||||
const code = tiers.getByRole("combobox", { name: "Code", exact: true });
|
||||
await expect(fast).toHaveValue("");
|
||||
// Every model the router lists becomes a choice for each tier.
|
||||
for (const modelId of ROUTER_MODELS) {
|
||||
await expect(code.locator(`option[value="${router!.id}::${modelId}"]`)).toHaveCount(1);
|
||||
}
|
||||
await fast.selectOption(`${router!.id}::router-fast`);
|
||||
await code.selectOption(`${router!.id}::router-code`);
|
||||
await expect.poll(() => mappedTiers(page)).toEqual(["code:router-code", "fast:router-fast"]);
|
||||
await captureScreenshot(page, testInfo, "model-tiers-settings");
|
||||
|
||||
await page.reload();
|
||||
await openUserSettings(page, "models");
|
||||
const reopened = await openTiers(page);
|
||||
await expect(reopened.getByRole("combobox", { name: "Code", exact: true })).toHaveValue(
|
||||
`${router!.id}::router-code`,
|
||||
);
|
||||
await reopened.getByRole("combobox", { name: "Fast", exact: true }).selectOption("");
|
||||
await expect.poll(() => mappedTiers(page)).toEqual(["code:router-code"]);
|
||||
await page.getByRole("button", { name: "Close model settings" }).click();
|
||||
|
||||
// A mapped tier is a model choice for a bot; an unmapped one is not offered.
|
||||
const { settings, model } = await openBotModel(page);
|
||||
await expect(model.locator('option[value="tier:code"]')).toHaveText("Code (router-code)");
|
||||
await expect(model.locator('option[value="tier:fast"]')).toHaveCount(0);
|
||||
await model.selectOption("tier:code");
|
||||
await model.scrollIntoViewIfNeeded();
|
||||
await captureScreenshot(page, testInfo, "model-tiers-bot-settings");
|
||||
const saved = page.waitForResponse(
|
||||
(response) => response.url().includes("/rpc/bots/update") && response.ok(),
|
||||
);
|
||||
await settings.getByRole("button", { name: "Save", exact: true }).click();
|
||||
await saved;
|
||||
const bots = await rpc<
|
||||
Array<{ name: string; modelTier: string | null; modelProvider: string | null }>
|
||||
>(page, "bots/list", {});
|
||||
expect(bots.find((bot) => bot.name === "Chief")).toMatchObject({
|
||||
modelTier: "code",
|
||||
modelProvider: null,
|
||||
});
|
||||
|
||||
await page.reload();
|
||||
const after = await openBotModel(page);
|
||||
await expect(after.model).toHaveValue("tier:code");
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
58
apps/web/src/lib/connected-models.test.ts
Normal file
58
apps/web/src/lib/connected-models.test.ts
Normal file
@ -0,0 +1,58 @@
|
||||
import type { ModelCatalogEntry, ModelCredential } from "@rakazo/contracts";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { connectedModelOptions } from "./connected-models";
|
||||
|
||||
const catalog: ModelCatalogEntry[] = [
|
||||
{ provider: "xai", providerName: "xAI", id: "grok-4.6", label: "Grok 4.6", billing: "" },
|
||||
{ provider: "xai", providerName: "xAI", id: "grok-mini", label: "Grok mini", billing: "" },
|
||||
{
|
||||
provider: "openai-compatible",
|
||||
providerName: "OpenAI-compatible",
|
||||
id: "custom",
|
||||
label: "Custom",
|
||||
billing: "",
|
||||
placeholder: true,
|
||||
},
|
||||
];
|
||||
|
||||
const credentials: ModelCredential[] = [
|
||||
{
|
||||
id: "router",
|
||||
provider: "openai-compatible",
|
||||
label: "OpenAI-compatible",
|
||||
hasKey: true,
|
||||
isDefault: true,
|
||||
modelId: "router-standard",
|
||||
},
|
||||
{ id: "xai", provider: "xai", label: "xAI", hasKey: true, isDefault: false, modelId: "grok-4.6" },
|
||||
];
|
||||
|
||||
describe("connected model options", () => {
|
||||
it("expands catalog providers and keeps a free-form connection's saved model", () => {
|
||||
expect(connectedModelOptions(credentials, catalog)).toEqual([
|
||||
{
|
||||
credentialId: "router",
|
||||
provider: "openai-compatible",
|
||||
modelId: "router-standard",
|
||||
label: "OpenAI-compatible · router-standard",
|
||||
},
|
||||
{ credentialId: "xai", provider: "xai", modelId: "grok-4.6", label: "xAI · Grok 4.6" },
|
||||
{ credentialId: "xai", provider: "xai", modelId: "grok-mini", label: "xAI · Grok mini" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("adds every model a router endpoint lists, once each", () => {
|
||||
const options = connectedModelOptions(credentials, catalog, {
|
||||
router: ["router-standard", "router-fast", "router-code"],
|
||||
});
|
||||
|
||||
expect(options.filter((option) => option.credentialId === "router")).toEqual(
|
||||
["router-standard", "router-fast", "router-code"].map((modelId) => ({
|
||||
credentialId: "router",
|
||||
provider: "openai-compatible",
|
||||
modelId,
|
||||
label: `OpenAI-compatible · ${modelId}`,
|
||||
})),
|
||||
);
|
||||
});
|
||||
});
|
||||
46
apps/web/src/lib/connected-models.ts
Normal file
46
apps/web/src/lib/connected-models.ts
Normal file
@ -0,0 +1,46 @@
|
||||
import type { ModelCatalogEntry, ModelCredential } from "@rakazo/contracts";
|
||||
|
||||
export type ConnectedModelOption = {
|
||||
credentialId: string;
|
||||
provider: string;
|
||||
modelId: string;
|
||||
label: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Models the user's connections can run: every catalog model of a catalog provider, and the
|
||||
* saved id of a free-form connection plus any ids its server lists (keyed by credential id).
|
||||
*/
|
||||
export function connectedModelOptions(
|
||||
credentials: ModelCredential[],
|
||||
catalog: ModelCatalogEntry[],
|
||||
serverModels: Record<string, string[]> = {},
|
||||
): ConnectedModelOption[] {
|
||||
const options: ConnectedModelOption[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const credential of credentials) {
|
||||
const providerModels = catalog.filter(
|
||||
(entry) => entry.provider === credential.provider && !entry.placeholder,
|
||||
);
|
||||
const listed = serverModels[credential.id] ?? [];
|
||||
const freeForm =
|
||||
Boolean(credential.modelId) &&
|
||||
!providerModels.some((entry) => entry.id === credential.modelId);
|
||||
const candidates =
|
||||
freeForm || listed.length
|
||||
? [credential.modelId, ...listed]
|
||||
.filter((modelId): modelId is string => Boolean(modelId))
|
||||
.map((modelId) => ({ modelId, label: `${credential.label} · ${modelId}` }))
|
||||
: providerModels.map((entry) => ({
|
||||
modelId: entry.id,
|
||||
label: `${entry.providerName ?? entry.provider} · ${entry.label}`,
|
||||
}));
|
||||
for (const candidate of candidates) {
|
||||
const key = `${credential.provider}::${candidate.modelId}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
options.push({ credentialId: credential.id, provider: credential.provider, ...candidate });
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
8
apps/web/src/lib/model-tier-label.ts
Normal file
8
apps/web/src/lib/model-tier-label.ts
Normal file
@ -0,0 +1,8 @@
|
||||
import { t } from "@lingui/core/macro";
|
||||
import type { ModelTier } from "@rakazo/contracts";
|
||||
|
||||
export function modelTierLabel(tier: ModelTier) {
|
||||
if (tier === "fast") return t`Fast`;
|
||||
if (tier === "code") return t`Code`;
|
||||
return t`Standard`;
|
||||
}
|
||||
@ -40,6 +40,7 @@ import { localizedProviderHint } from "../lib/localized-provider-hint";
|
||||
import type { ModelCatalogEntry, ModelCredential } from "../lib/model-auth";
|
||||
import { rpc } from "../lib/rpc";
|
||||
import { useModelOAuthSignIn } from "../lib/use-model-oauth-signin";
|
||||
import { ModelTierSettings } from "./ModelTierSettings";
|
||||
|
||||
export function ModelSettingsOverlay({
|
||||
onClose,
|
||||
@ -398,6 +399,7 @@ export function ModelSettingsOverlay({
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<ModelTierSettings credentials={credentials} catalog={catalog} />
|
||||
|
||||
<div className="flex min-h-0 flex-1 flex-col gap-6 overflow-hidden px-6 py-6 sm:px-8 md:flex-row">
|
||||
<div className="flex min-h-0 shrink-0 flex-col md:w-[310px]">
|
||||
|
||||
129
apps/web/src/pages/ModelTierSettings.tsx
Normal file
129
apps/web/src/pages/ModelTierSettings.tsx
Normal file
@ -0,0 +1,129 @@
|
||||
import { Trans, useLingui } from "@lingui/react/macro";
|
||||
import type {
|
||||
ModelCatalogEntry,
|
||||
ModelCredential,
|
||||
ModelTier,
|
||||
ModelTierMapping,
|
||||
} from "@rakazo/contracts";
|
||||
import { MODEL_TIERS, OPENAI_COMPATIBLE_PROVIDER_ID } from "@rakazo/contracts";
|
||||
import { NativeSelect, NativeSelectOption } from "@rakazo/ui-web";
|
||||
import { useId, useRef, useState } from "react";
|
||||
import { connectedModelOptions } from "../lib/connected-models";
|
||||
import { modelTierLabel } from "../lib/model-tier-label";
|
||||
import { rpc } from "../lib/rpc";
|
||||
|
||||
function tierKey(model: { credentialId: string; modelId: string }) {
|
||||
return `${model.credentialId}::${model.modelId}`;
|
||||
}
|
||||
|
||||
function parseTierKey(key: string) {
|
||||
const separator = key.indexOf("::");
|
||||
if (separator <= 0) return null;
|
||||
return { credentialId: key.slice(0, separator), modelId: key.slice(separator + 2) };
|
||||
}
|
||||
|
||||
/** Maps each tier to one connected model; bots and helpers that ask for a tier run on it. */
|
||||
export function ModelTierSettings({
|
||||
credentials,
|
||||
catalog,
|
||||
}: {
|
||||
credentials: ModelCredential[];
|
||||
catalog: ModelCatalogEntry[];
|
||||
}) {
|
||||
const { t } = useLingui();
|
||||
const ids = useId();
|
||||
const [tiers, setTiers] = useState<ModelTierMapping[] | null>(null);
|
||||
const [serverModels, setServerModels] = useState<Record<string, string[]>>({});
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const saveQueueRef = useRef(Promise.resolve());
|
||||
|
||||
function loadTiers() {
|
||||
return rpc.models.tiers
|
||||
.list()
|
||||
.then(setTiers)
|
||||
.catch((err: unknown) =>
|
||||
setError(err instanceof Error ? err.message : t`Could not load model settings`),
|
||||
);
|
||||
}
|
||||
|
||||
function load() {
|
||||
void loadTiers();
|
||||
// A router endpoint serves several models; list them so each tier can pick one.
|
||||
for (const credential of credentials) {
|
||||
if (credential.provider !== OPENAI_COMPATIBLE_PROVIDER_ID) continue;
|
||||
void rpc.models
|
||||
.probeOpenAiCompatible({ credentialId: credential.id })
|
||||
.then(({ models }) =>
|
||||
setServerModels((current) => ({ ...current, [credential.id]: models })),
|
||||
)
|
||||
// The saved model stays selectable when the server cannot be reached.
|
||||
.catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
function chooseModel(tier: ModelTier, key: string) {
|
||||
const model = parseTierKey(key);
|
||||
const provider = credentials.find((entry) => entry.id === model?.credentialId)?.provider;
|
||||
setError(null);
|
||||
setTiers((current) => [
|
||||
...(current ?? []).filter((entry) => entry.tier !== tier),
|
||||
...(model && provider ? [{ tier, provider, ...model }] : []),
|
||||
]);
|
||||
// Serialize saves so a slower earlier request cannot land after a newer choice.
|
||||
saveQueueRef.current = saveQueueRef.current.then(() =>
|
||||
rpc.models.tiers.set({ tier, model }).then(
|
||||
() => undefined,
|
||||
(err: unknown) => {
|
||||
setError(err instanceof Error ? err.message : t`Could not save`);
|
||||
return loadTiers();
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
if (!credentials.length) return null;
|
||||
const options = connectedModelOptions(credentials, catalog, serverModels);
|
||||
|
||||
return (
|
||||
<details
|
||||
data-testid="model-tiers"
|
||||
className="mx-6 mt-3 text-ui-body text-muted-foreground sm:mx-8"
|
||||
onToggle={(event) => {
|
||||
if (event.currentTarget.open && !tiers) load();
|
||||
}}
|
||||
>
|
||||
<summary className="w-fit cursor-pointer select-none">
|
||||
<Trans>Tiers</Trans>
|
||||
</summary>
|
||||
<div className="mt-2 grid gap-3 sm:grid-cols-3">
|
||||
{MODEL_TIERS.map((tier) => {
|
||||
const mapped = tiers?.find((entry) => entry.tier === tier);
|
||||
const value = mapped ? tierKey(mapped) : "";
|
||||
return (
|
||||
<label key={tier} htmlFor={`${ids}-${tier}`} className="block">
|
||||
{modelTierLabel(tier)}
|
||||
<NativeSelect
|
||||
id={`${ids}-${tier}`}
|
||||
className="mt-1.5 w-full text-foreground"
|
||||
value={value}
|
||||
disabled={!tiers}
|
||||
onChange={(event) => chooseModel(tier, event.target.value)}
|
||||
>
|
||||
<NativeSelectOption value="">{t`Space default`}</NativeSelectOption>
|
||||
{mapped && !options.some((option) => tierKey(option) === value) ? (
|
||||
<NativeSelectOption value={value}>{mapped.modelId}</NativeSelectOption>
|
||||
) : null}
|
||||
{options.map((option) => (
|
||||
<NativeSelectOption key={tierKey(option)} value={tierKey(option)}>
|
||||
{option.label}
|
||||
</NativeSelectOption>
|
||||
))}
|
||||
</NativeSelect>
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
{error ? <p className="mt-2 text-ui-body-sm text-destructive">{error}</p> : null}
|
||||
</details>
|
||||
);
|
||||
}
|
||||
@ -7,6 +7,8 @@ import type {
|
||||
Me,
|
||||
ModelCatalogEntry,
|
||||
ModelCredential,
|
||||
ModelTier,
|
||||
ModelTierMapping,
|
||||
ThinkingLevel,
|
||||
VoiceInfo,
|
||||
} from "@rakazo/contracts";
|
||||
@ -14,12 +16,15 @@ import {
|
||||
BOT_DESCRIPTION_MAX_LENGTH,
|
||||
BOT_NAME_MAX_LENGTH,
|
||||
BOT_TITLE_MAX_LENGTH,
|
||||
MODEL_TIERS,
|
||||
ModelTierSchema,
|
||||
normalizeBotTags,
|
||||
} from "@rakazo/contracts";
|
||||
import {
|
||||
Button,
|
||||
Input,
|
||||
NativeSelect,
|
||||
NativeSelectOptGroup,
|
||||
NativeSelectOption,
|
||||
Switch,
|
||||
Textarea,
|
||||
@ -27,7 +32,9 @@ import {
|
||||
} from "@rakazo/ui-web";
|
||||
import { X } from "lucide-react";
|
||||
import { Suspense, useEffect, useId, useRef, useState } from "react";
|
||||
import { connectedModelOptions } from "../../lib/connected-models";
|
||||
import { lazyWithRecovery } from "../../lib/lazy-recovery";
|
||||
import { modelTierLabel } from "../../lib/model-tier-label";
|
||||
import { rpc } from "../../lib/rpc";
|
||||
import { AvatarStudioPopover } from "./avatar-studio-popover";
|
||||
import { BotCapabilitiesFields } from "./bot-capabilities";
|
||||
@ -217,6 +224,7 @@ export function BotSettings({
|
||||
tags?: string[];
|
||||
restrictBuiltinTools?: boolean;
|
||||
allowedBuiltinTools?: string[];
|
||||
modelTier?: ModelTier | null;
|
||||
}) => Promise<void>;
|
||||
onExport: () => Promise<void>;
|
||||
onClear: () => void;
|
||||
@ -235,7 +243,11 @@ export function BotSettings({
|
||||
const [voiceId, setVoiceId] = useState(bot.voiceId ?? "");
|
||||
const [voices, setVoices] = useState<VoiceInfo[]>([]);
|
||||
const [modelKey, setModelKey] = useState(
|
||||
bot.modelProvider && bot.modelId ? modelOptionKey(bot.modelProvider, bot.modelId) : "",
|
||||
bot.modelTier
|
||||
? tierOptionKey(bot.modelTier)
|
||||
: bot.modelProvider && bot.modelId
|
||||
? modelOptionKey(bot.modelProvider, bot.modelId)
|
||||
: "",
|
||||
);
|
||||
const [thinkingLevel, setThinkingLevel] = useState(bot.thinkingLevel ?? "");
|
||||
const [tags, setTags] = useState(bot.tags.join(", "));
|
||||
@ -244,6 +256,7 @@ export function BotSettings({
|
||||
const [toolNames, setToolNames] = useState<string[] | null>(null);
|
||||
const [credentials, setCredentials] = useState<ModelCredential[]>([]);
|
||||
const [catalog, setCatalog] = useState<ModelCatalogEntry[]>([]);
|
||||
const [tiers, setTiers] = useState<ModelTierMapping[]>([]);
|
||||
const [me, setMe] = useState<Me | null>(null);
|
||||
const [modelMetaReady, setModelMetaReady] = useState(false);
|
||||
const [saving, setSaving] = useState(false);
|
||||
@ -267,11 +280,17 @@ export function BotSettings({
|
||||
.builtinTools({ botId: bot.id })
|
||||
.then(setToolNames)
|
||||
.catch(() => undefined);
|
||||
void Promise.all([rpc.models.credentials(), rpc.models.list(), rpc.me()])
|
||||
.then(([nextCredentials, nextCatalog, nextMe]) => {
|
||||
void Promise.all([
|
||||
rpc.models.credentials(),
|
||||
rpc.models.list(),
|
||||
rpc.me(),
|
||||
rpc.models.tiers.list().catch(() => []),
|
||||
])
|
||||
.then(([nextCredentials, nextCatalog, nextMe, nextTiers]) => {
|
||||
setCredentials(nextCredentials);
|
||||
setCatalog(nextCatalog);
|
||||
setMe(nextMe);
|
||||
setTiers(nextTiers);
|
||||
// Only mark ready on success — a failed catalog load must not clear
|
||||
// an existing thinkingLevel override on save.
|
||||
setModelMetaReady(true);
|
||||
@ -279,60 +298,49 @@ export function BotSettings({
|
||||
.catch(() => undefined);
|
||||
}, []);
|
||||
|
||||
const connectedOptions: Array<{
|
||||
key: string;
|
||||
provider: string;
|
||||
modelId: string;
|
||||
label: string;
|
||||
}> = [];
|
||||
const seenOptions = new Set<string>();
|
||||
for (const credential of credentials) {
|
||||
const providerModels = catalog.filter(
|
||||
(entry) => entry.provider === credential.provider && !entry.placeholder,
|
||||
);
|
||||
const credentialInCatalog = Boolean(
|
||||
credential.modelId && providerModels.some((entry) => entry.id === credential.modelId),
|
||||
);
|
||||
// Catalog providers expand to every model for that connection. Free-form
|
||||
// credentials (model id not in the catalog) stay a single connected pair.
|
||||
const options =
|
||||
credential.modelId && !credentialInCatalog
|
||||
? [
|
||||
{
|
||||
key: modelOptionKey(credential.provider, credential.modelId),
|
||||
provider: credential.provider,
|
||||
modelId: credential.modelId,
|
||||
label: `${credential.label} · ${credential.modelId}`,
|
||||
},
|
||||
]
|
||||
: providerModels.map((entry) => ({
|
||||
key: modelOptionKey(entry.provider, entry.id),
|
||||
provider: entry.provider,
|
||||
modelId: entry.id,
|
||||
label: `${entry.providerName ?? entry.provider} · ${entry.label}`,
|
||||
}));
|
||||
for (const option of options) {
|
||||
if (seenOptions.has(option.key)) continue;
|
||||
seenOptions.add(option.key);
|
||||
connectedOptions.push(option);
|
||||
}
|
||||
}
|
||||
|
||||
const effectiveProvider = modelKey
|
||||
? parseModelOptionKey(modelKey)?.provider
|
||||
: (me?.defaultProvider ?? null);
|
||||
const effectiveModelId = modelKey
|
||||
? parseModelOptionKey(modelKey)?.modelId
|
||||
: (me?.defaultModel ?? null);
|
||||
const connectedOptions = connectedModelOptions(credentials, catalog).map((option) => ({
|
||||
...option,
|
||||
key: modelOptionKey(option.provider, option.modelId),
|
||||
}));
|
||||
const selectedTier = parseTierOptionKey(modelKey);
|
||||
const selectedTierModel = tiers.find((entry) => entry.tier === selectedTier);
|
||||
// A mapped tier runs its model; an unmapped one falls back to the Space default.
|
||||
const effectiveProvider = selectedTierModel
|
||||
? selectedTierModel.provider
|
||||
: modelKey && !selectedTier
|
||||
? parseModelOptionKey(modelKey)?.provider
|
||||
: (me?.defaultProvider ?? null);
|
||||
const effectiveModelId = selectedTierModel
|
||||
? selectedTierModel.modelId
|
||||
: modelKey && !selectedTier
|
||||
? parseModelOptionKey(modelKey)?.modelId
|
||||
: (me?.defaultModel ?? null);
|
||||
const effectiveEntry =
|
||||
effectiveProvider && effectiveModelId
|
||||
? catalog.find(
|
||||
(entry) => entry.provider === effectiveProvider && entry.id === effectiveModelId,
|
||||
)
|
||||
: undefined;
|
||||
const effectiveCredential = credentials.find(
|
||||
(entry) => entry.provider === effectiveProvider && entry.modelId === effectiveModelId,
|
||||
);
|
||||
// One connection serves every model of its tiers with the same reasoning settings.
|
||||
const effectiveCredential = selectedTierModel
|
||||
? credentials.find((entry) => entry.id === selectedTierModel.credentialId)
|
||||
: credentials.find(
|
||||
(entry) => entry.provider === effectiveProvider && entry.modelId === effectiveModelId,
|
||||
);
|
||||
const tierOptions = MODEL_TIERS.flatMap((tier) => {
|
||||
const mapped = tiers.find((entry) => entry.tier === tier);
|
||||
// Tiers appear once mapped; an unmapped current choice stays visible.
|
||||
if (!mapped && tier !== selectedTier) return [];
|
||||
const model = mapped
|
||||
? (catalogLabel(catalog, mapped.provider, mapped.modelId) ?? mapped.modelId)
|
||||
: null;
|
||||
return [
|
||||
{
|
||||
key: tierOptionKey(tier),
|
||||
label: model ? `${modelTierLabel(tier)} (${model})` : modelTierLabel(tier),
|
||||
},
|
||||
];
|
||||
});
|
||||
const thinkingOptions = (
|
||||
effectiveCredential?.thinkingLevels ??
|
||||
effectiveEntry?.thinkingLevels ??
|
||||
@ -347,7 +355,7 @@ export function BotSettings({
|
||||
color?: string;
|
||||
notifyOnFinish?: boolean;
|
||||
}) {
|
||||
const selected = modelKey ? parseModelOptionKey(modelKey) : null;
|
||||
const selected = modelKey && !selectedTier ? parseModelOptionKey(modelKey) : null;
|
||||
const nextName = (patchOverrides?.name !== undefined ? patchOverrides.name : name).trim();
|
||||
const nextTitle = (patchOverrides?.title !== undefined ? patchOverrides.title : title).trim();
|
||||
const nextDescription = (
|
||||
@ -386,6 +394,7 @@ export function BotSettings({
|
||||
allowedBuiltinTools: toolNames
|
||||
? allowedBuiltinTools.filter((tool) => toolNames.includes(tool))
|
||||
: allowedBuiltinTools,
|
||||
modelTier: selectedTier,
|
||||
...(modelMetaReady
|
||||
? {
|
||||
thinkingLevel: thinkingOptions.length
|
||||
@ -532,7 +541,18 @@ export function BotSettings({
|
||||
? ` (${catalogLabel(catalog, me.defaultProvider, me.defaultModel) ?? me.defaultModel})`
|
||||
: ""}
|
||||
</NativeSelectOption>
|
||||
{modelKey && !connectedOptions.some((option) => option.key === modelKey) ? (
|
||||
{tierOptions.length ? (
|
||||
<NativeSelectOptGroup label={t`Tiers`}>
|
||||
{tierOptions.map((option) => (
|
||||
<NativeSelectOption key={option.key} value={option.key}>
|
||||
{option.label}
|
||||
</NativeSelectOption>
|
||||
))}
|
||||
</NativeSelectOptGroup>
|
||||
) : null}
|
||||
{modelKey &&
|
||||
!selectedTier &&
|
||||
!connectedOptions.some((option) => option.key === modelKey) ? (
|
||||
<NativeSelectOption value={modelKey}>
|
||||
{parseModelOptionKey(modelKey)?.modelId ?? modelKey}
|
||||
</NativeSelectOption>
|
||||
@ -666,6 +686,17 @@ function modelOptionKey(provider: string, modelId: string) {
|
||||
return `${provider}::${modelId}`;
|
||||
}
|
||||
|
||||
// Model keys always contain "::", so a tier key can never collide with one.
|
||||
function tierOptionKey(tier: ModelTier) {
|
||||
return `tier:${tier}`;
|
||||
}
|
||||
|
||||
function parseTierOptionKey(key: string): ModelTier | null {
|
||||
if (!key.startsWith("tier:")) return null;
|
||||
const tier = ModelTierSchema.safeParse(key.slice("tier:".length));
|
||||
return tier.success ? tier.data : null;
|
||||
}
|
||||
|
||||
function thinkingLevelLabel(level: ThinkingLevel) {
|
||||
if (level === "xhigh") return t`Extra high`;
|
||||
if (level === "low") return t`Low`;
|
||||
|
||||
@ -223,6 +223,12 @@ screenshot computer tools stay available. Existing connections default to disabl
|
||||
managed endpoints, the deployment-wide fallback remains
|
||||
`RAKAZO_OPENAI_COMPATIBLE_VISION_MODELS=gpt4o-vision,llava`.
|
||||
|
||||
**Settings → Models → Tiers** maps the tiers fast, standard, and code to models of your
|
||||
connections, per space. One router endpoint that serves several models (for example llama.cpp
|
||||
behind a model router) can cover all three, because each tier offers every model the server
|
||||
lists. A bot asks for a tier through its **Model** setting, and a helper can ask for one per
|
||||
call. A tier with no model mapped falls back to the usual model choice.
|
||||
|
||||
For servers that accept standard `reasoning_effort`, enable **Supports thinking** under
|
||||
**Advanced** when connecting. The setting is saved on the connection (no env var or restart).
|
||||
Existing connections default to disabled. Reconnect former Qwen-list or deployment-local models
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import type { ConnectionCatalogItem, SandboxKind } from "@rakazo/contracts";
|
||||
import type { ConnectionCatalogItem, ModelTier, SandboxKind } from "@rakazo/contracts";
|
||||
|
||||
export interface AdapterContext {
|
||||
operationId: string;
|
||||
@ -393,6 +393,8 @@ export interface AgentRunRequest {
|
||||
model: AgentRunModel;
|
||||
/** Resolve an explicitly requested helper model within the active user and space scope. */
|
||||
resolveModel?: (provider: string, modelId: string) => Promise<AgentRunModel>;
|
||||
/** Resolve the model the space maps a helper's tier to; null when the tier is unmapped. */
|
||||
resolveModelTier?: (tier: ModelTier) => Promise<AgentRunModel | null>;
|
||||
resumeFromCheckpoint?: string;
|
||||
script?: ScriptedTurn[];
|
||||
/**
|
||||
|
||||
@ -2,12 +2,16 @@ import type { ConnectorTool } from "@rakazo/adapter-kit";
|
||||
import {
|
||||
BotSecretDestination,
|
||||
BotSecretName,
|
||||
MODEL_TIERS,
|
||||
SecretAskPurpose,
|
||||
SecretHttpRequest,
|
||||
USER_FACT_CATEGORIES,
|
||||
} from "@rakazo/contracts";
|
||||
import { z } from "zod";
|
||||
|
||||
export const SUBAGENT_MODEL_TIER_DESCRIPTION =
|
||||
"Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set.";
|
||||
|
||||
export const DELEGATION_TOOL_NAMES = new Set([
|
||||
"run_subagent",
|
||||
"spawn_bot",
|
||||
@ -772,6 +776,11 @@ export const builtinAgentTools: ConnectorTool[] = [
|
||||
type: "string",
|
||||
description: "Optional connected model ID. Set together with model_provider.",
|
||||
},
|
||||
model_tier: {
|
||||
type: "string",
|
||||
enum: [...MODEL_TIERS],
|
||||
description: SUBAGENT_MODEL_TIER_DESCRIPTION,
|
||||
},
|
||||
},
|
||||
required: ["name", "task"],
|
||||
},
|
||||
|
||||
@ -1704,4 +1704,113 @@ description: Prepare standup notes
|
||||
thinkingLevel: "high",
|
||||
});
|
||||
});
|
||||
|
||||
// One router connection serves several local models; tiers pick among them.
|
||||
function tierPrisma(bot: Record<string, unknown> | null, tierRow: unknown) {
|
||||
const plaintext = serializeModelSecret({
|
||||
kind: "openai_compatible",
|
||||
baseUrl: "http://127.0.0.1:8080/v1",
|
||||
reasoning: true,
|
||||
});
|
||||
const findUnique = vi.fn(async () => tierRow);
|
||||
const prisma = {
|
||||
bot: { findFirst: vi.fn(async () => bot) },
|
||||
spaceModelTier: { findUnique },
|
||||
spaceModelPreference: {
|
||||
findFirst: vi.fn(async (args: { where: { credential?: { provider?: string } } }) =>
|
||||
args.where.credential?.provider === "xai"
|
||||
? modelPreference({
|
||||
provider: "xai",
|
||||
secretId: "secret-xai",
|
||||
modelId: "grok-4.6",
|
||||
isDefault: false,
|
||||
})
|
||||
: null,
|
||||
),
|
||||
},
|
||||
userModelCredential: { findFirst: vi.fn(async () => null) },
|
||||
deploymentSettings: { findUnique: vi.fn(async () => null) },
|
||||
secret: {
|
||||
findFirst: vi.fn(async (args: { where: { id: string } }) =>
|
||||
args.where.id === "secret-router" ? { id: "secret-router", ciphertext: plaintext } : null,
|
||||
),
|
||||
findUnique: vi.fn(async () => null),
|
||||
},
|
||||
} as unknown as PrismaClient;
|
||||
const executor = createRunExecutor({
|
||||
prisma,
|
||||
secretStore: { load: vi.fn(() => plaintext), put: vi.fn() },
|
||||
} as unknown as Parameters<typeof createRunExecutor>[0]);
|
||||
return { executor, findUnique };
|
||||
}
|
||||
|
||||
const routerTier = {
|
||||
tier: "code",
|
||||
credentialId: "credential-router",
|
||||
modelId: "router-code",
|
||||
credential: {
|
||||
id: "credential-router",
|
||||
userId: "user-1",
|
||||
provider: "openai-compatible",
|
||||
label: "Router",
|
||||
secretId: "secret-router",
|
||||
supportsImages: false,
|
||||
createdAt: new Date(0),
|
||||
updatedAt: new Date(0),
|
||||
preferences: [{ isDefault: true, modelId: "router-default" }],
|
||||
},
|
||||
};
|
||||
|
||||
it("runs a bot on the model its space maps to the bot's tier", async () => {
|
||||
const { executor, findUnique } = tierPrisma(
|
||||
{ modelProvider: "xai", modelId: "grok-4.6", thinkingLevel: "high", modelTier: "code" },
|
||||
routerTier,
|
||||
);
|
||||
|
||||
const model = await executor.resolveModel({
|
||||
userId: "user-1",
|
||||
spaceId: "ws-1",
|
||||
botId: "bot-1",
|
||||
});
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "openai-compatible",
|
||||
id: "router-code",
|
||||
baseUrl: "http://127.0.0.1:8080/v1",
|
||||
reasoning: true,
|
||||
thinkingLevel: "high",
|
||||
});
|
||||
expect(findUnique).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
where: { spaceId_userId_tier: { spaceId: "ws-1", userId: "user-1", tier: "code" } },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the bot's own model when its tier is not mapped", async () => {
|
||||
const { executor } = tierPrisma(
|
||||
{ modelProvider: "xai", modelId: "grok-4.6", thinkingLevel: null, modelTier: "code" },
|
||||
null,
|
||||
);
|
||||
|
||||
await expect(
|
||||
executor.resolveModel({ userId: "user-1", spaceId: "ws-1", botId: "bot-1" }),
|
||||
).resolves.toMatchObject({ provider: "xai", id: "grok-4.6" });
|
||||
});
|
||||
|
||||
it("resolves a helper tier and reports an unmapped one as null", async () => {
|
||||
const mapped = tierPrisma(null, routerTier);
|
||||
await expect(
|
||||
mapped.executor.resolveTierModel({ userId: "user-1", spaceId: "ws-1" }, "code"),
|
||||
).resolves.toMatchObject({
|
||||
provider: "openai-compatible",
|
||||
id: "router-code",
|
||||
baseUrl: "http://127.0.0.1:8080/v1",
|
||||
});
|
||||
|
||||
const unmapped = tierPrisma(null, null);
|
||||
await expect(
|
||||
unmapped.executor.resolveTierModel({ userId: "user-1", spaceId: "ws-1" }, "fast"),
|
||||
).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@ -25,7 +25,7 @@ import {
|
||||
routineWakeupJob,
|
||||
runContinueJob,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type { RunStatus } from "@rakazo/contracts";
|
||||
import type { ModelTier, RunStatus } from "@rakazo/contracts";
|
||||
import {
|
||||
ATTACHMENT_MAX_BYTES,
|
||||
BOT_DESCRIPTION_MAX_LENGTH,
|
||||
@ -92,6 +92,7 @@ import {
|
||||
effectiveMemoryScope,
|
||||
findDefaultModelCredential,
|
||||
findModelCredential,
|
||||
findSpaceModelTier,
|
||||
InvalidSpaceNameError,
|
||||
isTooManyDatabaseConnections,
|
||||
listSpaceUserBotIds,
|
||||
@ -649,27 +650,13 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
const web = deps.web ?? createWebProvider();
|
||||
const browser = deps.browser ?? createBrowserProvider(undefined, { sandbox: deps.sandbox });
|
||||
const cloudAgent = deps.cloudAgent;
|
||||
const resolveConnectedModel = async (
|
||||
const connectedModel = async (
|
||||
scope: { userId: string; spaceId: string },
|
||||
credential: Parameters<typeof resolveModelKey>[3],
|
||||
provider: string,
|
||||
modelId: string,
|
||||
registerSecrets?: (values: string[]) => void,
|
||||
): Promise<AgentRunRequest["model"]> => {
|
||||
const validationError = await validateConnectedModelChoice(
|
||||
deps.prisma,
|
||||
scope,
|
||||
provider,
|
||||
modelId,
|
||||
);
|
||||
if (validationError) throw new Error(validationError);
|
||||
const credential = await findModelCredential(deps.prisma, scope, provider, modelId);
|
||||
if (!credential) throw new Error("Connect that model provider first");
|
||||
// Free-form selections must keep the preference that owns this modelId. A
|
||||
// intervening delete/change can make findModelCredential fall back to another
|
||||
// same-provider credential; reject that mismatch instead of mixing baseUrl.
|
||||
if (!isCatalogModelChoice(provider, modelId) && credential.defaultModel !== modelId) {
|
||||
throw new Error("Unknown model for that provider");
|
||||
}
|
||||
const resolved = await resolveModelKey(
|
||||
deps,
|
||||
scope.userId,
|
||||
@ -695,8 +682,48 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
: undefined,
|
||||
};
|
||||
};
|
||||
const resolveConnectedModel = async (
|
||||
scope: { userId: string; spaceId: string },
|
||||
provider: string,
|
||||
modelId: string,
|
||||
registerSecrets?: (values: string[]) => void,
|
||||
): Promise<AgentRunRequest["model"]> => {
|
||||
const validationError = await validateConnectedModelChoice(
|
||||
deps.prisma,
|
||||
scope,
|
||||
provider,
|
||||
modelId,
|
||||
);
|
||||
if (validationError) throw new Error(validationError);
|
||||
const credential = await findModelCredential(deps.prisma, scope, provider, modelId);
|
||||
if (!credential) throw new Error("Connect that model provider first");
|
||||
// Free-form selections must keep the preference that owns this modelId. A
|
||||
// intervening delete/change can make findModelCredential fall back to another
|
||||
// same-provider credential; reject that mismatch instead of mixing baseUrl.
|
||||
if (!isCatalogModelChoice(provider, modelId) && credential.defaultModel !== modelId) {
|
||||
throw new Error("Unknown model for that provider");
|
||||
}
|
||||
return connectedModel(scope, credential, provider, modelId, registerSecrets);
|
||||
};
|
||||
/** The model a space maps this tier to; null when the tier is not mapped. */
|
||||
const resolveTierModel = async (
|
||||
scope: { userId: string; spaceId: string },
|
||||
tier: ModelTier,
|
||||
registerSecrets?: (values: string[]) => void,
|
||||
): Promise<AgentRunRequest["model"] | null> => {
|
||||
const mapped = await findSpaceModelTier(deps.prisma, scope, tier);
|
||||
if (!mapped) return null;
|
||||
return connectedModel(
|
||||
scope,
|
||||
mapped.credential,
|
||||
mapped.credential.provider,
|
||||
mapped.modelId,
|
||||
registerSecrets,
|
||||
);
|
||||
};
|
||||
return {
|
||||
resolveConnectedModel,
|
||||
resolveTierModel,
|
||||
async resolveModel(scope: {
|
||||
userId: string;
|
||||
spaceId: string;
|
||||
@ -709,19 +736,21 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
userId: scope.userId,
|
||||
spaceId: scope.spaceId,
|
||||
},
|
||||
select: { modelProvider: true, modelId: true, thinkingLevel: true },
|
||||
select: { modelProvider: true, modelId: true, thinkingLevel: true, modelTier: true },
|
||||
})
|
||||
: null;
|
||||
const hasOverride = Boolean(override?.modelProvider && override.modelId);
|
||||
const [overrideCredential, defaultCredential, settings] = await Promise.all([
|
||||
const [overrideCredential, tierModel, defaultCredential, settings] = await Promise.all([
|
||||
hasOverride
|
||||
? findModelCredential(deps.prisma, scope, override!.modelProvider!, override!.modelId)
|
||||
: Promise.resolve(null),
|
||||
override?.modelTier ? findSpaceModelTier(deps.prisma, scope, override.modelTier) : null,
|
||||
findDefaultModelCredential(deps.prisma, scope),
|
||||
deps.prisma.deploymentSettings.findUnique({ where: { id: "default" } }),
|
||||
]);
|
||||
const selected = selectConfiguredModel({
|
||||
bot: override,
|
||||
tierModel,
|
||||
overrideCredential,
|
||||
defaultCredential,
|
||||
settings,
|
||||
@ -1098,6 +1127,9 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
hasModelOverride && bot.modelProvider
|
||||
? await findModelCredential(deps.prisma, run, bot.modelProvider, bot.modelId)
|
||||
: null;
|
||||
const tierModel = bot.modelTier
|
||||
? await findSpaceModelTier(deps.prisma, run, bot.modelTier)
|
||||
: null;
|
||||
runAbortController = new AbortController();
|
||||
if (!leaseValid) runAbortController.abort();
|
||||
if (deps.shutdownSignal?.aborted) runAbortController.abort(deps.shutdownSignal.reason);
|
||||
@ -1263,6 +1295,7 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
const runtimeFallback = runtimeFallbackModel(deps.runtime);
|
||||
const selected = selectConfiguredModel({
|
||||
bot,
|
||||
tierModel,
|
||||
overrideCredential,
|
||||
defaultCredential,
|
||||
settings,
|
||||
@ -3791,6 +3824,9 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
resolveConnectedModel(run, provider, modelId, (values) =>
|
||||
runSecrets.push(...values),
|
||||
),
|
||||
resolveModelTier: scripted
|
||||
? undefined
|
||||
: (tier) => resolveTierModel(run, tier, (values) => runSecrets.push(...values)),
|
||||
onToolCompleted: (completion) =>
|
||||
appendToolCompletionAudit(
|
||||
deps,
|
||||
|
||||
@ -12,6 +12,7 @@ function credential(provider: string, defaultModel: string | null) {
|
||||
provider,
|
||||
label: provider,
|
||||
secretId: `secret-${provider}`,
|
||||
supportsImages: false,
|
||||
createdAt: new Date(0),
|
||||
updatedAt: new Date(0),
|
||||
isDefault: false,
|
||||
@ -21,6 +22,9 @@ function credential(provider: string, defaultModel: string | null) {
|
||||
|
||||
const spaceCredential = credential("space-provider", "space-model");
|
||||
const overrideCredential = credential("bot-provider", "stored-model");
|
||||
// One router connection serves every tier model; its saved model is a different one.
|
||||
const routerCredential = credential("openai-compatible", "router-default");
|
||||
const tierModel = { modelId: "router-code", credential: routerCredential };
|
||||
const bot = { modelProvider: "bot-provider", modelId: "bot-model", thinkingLevel: "high" };
|
||||
const defaults: SelectionInput = {
|
||||
bot: null,
|
||||
@ -76,6 +80,46 @@ describe("configured model selection", () => {
|
||||
thinkingLevel: "high",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "runs a mapped tier ahead of the bot's own model",
|
||||
input: { bot, overrideCredential, tierModel },
|
||||
expected: {
|
||||
provider: "openai-compatible",
|
||||
id: "router-code",
|
||||
credential: routerCredential,
|
||||
thinkingLevel: "high",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "runs a mapped tier ahead of the Space default",
|
||||
input: { bot: { modelProvider: null, modelId: null, thinkingLevel: null }, tierModel },
|
||||
expected: {
|
||||
provider: "openai-compatible",
|
||||
id: "router-code",
|
||||
credential: routerCredential,
|
||||
thinkingLevel: null,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "keeps the bot's own model when its tier is unmapped",
|
||||
input: { bot, overrideCredential, tierModel: null },
|
||||
expected: {
|
||||
provider: "bot-provider",
|
||||
id: "bot-model",
|
||||
credential: overrideCredential,
|
||||
thinkingLevel: "high",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "keeps the Space default when the tier is unmapped and nothing is pinned",
|
||||
input: { bot: { modelProvider: null, modelId: null, thinkingLevel: "low" }, tierModel: null },
|
||||
expected: {
|
||||
provider: "space-provider",
|
||||
id: "space-model",
|
||||
credential: spaceCredential,
|
||||
thinkingLevel: "low",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "uses settings before deployment defaults without inventing a credential",
|
||||
input: { defaultCredential: null },
|
||||
|
||||
@ -48,12 +48,23 @@ export function selectConfiguredModel(input: {
|
||||
modelId: string | null;
|
||||
thinkingLevel: string | null;
|
||||
} | null;
|
||||
/** The space's model for the bot's tier (findSpaceModelTier); null when the tier is unmapped. */
|
||||
tierModel?: { modelId: string; credential: NonNullable<ModelCredential> } | null;
|
||||
overrideCredential: ModelCredential;
|
||||
defaultCredential: ModelCredential;
|
||||
settings: { defaultModelProvider: string | null; defaultModelId: string | null } | null;
|
||||
deployment: { provider: string; model: string } | null;
|
||||
}) {
|
||||
const { bot, overrideCredential, defaultCredential, settings, deployment } = input;
|
||||
const { bot, tierModel, overrideCredential, defaultCredential, settings, deployment } = input;
|
||||
// A mapped tier wins; an unmapped one leaves the usual choice below untouched.
|
||||
if (tierModel) {
|
||||
return {
|
||||
provider: tierModel.credential.provider,
|
||||
id: tierModel.modelId,
|
||||
credential: tierModel.credential,
|
||||
thinkingLevel: (bot?.thinkingLevel as AgentRunRequest["model"]["thinkingLevel"]) ?? null,
|
||||
};
|
||||
}
|
||||
const hasOverride = Boolean(bot?.modelProvider && bot.modelId);
|
||||
// The override provider, model and credential must win together.
|
||||
const useOverride = Boolean(hasOverride && overrideCredential);
|
||||
|
||||
@ -121,6 +121,10 @@ async function runWithModel(
|
||||
thinkingLevel?: "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" | null;
|
||||
}>,
|
||||
maxImagesPerPrompt?: number,
|
||||
resolveModelTier?: (tier: "fast" | "standard" | "code") => Promise<{
|
||||
provider: string;
|
||||
id: string;
|
||||
} | null>,
|
||||
) {
|
||||
const runtime = new PiAgentRuntime();
|
||||
for await (const _event of runtime.run(
|
||||
@ -135,6 +139,7 @@ async function runWithModel(
|
||||
model: { provider, id: modelId, thinkingLevel, maxImagesPerPrompt },
|
||||
executeTool: vi.fn(async () => ({ ok: true })),
|
||||
resolveModel,
|
||||
resolveModelTier,
|
||||
},
|
||||
{
|
||||
operationId: "1",
|
||||
@ -301,6 +306,98 @@ describe("Pi agent thinking level", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("runs a helper on the model its space maps to the requested tier", async () => {
|
||||
fakeAgentState.subagentArgs = { name: "helper", task: "help", model_tier: "fast" };
|
||||
const resolveModelTier = vi.fn(async () => ({ provider: "test", id: "reasoning-model" }));
|
||||
|
||||
await runWithModel(
|
||||
"plain-model",
|
||||
"test",
|
||||
new AbortController().signal,
|
||||
null,
|
||||
undefined,
|
||||
undefined,
|
||||
resolveModelTier,
|
||||
);
|
||||
|
||||
expect(resolveModelTier).toHaveBeenCalledWith("fast");
|
||||
expect(fakeAgentState.models.map((model) => model.id)).toEqual([
|
||||
"plain-model",
|
||||
"reasoning-model",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps a helper on the run's model when its tier is not mapped", async () => {
|
||||
fakeAgentState.subagentArgs = { name: "helper", task: "help", model_tier: "code" };
|
||||
const resolveModelTier = vi.fn(async () => null);
|
||||
|
||||
await runWithModel(
|
||||
"plain-model",
|
||||
"test",
|
||||
new AbortController().signal,
|
||||
null,
|
||||
undefined,
|
||||
undefined,
|
||||
resolveModelTier,
|
||||
);
|
||||
|
||||
expect(resolveModelTier).toHaveBeenCalledWith("code");
|
||||
expect(fakeAgentState.models.map((model) => model.id)).toEqual(["plain-model", "plain-model"]);
|
||||
expect(fakeAgentState.lastSubagentResult).not.toMatchObject({
|
||||
details: { result: expect.stringContaining("Subagent failed") },
|
||||
});
|
||||
});
|
||||
|
||||
it("prefers an explicit helper model over a tier", async () => {
|
||||
fakeAgentState.subagentArgs = {
|
||||
name: "helper",
|
||||
task: "help",
|
||||
model_provider: "xai",
|
||||
model_id: "grok-4.6",
|
||||
model_tier: "fast",
|
||||
};
|
||||
const resolveModel = vi.fn(async (provider: string, modelId: string) => ({
|
||||
provider,
|
||||
id: modelId,
|
||||
}));
|
||||
const resolveModelTier = vi.fn(async () => ({ provider: "test", id: "reasoning-model" }));
|
||||
|
||||
await runWithModel(
|
||||
"plain-model",
|
||||
"test",
|
||||
new AbortController().signal,
|
||||
null,
|
||||
resolveModel,
|
||||
undefined,
|
||||
resolveModelTier,
|
||||
);
|
||||
|
||||
expect(resolveModel).toHaveBeenCalledWith("xai", "grok-4.6");
|
||||
expect(resolveModelTier).not.toHaveBeenCalled();
|
||||
expect(fakeAgentState.models.map((model) => model.id)).toEqual(["plain-model", "grok-4.6"]);
|
||||
});
|
||||
|
||||
it("rejects an unknown helper tier without starting the helper", async () => {
|
||||
fakeAgentState.subagentArgs = { name: "helper", task: "help", model_tier: "huge" };
|
||||
const resolveModelTier = vi.fn();
|
||||
|
||||
await runWithModel(
|
||||
"plain-model",
|
||||
"test",
|
||||
new AbortController().signal,
|
||||
null,
|
||||
undefined,
|
||||
undefined,
|
||||
resolveModelTier,
|
||||
);
|
||||
|
||||
expect(resolveModelTier).not.toHaveBeenCalled();
|
||||
expect(fakeAgentState.models).toHaveLength(1);
|
||||
expect(fakeAgentState.lastSubagentResult).toMatchObject({
|
||||
details: { result: "Subagent failed: model_tier must be one of: fast, standard, code" },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects an incomplete per-call subagent model pair", async () => {
|
||||
fakeAgentState.subagentArgs = {
|
||||
name: "helper",
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
import { validateToolArguments } from "@earendil-works/pi-ai";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { builtinAgentTools } from "./builtin-tools.js";
|
||||
import { parseConnectorToolArgs } from "./lazy-tool-catalog.js";
|
||||
import { jsonSchemaParameters, parametersFor } from "./pi-runtime.js";
|
||||
|
||||
@ -106,3 +108,33 @@ describe("jsonSchemaParameters", () => {
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("run_subagent parameters", () => {
|
||||
const tool = builtinAgentTools.find((entry) => entry.name === "run_subagent")!;
|
||||
const parameters = parametersFor(tool);
|
||||
const call = (args: Record<string, unknown>) => ({
|
||||
type: "toolCall" as const,
|
||||
id: "call-1",
|
||||
name: "run_subagent",
|
||||
arguments: { name: "scout", task: "look", ...args },
|
||||
});
|
||||
|
||||
it("takes an optional model tier and only the known tiers", () => {
|
||||
expect(validateToolArguments({ name: "run_subagent", parameters } as never, call({}))).toEqual({
|
||||
name: "scout",
|
||||
task: "look",
|
||||
});
|
||||
expect(
|
||||
validateToolArguments(
|
||||
{ name: "run_subagent", parameters } as never,
|
||||
call({ model_tier: "code" }),
|
||||
),
|
||||
).toMatchObject({ model_tier: "code" });
|
||||
expect(() =>
|
||||
validateToolArguments(
|
||||
{ name: "run_subagent", parameters } as never,
|
||||
call({ model_tier: "huge" }),
|
||||
),
|
||||
).toThrow(/model_tier/);
|
||||
});
|
||||
});
|
||||
|
||||
@ -14,9 +14,10 @@ import type {
|
||||
AgentToolCompletion,
|
||||
ConnectorTool,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import { MODEL_TIERS, ModelTierSchema } from "@rakazo/contracts";
|
||||
import { getLogger } from "@rakazo/logging";
|
||||
import { isToolPauseResult } from "./approval-effect.js";
|
||||
import { DELEGATION_TOOL_NAMES } from "./builtin-tools.js";
|
||||
import { DELEGATION_TOOL_NAMES, SUBAGENT_MODEL_TIER_DESCRIPTION } from "./builtin-tools.js";
|
||||
import {
|
||||
normalizeOpenAiToolParameters,
|
||||
openAiToolParametersNeedNormalization,
|
||||
@ -650,6 +651,7 @@ function toAgentTool(tool: ConnectorTool, host: ToolHost, exposedName: string):
|
||||
instructions: raw.instructions ? String(raw.instructions) : "",
|
||||
model_provider: raw.model_provider ? String(raw.model_provider) : "",
|
||||
model_id: raw.model_id ? String(raw.model_id) : "",
|
||||
...(raw.model_tier ? { model_tier: String(raw.model_tier).trim().toLowerCase() } : {}),
|
||||
};
|
||||
}
|
||||
if (tool.name === "spawn_bot") {
|
||||
@ -826,6 +828,7 @@ async function executeSubagent(host: ToolHost, executionId: string, args: Record
|
||||
|
||||
const requestedProvider = String(args.model_provider ?? "").trim();
|
||||
const requestedModelId = String(args.model_id ?? "").trim();
|
||||
const requestedTier = String(args.model_tier ?? "").trim();
|
||||
let requestModel = host.request.model;
|
||||
try {
|
||||
if (Boolean(requestedProvider) !== Boolean(requestedModelId)) {
|
||||
@ -836,6 +839,11 @@ async function executeSubagent(host: ToolHost, executionId: string, args: Record
|
||||
throw new Error("Per-call subagent model selection is unavailable");
|
||||
}
|
||||
requestModel = await host.request.resolveModel(requestedProvider, requestedModelId);
|
||||
} else if (requestedTier) {
|
||||
const tier = ModelTierSchema.safeParse(requestedTier);
|
||||
if (!tier.success) throw new Error(`model_tier must be one of: ${MODEL_TIERS.join(", ")}`);
|
||||
// An unmapped tier keeps the helper on this run's model.
|
||||
requestModel = (await host.request.resolveModelTier?.(tier.data)) ?? requestModel;
|
||||
}
|
||||
} catch (error) {
|
||||
const message = sanitizeError(error instanceof Error ? error.message : String(error));
|
||||
@ -1065,6 +1073,12 @@ function builtinParameters(tool: ConnectorTool) {
|
||||
instructions: Type.Optional(Type.String()),
|
||||
model_provider: Type.Optional(Type.String()),
|
||||
model_id: Type.Optional(Type.String()),
|
||||
model_tier: Type.Optional(
|
||||
Type.Union(
|
||||
MODEL_TIERS.map((tier) => Type.Literal(tier)),
|
||||
{ description: SUBAGENT_MODEL_TIER_DESCRIPTION },
|
||||
),
|
||||
),
|
||||
});
|
||||
}
|
||||
if (tool.name === "spawn_bot") {
|
||||
|
||||
@ -388,6 +388,7 @@ export const UpdateBotInput = z
|
||||
modelProvider: z.string().trim().min(1).max(80).nullable().optional(),
|
||||
modelId: z.string().trim().min(1).max(200).nullable().optional(),
|
||||
thinkingLevel: ThinkingLevelSchema.nullable().optional(),
|
||||
modelTier: ModelTierSchema.nullable().optional(),
|
||||
teamChatAmbientEnabled: z.boolean().optional(),
|
||||
teamChatRules: z.string().max(TEAM_CHAT_RULES_MAX_LENGTH).optional(),
|
||||
tags: BotTags.optional(),
|
||||
@ -395,6 +396,14 @@ export const UpdateBotInput = z
|
||||
allowedBuiltinTools: BotAllowedBuiltinTools.optional(),
|
||||
})
|
||||
.superRefine((value, ctx) => {
|
||||
// A bot runs on a pinned model or a tier; setting one clears the other.
|
||||
if (value.modelTier && (value.modelProvider || value.modelId)) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
message: "Choose a model or a tier, not both",
|
||||
path: ["modelTier"],
|
||||
});
|
||||
}
|
||||
const providerProvided = value.modelProvider !== undefined;
|
||||
const modelProvided = value.modelId !== undefined;
|
||||
if (!providerProvided && !modelProvided) return;
|
||||
|
||||
@ -21,6 +21,7 @@ import {
|
||||
ReorderBotsInput,
|
||||
RunActivityRowSchema,
|
||||
RunSchema,
|
||||
SetModelTierInput,
|
||||
UpdateBotInput,
|
||||
UpdateGroupInput,
|
||||
} from "./index.js";
|
||||
@ -231,6 +232,44 @@ describe("contracts", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("lets a bot ask for a model tier and a space map or clear one", () => {
|
||||
expect(UpdateBotInput.parse({ botId: "bot-1", modelTier: "code" }).modelTier).toBe("code");
|
||||
expect(UpdateBotInput.parse({ botId: "bot-1", modelTier: null }).modelTier).toBeNull();
|
||||
expect(UpdateBotInput.safeParse({ botId: "bot-1", modelTier: "huge" }).success).toBe(false);
|
||||
expect(
|
||||
UpdateBotInput.safeParse({
|
||||
botId: "bot-1",
|
||||
modelTier: "code",
|
||||
modelProvider: "xai",
|
||||
modelId: "grok-4.6",
|
||||
}).success,
|
||||
).toBe(false);
|
||||
expect(
|
||||
UpdateBotInput.safeParse({
|
||||
botId: "bot-1",
|
||||
modelTier: "code",
|
||||
modelProvider: null,
|
||||
modelId: null,
|
||||
}).success,
|
||||
).toBe(true);
|
||||
|
||||
expect(
|
||||
SetModelTierInput.parse({
|
||||
tier: "fast",
|
||||
model: { credentialId: "router", modelId: " qwen3-4b " },
|
||||
}),
|
||||
).toEqual({ tier: "fast", model: { credentialId: "router", modelId: "qwen3-4b" } });
|
||||
expect(SetModelTierInput.parse({ tier: "code", model: null })).toEqual({
|
||||
tier: "code",
|
||||
model: null,
|
||||
});
|
||||
expect(
|
||||
SetModelTierInput.safeParse({ tier: "code", model: { credentialId: "router", modelId: " " } })
|
||||
.success,
|
||||
).toBe(false);
|
||||
expect(SetModelTierInput.safeParse({ tier: "large", model: null }).success).toBe(false);
|
||||
});
|
||||
|
||||
it("normalizes group names and rejects duplicate members", () => {
|
||||
expect(CreateGroupInput.parse({ name: " Draft team ", botIds: ["bot-1", "bot-2"] })).toEqual({
|
||||
name: "Draft team",
|
||||
@ -287,6 +326,8 @@ describe("contracts", () => {
|
||||
expect(appContract.agentSecrets.put).toBeTruthy();
|
||||
expect(appContract.agentSecrets.remove).toBeTruthy();
|
||||
expect(appContract.notifications.registerPush).toBeTruthy();
|
||||
expect(appContract.models.tiers.list).toBeTruthy();
|
||||
expect(appContract.models.tiers.set).toBeTruthy();
|
||||
expect(ProductEventType.options).toContain("thread.message.created");
|
||||
expect(ProductEventType.options).toContain("thread.cleared");
|
||||
expect(ProductEventType.options).toContain("thread.subagent");
|
||||
|
||||
@ -9,6 +9,7 @@ export * from "./inbox.js";
|
||||
export * from "./integration-settings.js";
|
||||
export * from "./local-settings.js";
|
||||
export * from "./mcp.js";
|
||||
export * from "./model-tiers.js";
|
||||
export * from "./openai-compatible-ui.js";
|
||||
export * from "./project-goals.js";
|
||||
export * from "./project-sessions.js";
|
||||
|
||||
@ -15,6 +15,8 @@ const procedures = new Set([
|
||||
"models/finishOAuth",
|
||||
"models/cancelOAuth",
|
||||
"models/setDefault",
|
||||
"models/tiers/list",
|
||||
"models/tiers/set",
|
||||
"integrationSetup/get",
|
||||
"integrationSetup/save",
|
||||
]);
|
||||
|
||||
24
packages/contracts/src/model-tiers.ts
Normal file
24
packages/contracts/src/model-tiers.ts
Normal file
@ -0,0 +1,24 @@
|
||||
import * as z from "zod";
|
||||
import { ModelTierSchema } from "./domain.js";
|
||||
import { Id } from "./ids.js";
|
||||
|
||||
/** One tier of a space mapped to a model of a connected credential. */
|
||||
export const ModelTierMappingSchema = z.object({
|
||||
tier: ModelTierSchema,
|
||||
credentialId: Id,
|
||||
provider: z.string(),
|
||||
modelId: z.string(),
|
||||
});
|
||||
export type ModelTierMapping = z.infer<typeof ModelTierMappingSchema>;
|
||||
|
||||
export const SetModelTierInput = z.object({
|
||||
tier: ModelTierSchema,
|
||||
/** null removes the mapping, so bots asking for this tier use their usual model again. */
|
||||
model: z
|
||||
.object({
|
||||
credentialId: Id,
|
||||
modelId: z.string().trim().min(1).max(200),
|
||||
})
|
||||
.nullable(),
|
||||
});
|
||||
export type SetModelTierInput = z.infer<typeof SetModelTierInput>;
|
||||
@ -82,6 +82,7 @@ import {
|
||||
IntegrationProviderConfigSchema,
|
||||
IntegrationSetupStateSchema,
|
||||
} from "./integration-settings.js";
|
||||
import { ModelTierMappingSchema, SetModelTierInput } from "./model-tiers.js";
|
||||
import {
|
||||
CreateProjectGoalInput,
|
||||
ProjectGoalSchema,
|
||||
@ -224,10 +225,14 @@ export const appContract = {
|
||||
connect: oc.input(ModelConnectInputSchema).output(ModelCredentialSchema),
|
||||
probeOpenAiCompatible: oc
|
||||
.input(
|
||||
z.object({
|
||||
baseUrl: z.string(),
|
||||
apiKey: z.string().optional(),
|
||||
}),
|
||||
z.union([
|
||||
z.object({
|
||||
baseUrl: z.string(),
|
||||
apiKey: z.string().optional(),
|
||||
}),
|
||||
// A connected server, probed with its stored address and key.
|
||||
z.object({ credentialId: Id }),
|
||||
]),
|
||||
)
|
||||
.output(z.object({ models: z.array(z.string()) })),
|
||||
beginOAuth: oc
|
||||
@ -258,6 +263,10 @@ export const appContract = {
|
||||
setDefault: oc
|
||||
.input(z.object({ provider: z.string(), modelId: z.string() }))
|
||||
.output(z.object({ ok: z.literal(true) })),
|
||||
tiers: {
|
||||
list: oc.output(z.array(ModelTierMappingSchema)),
|
||||
set: oc.input(SetModelTierInput).output(z.array(ModelTierMappingSchema)),
|
||||
},
|
||||
},
|
||||
bots: {
|
||||
list: oc.output(z.array(BotSchema)),
|
||||
|
||||
@ -11,6 +11,7 @@ export * from "./memory-config.js";
|
||||
export * from "./messages.js";
|
||||
export * from "./messaging.js";
|
||||
export * from "./model-credentials.js";
|
||||
export * from "./model-tiers.js";
|
||||
export * from "./project-goals.js";
|
||||
export * from "./project-workspace.js";
|
||||
export * from "./repos.js";
|
||||
|
||||
108
packages/db/src/model-tiers.test.ts
Normal file
108
packages/db/src/model-tiers.test.ts
Normal file
@ -0,0 +1,108 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { PrismaClient } from "./client.js";
|
||||
import { findSpaceModelTier, listSpaceModelTiers, setSpaceModelTier } from "./model-tiers.js";
|
||||
|
||||
const scope = { userId: "user", spaceId: "space" };
|
||||
const credential = {
|
||||
id: "router",
|
||||
userId: "user",
|
||||
provider: "openai-compatible",
|
||||
label: "Router",
|
||||
secretId: "router-secret",
|
||||
supportsImages: true,
|
||||
createdAt: new Date(0),
|
||||
updatedAt: new Date(0),
|
||||
};
|
||||
|
||||
describe("space model tiers", () => {
|
||||
it("finds a tier within the user's space with its credential shaped for model resolution", async () => {
|
||||
const findUnique = vi.fn().mockResolvedValue({
|
||||
tier: "code",
|
||||
credentialId: "router",
|
||||
modelId: "router-code",
|
||||
credential: { ...credential, preferences: [{ isDefault: true, modelId: "router-default" }] },
|
||||
});
|
||||
const prisma = { spaceModelTier: { findUnique } } as unknown as PrismaClient;
|
||||
|
||||
await expect(findSpaceModelTier(prisma, scope, "code")).resolves.toEqual({
|
||||
tier: "code",
|
||||
credentialId: "router",
|
||||
modelId: "router-code",
|
||||
// The space's saved model stays the credential default, so its image support does not
|
||||
// spread to the tier's model.
|
||||
credential: { ...credential, isDefault: true, defaultModel: "router-default" },
|
||||
});
|
||||
expect(findUnique).toHaveBeenCalledWith({
|
||||
where: { spaceId_userId_tier: { spaceId: "space", userId: "user", tier: "code" } },
|
||||
include: {
|
||||
credential: {
|
||||
include: {
|
||||
preferences: {
|
||||
where: { spaceId: "space", userId: "user" },
|
||||
select: { isDefault: true, modelId: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("reports an unmapped tier as null", async () => {
|
||||
const prisma = {
|
||||
spaceModelTier: { findUnique: vi.fn().mockResolvedValue(null) },
|
||||
} as unknown as PrismaClient;
|
||||
|
||||
await expect(findSpaceModelTier(prisma, scope, "fast")).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it("lists only the user's tiers in the space", async () => {
|
||||
const findMany = vi.fn().mockResolvedValue([
|
||||
{
|
||||
tier: "fast",
|
||||
credentialId: "router",
|
||||
modelId: "router-fast",
|
||||
credential: { ...credential, preferences: [] },
|
||||
},
|
||||
]);
|
||||
const prisma = { spaceModelTier: { findMany } } as unknown as PrismaClient;
|
||||
|
||||
await expect(listSpaceModelTiers(prisma, scope)).resolves.toEqual([
|
||||
{
|
||||
tier: "fast",
|
||||
credentialId: "router",
|
||||
modelId: "router-fast",
|
||||
credential: { ...credential, isDefault: false, defaultModel: null },
|
||||
},
|
||||
]);
|
||||
expect(findMany).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ where: { spaceId: "space", userId: "user" } }),
|
||||
);
|
||||
});
|
||||
|
||||
it("maps a tier with an upsert and clears it with a scoped delete", async () => {
|
||||
const upsert = vi.fn().mockResolvedValue({});
|
||||
const deleteMany = vi.fn().mockResolvedValue({ count: 1 });
|
||||
const prisma = { spaceModelTier: { upsert, deleteMany } } as unknown as PrismaClient;
|
||||
|
||||
await setSpaceModelTier(prisma, scope, "code", {
|
||||
credentialId: "router",
|
||||
modelId: "router-code",
|
||||
});
|
||||
expect(upsert).toHaveBeenCalledWith({
|
||||
where: { spaceId_userId_tier: { spaceId: "space", userId: "user", tier: "code" } },
|
||||
create: {
|
||||
spaceId: "space",
|
||||
userId: "user",
|
||||
tier: "code",
|
||||
credentialId: "router",
|
||||
modelId: "router-code",
|
||||
},
|
||||
update: { credentialId: "router", modelId: "router-code" },
|
||||
});
|
||||
|
||||
await setSpaceModelTier(prisma, scope, "code", null);
|
||||
expect(deleteMany).toHaveBeenCalledWith({
|
||||
where: { spaceId: "space", userId: "user", tier: "code" },
|
||||
});
|
||||
});
|
||||
});
|
||||
83
packages/db/src/model-tiers.ts
Normal file
83
packages/db/src/model-tiers.ts
Normal file
@ -0,0 +1,83 @@
|
||||
import type { Prisma, PrismaClient } from "./client.js";
|
||||
|
||||
type ModelTierScope = { userId: string; spaceId: string };
|
||||
|
||||
// The composite (credentialId, userId) foreign key keeps every mapped credential the user's own.
|
||||
function withCredential(scope: ModelTierScope) {
|
||||
return {
|
||||
credential: {
|
||||
include: {
|
||||
preferences: {
|
||||
where: { spaceId: scope.spaceId, userId: scope.userId },
|
||||
select: { isDefault: true, modelId: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
}
|
||||
|
||||
type ModelTierRow = Prisma.SpaceModelTierGetPayload<{
|
||||
include: ReturnType<typeof withCredential>;
|
||||
}>;
|
||||
|
||||
/**
|
||||
* Shape the credential like findModelCredential does. defaultModel stays the space's saved
|
||||
* model, not the tier's, so capabilities kept for that saved model never spread to tier models.
|
||||
*/
|
||||
function toModelTier(row: ModelTierRow) {
|
||||
const { preferences, ...credential } = row.credential;
|
||||
return {
|
||||
tier: row.tier,
|
||||
credentialId: row.credentialId,
|
||||
modelId: row.modelId,
|
||||
credential: {
|
||||
...credential,
|
||||
isDefault: preferences[0]?.isDefault ?? false,
|
||||
defaultModel: preferences[0]?.modelId ?? null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function listSpaceModelTiers(
|
||||
prisma: Pick<PrismaClient, "spaceModelTier">,
|
||||
scope: ModelTierScope,
|
||||
) {
|
||||
const rows = await prisma.spaceModelTier.findMany({
|
||||
where: { spaceId: scope.spaceId, userId: scope.userId },
|
||||
include: withCredential(scope),
|
||||
orderBy: { createdAt: "asc" },
|
||||
});
|
||||
return rows.map(toModelTier);
|
||||
}
|
||||
|
||||
/** The connected model a space maps this tier to, or null when the tier is not mapped. */
|
||||
export async function findSpaceModelTier(
|
||||
prisma: Pick<PrismaClient, "spaceModelTier">,
|
||||
scope: ModelTierScope,
|
||||
tier: string,
|
||||
) {
|
||||
const row = await prisma.spaceModelTier.findUnique({
|
||||
where: { spaceId_userId_tier: { spaceId: scope.spaceId, userId: scope.userId, tier } },
|
||||
include: withCredential(scope),
|
||||
});
|
||||
return row ? toModelTier(row) : null;
|
||||
}
|
||||
|
||||
export async function setSpaceModelTier(
|
||||
prisma: Pick<PrismaClient, "spaceModelTier">,
|
||||
scope: ModelTierScope,
|
||||
tier: string,
|
||||
model: { credentialId: string; modelId: string } | null,
|
||||
) {
|
||||
if (!model) {
|
||||
await prisma.spaceModelTier.deleteMany({
|
||||
where: { spaceId: scope.spaceId, userId: scope.userId, tier },
|
||||
});
|
||||
return;
|
||||
}
|
||||
await prisma.spaceModelTier.upsert({
|
||||
where: { spaceId_userId_tier: { spaceId: scope.spaceId, userId: scope.userId, tier } },
|
||||
create: { spaceId: scope.spaceId, userId: scope.userId, tier, ...model },
|
||||
update: model,
|
||||
});
|
||||
}
|
||||
@ -407,6 +407,7 @@ export function createRepos(prisma: PrismaClient) {
|
||||
modelProvider?: string | null;
|
||||
modelId?: string | null;
|
||||
thinkingLevel?: string | null;
|
||||
modelTier?: string | null;
|
||||
initialMessage?: {
|
||||
role: "user" | "bot" | "system";
|
||||
blocks: MessageBlock[];
|
||||
@ -426,6 +427,7 @@ export function createRepos(prisma: PrismaClient) {
|
||||
let thinkingLevel = input.thinkingLevel ?? null;
|
||||
let restrictBuiltinTools = input.restrictBuiltinTools ?? false;
|
||||
let allowedBuiltinTools = input.allowedBuiltinTools ?? [];
|
||||
let modelTier = input.modelTier ?? null;
|
||||
if (input.parentBotId) {
|
||||
const parent = await prisma.bot.findFirst({
|
||||
where: {
|
||||
@ -435,9 +437,10 @@ export function createRepos(prisma: PrismaClient) {
|
||||
},
|
||||
});
|
||||
if (!parent) throw new IsolationError();
|
||||
if (!modelId) {
|
||||
if (!modelId && !modelTier) {
|
||||
modelProvider = parent.modelProvider ?? null;
|
||||
modelId = parent.modelId ?? null;
|
||||
modelTier = parent.modelTier ?? null;
|
||||
}
|
||||
if (thinkingLevel == null) thinkingLevel = parent.thinkingLevel ?? null;
|
||||
// A bot it creates never gets builtin tools the restricted parent lacks.
|
||||
@ -489,6 +492,7 @@ export function createRepos(prisma: PrismaClient) {
|
||||
modelProvider,
|
||||
modelId,
|
||||
thinkingLevel,
|
||||
modelTier,
|
||||
},
|
||||
});
|
||||
const thread = await tx.thread.create({
|
||||
|
||||
@ -69,6 +69,7 @@ type SpaceClient = Pick<
|
||||
| "space"
|
||||
| "spaceMember"
|
||||
| "spaceModelPreference"
|
||||
| "spaceModelTier"
|
||||
| "spaceVoicePreference"
|
||||
| "memoryDocument"
|
||||
| "notificationPreference"
|
||||
@ -132,10 +133,13 @@ async function copyProviderPreferences(
|
||||
prisma: SpaceClient,
|
||||
input: { sourceSpaceId: string; targetSpaceId: string; userId: string; createdAt: Date },
|
||||
): Promise<void> {
|
||||
const [modelPreferences, voicePreferences] = await Promise.all([
|
||||
const [modelPreferences, modelTiers, voicePreferences] = await Promise.all([
|
||||
prisma.spaceModelPreference.findMany({
|
||||
where: { spaceId: input.sourceSpaceId, userId: input.userId },
|
||||
}),
|
||||
prisma.spaceModelTier.findMany({
|
||||
where: { spaceId: input.sourceSpaceId, userId: input.userId },
|
||||
}),
|
||||
prisma.spaceVoicePreference.findMany({
|
||||
where: { spaceId: input.sourceSpaceId, userId: input.userId },
|
||||
}),
|
||||
@ -154,6 +158,19 @@ async function copyProviderPreferences(
|
||||
})),
|
||||
})
|
||||
: Promise.resolve(),
|
||||
modelTiers.length
|
||||
? prisma.spaceModelTier.createMany({
|
||||
data: modelTiers.map((tier) => ({
|
||||
spaceId: input.targetSpaceId,
|
||||
userId: input.userId,
|
||||
tier: tier.tier,
|
||||
credentialId: tier.credentialId,
|
||||
modelId: tier.modelId,
|
||||
createdAt: input.createdAt,
|
||||
updatedAt: input.createdAt,
|
||||
})),
|
||||
})
|
||||
: Promise.resolve(),
|
||||
voicePreferences.length
|
||||
? prisma.spaceVoicePreference.createMany({
|
||||
data: voicePreferences.map((preference) => ({
|
||||
|
||||
@ -78,6 +78,9 @@ describeWithDatabase("API authorization and resource isolation", () => {
|
||||
["models/finishOAuth", { loginId: "missing-login" }],
|
||||
["models/cancelOAuth", { loginId: "missing-login" }],
|
||||
["models/setDefault", { provider: "test", modelId: "test/model" }],
|
||||
["models/probeOpenAiCompatible", { credentialId: "missing-credential" }],
|
||||
["models/tiers/list"],
|
||||
["models/tiers/set", { tier: "fast", model: null }],
|
||||
["spaces/list"],
|
||||
["spaces/create", { name: "Nope" }],
|
||||
["spaces/remove", { spaceId: "missing-space" }],
|
||||
@ -410,7 +413,24 @@ describeWithDatabase("API authorization and resource isolation", () => {
|
||||
answer: "mixed-resource attack",
|
||||
});
|
||||
|
||||
const ownerModel = await rpc<ModelCredential>(app, owner, "models/connect", {
|
||||
provider: "scripted",
|
||||
apiKey: "fake-owner-scripted-key",
|
||||
});
|
||||
const ownerTiers = await rpc<unknown[]>(app, owner, "models/tiers/set", {
|
||||
tier: "code",
|
||||
model: { credentialId: ownerModel.id, modelId: "scripted" },
|
||||
});
|
||||
expect(ownerTiers).toEqual([
|
||||
{ tier: "code", credentialId: ownerModel.id, provider: "scripted", modelId: "scripted" },
|
||||
]);
|
||||
|
||||
const resourceIdCalls = [
|
||||
[
|
||||
"models/tiers/set",
|
||||
{ tier: "fast", model: { credentialId: ownerModel.id, modelId: "scripted" } },
|
||||
],
|
||||
["models/probeOpenAiCompatible", { credentialId: ownerModel.id }],
|
||||
["routines/update", { routineId: ownerRoutine.id, name: "Stolen Routine" }],
|
||||
["routines/remove", { routineId: ownerRoutine.id }],
|
||||
["routines/testRun", { routineId: ownerRoutine.id }],
|
||||
@ -437,6 +457,8 @@ describeWithDatabase("API authorization and resource isolation", () => {
|
||||
resourceIdCalls.map(([procedure, input]) => expectDenied(app, intruder, procedure, input)),
|
||||
);
|
||||
|
||||
expect(await rpc<unknown[]>(app, intruder, "models/tiers/list")).toEqual([]);
|
||||
expect(await rpc<unknown[]>(app, owner, "models/tiers/list")).toEqual(ownerTiers);
|
||||
expect(await rpc<unknown[]>(app, intruder, "memory/list", { botId: ownerBot.id })).toEqual([]);
|
||||
expect(await rpc<string>(app, intruder, "memory/exportMarkdown", { botId: ownerBot.id })).toBe(
|
||||
"",
|
||||
@ -1344,6 +1366,64 @@ describeWithDatabase("API authorization and resource isolation", () => {
|
||||
).resolves.toBe(32);
|
||||
});
|
||||
|
||||
it("maps model tiers per Space and carries them into a new Space", async () => {
|
||||
const cookie = await signup(app, `model-tiers-${stamp}@rakazo.test`, "Model Tiers");
|
||||
const model = await rpc<ModelCredential>(app, cookie, "models/connect", {
|
||||
provider: "scripted",
|
||||
apiKey: "fake-tier-scripted-key",
|
||||
});
|
||||
const mapped = [
|
||||
{ tier: "fast", credentialId: model.id, provider: "scripted", modelId: "scripted" },
|
||||
];
|
||||
await expect(
|
||||
rpc(app, cookie, "models/tiers/set", {
|
||||
tier: "fast",
|
||||
model: { credentialId: model.id, modelId: "scripted" },
|
||||
}),
|
||||
).resolves.toEqual(mapped);
|
||||
// Catalog providers only run catalog models; free-form ids belong to OpenAI-compatible servers.
|
||||
const unknownModel = await raw(app, cookie, "models/tiers/set", {
|
||||
tier: "code",
|
||||
model: { credentialId: model.id, modelId: "not-a-catalog-model" },
|
||||
});
|
||||
expect(unknownModel.status).toBe(400);
|
||||
|
||||
const created = await rpc<Space>(app, cookie, "spaces/create", { name: "Tier copy" });
|
||||
await expect(rpc(app, cookie, "models/tiers/list", {}, created.id)).resolves.toEqual(mapped);
|
||||
|
||||
await rpc(app, cookie, "models/tiers/set", { tier: "fast", model: null }, created.id);
|
||||
await expect(rpc(app, cookie, "models/tiers/list", {}, created.id)).resolves.toEqual([]);
|
||||
await expect(rpc(app, cookie, "models/tiers/list")).resolves.toEqual(mapped);
|
||||
|
||||
const bot = await rpc<{ id: string; modelTier: string | null }>(
|
||||
app,
|
||||
cookie,
|
||||
"bots/create",
|
||||
botInput("Tiered"),
|
||||
);
|
||||
await expect(
|
||||
rpc(app, cookie, "bots/update", { botId: bot.id, modelTier: "fast" }),
|
||||
).resolves.toMatchObject({ id: bot.id, modelTier: "fast" });
|
||||
await expect(rpc(app, cookie, "bots/duplicate", { botId: bot.id })).resolves.toMatchObject({
|
||||
modelTier: "fast",
|
||||
});
|
||||
// A pinned model and a tier replace each other, so a client that only knows models
|
||||
// (mobile) never leaves a tier silently overriding the model it picked.
|
||||
await expect(
|
||||
rpc(app, cookie, "bots/update", {
|
||||
botId: bot.id,
|
||||
modelProvider: "scripted",
|
||||
modelId: "scripted",
|
||||
}),
|
||||
).resolves.toMatchObject({ modelProvider: "scripted", modelId: "scripted", modelTier: null });
|
||||
await expect(
|
||||
rpc(app, cookie, "bots/update", { botId: bot.id, modelTier: "code" }),
|
||||
).resolves.toMatchObject({ modelProvider: null, modelId: null, modelTier: "code" });
|
||||
await expect(
|
||||
rpc(app, cookie, "bots/update", { botId: bot.id, modelTier: null }),
|
||||
).resolves.toMatchObject({ id: bot.id, modelTier: null });
|
||||
});
|
||||
|
||||
it("reuses provider credentials and copies their selections into a new Space", async () => {
|
||||
const cookie = await signup(app, `space-provider-copy-${stamp}@rakazo.test`, "Provider Copy");
|
||||
const actor = await rpc<Actor>(app, cookie, "me");
|
||||
|
||||
@ -44,9 +44,12 @@ export interface ModelEmulatorStep {
|
||||
export async function startModelEmulator(options: {
|
||||
steps: ModelEmulatorStep[];
|
||||
modelId?: string;
|
||||
/** Further ids this endpoint routes, like a router serving several local models. */
|
||||
routedModelIds?: string[];
|
||||
apiKey?: string;
|
||||
}) {
|
||||
const modelId = options.modelId ?? "offline-fixture";
|
||||
const routedModelIds = new Set([modelId, ...(options.routedModelIds ?? [])]);
|
||||
const requests: ModelEmulatorRequest[] = [];
|
||||
const failures: Error[] = [];
|
||||
let stepIndex = 0;
|
||||
@ -64,7 +67,7 @@ export async function startModelEmulator(options: {
|
||||
}
|
||||
const body = JSON.parse(Buffer.concat(chunks).toString("utf8")) as ModelEmulatorRequest;
|
||||
requests.push(body);
|
||||
assert.equal(body.model, modelId, "Model routing changed");
|
||||
assert.ok(routedModelIds.has(body.model), "Model routing changed");
|
||||
assert.equal(body.stream, true, "Expected streaming model request");
|
||||
assert.ok(Array.isArray(body.messages), "Expected model messages");
|
||||
const step = options.steps[stepIndex++];
|
||||
@ -87,7 +90,7 @@ export async function startModelEmulator(options: {
|
||||
id: `fixture-${stepIndex}`,
|
||||
object: "chat.completion.chunk",
|
||||
created: 0,
|
||||
model: modelId,
|
||||
model: body.model,
|
||||
choices: [{ index: 0, delta, finish_reason: finishReason }],
|
||||
})}\n\n`,
|
||||
);
|
||||
|
||||
@ -147,6 +147,154 @@ describe.skipIf(!databaseAvailable)("offline Pi product journey", () => {
|
||||
}
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it("runs a bot's tier and a helper's tier on models of one router connection", async () => {
|
||||
const fixtureKey = "offline-router-fixture-key";
|
||||
const lastToolResult = (request: { messages: Array<{ role: string; content?: unknown }> }) =>
|
||||
String(request.messages.findLast((message) => message.role === "tool")?.content);
|
||||
const model = await startModelEmulator({
|
||||
apiKey: fixtureKey,
|
||||
modelId: "router-standard",
|
||||
routedModelIds: ["router-fast", "router-code"],
|
||||
steps: [
|
||||
{
|
||||
expect(request) {
|
||||
expect(request.model).toBe("router-code");
|
||||
const helper = request.tools?.find((tool) => tool.function.name === "run_subagent");
|
||||
expect(JSON.stringify(helper?.function.parameters)).toContain("model_tier");
|
||||
},
|
||||
response: {
|
||||
type: "tool",
|
||||
id: "tier-helper",
|
||||
name: "run_subagent",
|
||||
arguments: { name: "scout", task: "Count to three.", model_tier: "fast" },
|
||||
},
|
||||
},
|
||||
{
|
||||
expect(request) {
|
||||
expect(request.model).toBe("router-fast");
|
||||
expect(JSON.stringify(request.messages)).toContain("Count to three.");
|
||||
},
|
||||
response: { type: "text", text: "one two three" },
|
||||
},
|
||||
{
|
||||
expect(request) {
|
||||
expect(request.model).toBe("router-code");
|
||||
expect(lastToolResult(request)).toContain("one two three");
|
||||
},
|
||||
response: { type: "text", text: "The helper counted." },
|
||||
},
|
||||
{
|
||||
// With its tier cleared the bot is back on the connection's saved model.
|
||||
expect(request) {
|
||||
expect(request.model).toBe("router-standard");
|
||||
},
|
||||
response: { type: "text", text: "Back on the saved model." },
|
||||
},
|
||||
],
|
||||
});
|
||||
const dataDir = await mkdtemp(path.join(tmpdir(), "rakazo-offline-tiers-"));
|
||||
let stop: (() => Promise<void>) | undefined;
|
||||
try {
|
||||
const { createApp } = await import("../../../apps/api/src/app.ts");
|
||||
const handles = await createApp({
|
||||
databaseUrl: process.env.DATABASE_URL!,
|
||||
realtimeDatabaseUrl: process.env.DATABASE_URL!,
|
||||
authUrl: fixtureOrigin,
|
||||
webOrigin: fixtureOrigin,
|
||||
dataDir,
|
||||
sandboxProvider: "fake",
|
||||
agentRuntime: "pi",
|
||||
wakeupDriver: "memory",
|
||||
signupsEnabled: "true",
|
||||
composio: new ComposioEmulator(),
|
||||
encryptionKey: "offline-model-fixture-encryption-key",
|
||||
});
|
||||
stop = handles.stop;
|
||||
const signup = await handles.app.request("/api/auth/sign-up/email", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", origin: fixtureOrigin },
|
||||
body: JSON.stringify({
|
||||
email: `offline-tiers-${randomUUID()}@rakazo.test`,
|
||||
password: "password12",
|
||||
name: "Offline tiers",
|
||||
}),
|
||||
});
|
||||
expect(signup.status).toBeLessThan(400);
|
||||
const cookie = sessionCookieHeader(signup);
|
||||
const router = await rpc<{ id: string }>(handles.app, cookie, "models/connect", {
|
||||
provider: model.model.provider,
|
||||
modelId: model.model.id,
|
||||
baseUrl: model.baseUrl,
|
||||
apiKey: fixtureKey,
|
||||
});
|
||||
for (const [tier, modelId] of [
|
||||
["code", "router-code"],
|
||||
["fast", "router-fast"],
|
||||
]) {
|
||||
await rpc(handles.app, cookie, "models/tiers/set", {
|
||||
tier,
|
||||
model: { credentialId: router.id, modelId },
|
||||
});
|
||||
}
|
||||
const bot = await rpc<{ id: string }>(handles.app, cookie, "bots/create", {
|
||||
name: "Tier fixture",
|
||||
title: "",
|
||||
description: "",
|
||||
instructions: "Complete the task.",
|
||||
notifyOnFinish: false,
|
||||
});
|
||||
await rpc(handles.app, cookie, "bots/update", { botId: bot.id, modelTier: "code" });
|
||||
const completedRun = async (text: string) => {
|
||||
const sent = await rpc<{ runId: string }>(handles.app, cookie, "threads/send", {
|
||||
botId: bot.id,
|
||||
text,
|
||||
});
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const run = await handles.prisma.run.findUnique({
|
||||
where: { id: sent.runId },
|
||||
select: { status: true },
|
||||
});
|
||||
if (run?.status === "failed") model.assertComplete();
|
||||
return run?.status;
|
||||
},
|
||||
{ timeout: 15_000, interval: 100 },
|
||||
)
|
||||
.toBe("completed");
|
||||
return handles.prisma.run.findUniqueOrThrow({
|
||||
where: { id: sent.runId },
|
||||
select: { modelProvider: true, modelId: true },
|
||||
});
|
||||
};
|
||||
|
||||
await expect(completedRun("Ask a fast helper to count.")).resolves.toEqual({
|
||||
modelProvider: "openai-compatible",
|
||||
modelId: "router-code",
|
||||
});
|
||||
await rpc(handles.app, cookie, "models/tiers/set", { tier: "code", model: null });
|
||||
await expect(completedRun("Answer directly.")).resolves.toEqual({
|
||||
modelProvider: "openai-compatible",
|
||||
modelId: "router-standard",
|
||||
});
|
||||
model.assertComplete();
|
||||
const thread = await rpc<{ messages: unknown[] }>(handles.app, cookie, "threads/get", {
|
||||
botId: bot.id,
|
||||
});
|
||||
const messages = JSON.stringify(thread.messages);
|
||||
expect(messages).toContain("The helper counted.");
|
||||
expect(messages).toContain("Back on the saved model.");
|
||||
expect(messages).not.toContain(fixtureKey);
|
||||
} finally {
|
||||
try {
|
||||
await stop?.();
|
||||
} finally {
|
||||
await model.close();
|
||||
await rm(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}, 45_000);
|
||||
});
|
||||
|
||||
async function rpc<T>(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user