import { createServer } from "node:http"; import type { AddressInfo } from "node:net"; import { expect, type Page, test } from "@playwright/test"; import { captureScreenshot, completeOnboarding, openUserSettings, rpc, signup } from "./helpers"; // One OpenAI-compatible router endpoint serving several local models. const ROUTER_MODELS = ["router-standard", "router-fast", "router-code"]; async function mappedTiers(page: Page) { const tiers = await rpc>(page, "models/tiers/list", {}); return tiers.map((entry) => `${entry.tier}:${entry.modelId}`).sort(); } async function openTiers(page: Page) { const tiers = page.getByTestId("model-tiers"); await tiers.getByText("Tiers", { exact: true }).click(); return tiers; } async function openBotModel(page: Page) { await page.locator("main").getByRole("button", { name: "Chief", exact: true }).click(); const settings = page.getByTestId("bot-settings"); await expect(settings).toBeVisible(); await settings.getByTestId("bot-settings-advanced").evaluate((element) => { (element as HTMLDetailsElement).open = true; }); return { settings, model: settings.getByRole("combobox", { name: "Model", exact: true }) }; } test("maps tiers to the models of one router endpoint and lets a bot ask for a tier", async ({ page, }, testInfo) => { const server = createServer((request, response) => { if (request.method === "GET" && request.url === "/v1/models") { response.writeHead(200, { "content-type": "application/json" }); response.end(JSON.stringify({ object: "list", data: ROUTER_MODELS.map((id) => ({ id })) })); return; } response.writeHead(404); response.end(); }); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); try { const baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1`; const stamp = Date.now(); await signup(page, `model-tiers-${stamp}@rakazo.test`, "password12", `Tiers ${stamp}`); await completeOnboarding(page); await openUserSettings(page, "models"); // Nothing to map before a model is connected. await expect(page.getByTestId("model-tiers")).toBeHidden(); await page.getByPlaceholder("Search providers").fill("openai-compatible"); await page.getByRole("button", { name: /OpenAI-compatible/ }).click(); await page.getByLabel("OpenAI-compatible server URL").fill(baseUrl); await page.getByLabel("Model id").fill("router-standard"); await page.getByRole("button", { name: "Save", exact: true }).click(); await expect(page.getByText("Saved.", { exact: true })).toBeVisible(); const [router] = await rpc>(page, "models/credentials", {}); const tiers = await openTiers(page); const fast = tiers.getByRole("combobox", { name: "Fast", exact: true }); const code = tiers.getByRole("combobox", { name: "Code", exact: true }); await expect(fast).toHaveValue(""); // Every model the router lists becomes a choice for each tier. for (const modelId of ROUTER_MODELS) { await expect(code.locator(`option[value="${router!.id}::${modelId}"]`)).toHaveCount(1); } await fast.selectOption(`${router!.id}::router-fast`); await code.selectOption(`${router!.id}::router-code`); await expect.poll(() => mappedTiers(page)).toEqual(["code:router-code", "fast:router-fast"]); await captureScreenshot(page, testInfo, "model-tiers-settings"); await page.reload(); await openUserSettings(page, "models"); const reopened = await openTiers(page); await expect(reopened.getByRole("combobox", { name: "Code", exact: true })).toHaveValue( `${router!.id}::router-code`, ); await reopened.getByRole("combobox", { name: "Fast", exact: true }).selectOption(""); await expect.poll(() => mappedTiers(page)).toEqual(["code:router-code"]); await page.getByRole("button", { name: "Close model settings" }).click(); // A mapped tier is a model choice for a bot; an unmapped one is not offered. const { settings, model } = await openBotModel(page); await expect(model.locator('option[value="tier:code"]')).toHaveText("Code (router-code)"); await expect(model.locator('option[value="tier:fast"]')).toHaveCount(0); await model.selectOption("tier:code"); await model.scrollIntoViewIfNeeded(); await captureScreenshot(page, testInfo, "model-tiers-bot-settings"); const saved = page.waitForResponse( (response) => response.url().includes("/rpc/bots/update") && response.ok(), ); await settings.getByRole("button", { name: "Save", exact: true }).click(); await saved; const bots = await rpc< Array<{ name: string; modelTier: string | null; modelProvider: string | null }> >(page, "bots/list", {}); expect(bots.find((bot) => bot.name === "Chief")).toMatchObject({ modelTier: "code", modelProvider: null, }); await page.reload(); const after = await openBotModel(page); await expect(after.model).toHaveValue("tier:code"); } finally { await new Promise((resolve) => server.close(() => resolve())); } });