evOo-local/apps/web/e2e/model-tiers.spec.ts

import { createServer } from "node:http";
import type { AddressInfo } from "node:net";
import { expect, type Page, test } from "@playwright/test";
import { captureScreenshot, completeOnboarding, openUserSettings, rpc, signup } from "./helpers";

// One OpenAI-compatible router endpoint serving several local models.
const ROUTER_MODELS = ["router-standard", "router-fast", "router-code"];

async function mappedTiers(page: Page) {
  const tiers = await rpc<Array<{ tier: string; modelId: string }>>(page, "models/tiers/list", {});
  return tiers.map((entry) => `${entry.tier}:${entry.modelId}`).sort();
}

async function openTiers(page: Page) {
  const tiers = page.getByTestId("model-tiers");
  await tiers.getByText("Tiers", { exact: true }).click();
  return tiers;
}

async function openBotModel(page: Page) {
  await page.locator("main").getByRole("button", { name: "Chief", exact: true }).click();
  const settings = page.getByTestId("bot-settings");
  await expect(settings).toBeVisible();
  await settings.getByTestId("bot-settings-advanced").evaluate((element) => {
    (element as HTMLDetailsElement).open = true;
  });
  return { settings, model: settings.getByRole("combobox", { name: "Model", exact: true }) };
}

test("maps tiers to the models of one router endpoint and lets a bot ask for a tier", async ({
  page,
}, testInfo) => {
  const server = createServer((request, response) => {
    if (request.method === "GET" && request.url === "/v1/models") {
      response.writeHead(200, { "content-type": "application/json" });
      response.end(JSON.stringify({ object: "list", data: ROUTER_MODELS.map((id) => ({ id })) }));
      return;
    }
    response.writeHead(404);
    response.end();
  });
  await new Promise<void>((resolve, reject) => {
    server.once("error", reject);
    server.listen(0, "127.0.0.1", resolve);
  });

  try {
    const baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1`;
    const stamp = Date.now();
    await signup(page, `model-tiers-${stamp}@rakazo.test`, "password12", `Tiers ${stamp}`);
    await completeOnboarding(page);

    await openUserSettings(page, "models");
    // Nothing to map before a model is connected.
    await expect(page.getByTestId("model-tiers")).toBeHidden();
    await page.getByPlaceholder("Search providers").fill("openai-compatible");
    await page.getByRole("button", { name: /OpenAI-compatible/ }).click();
    await page.getByLabel("OpenAI-compatible server URL").fill(baseUrl);
    await page.getByLabel("Model id").fill("router-standard");
    await page.getByRole("button", { name: "Save", exact: true }).click();
    await expect(page.getByText("Saved.", { exact: true })).toBeVisible();
    const [router] = await rpc<Array<{ id: string }>>(page, "models/credentials", {});

    const tiers = await openTiers(page);
    const fast = tiers.getByRole("combobox", { name: "Fast", exact: true });
    const code = tiers.getByRole("combobox", { name: "Code", exact: true });
    await expect(fast).toHaveValue("");
    // Every model the router lists becomes a choice for each tier.
    for (const modelId of ROUTER_MODELS) {
      await expect(code.locator(`option[value="${router!.id}::${modelId}"]`)).toHaveCount(1);
    }
    await fast.selectOption(`${router!.id}::router-fast`);
    await code.selectOption(`${router!.id}::router-code`);
    await expect.poll(() => mappedTiers(page)).toEqual(["code:router-code", "fast:router-fast"]);
    await captureScreenshot(page, testInfo, "model-tiers-settings");

    await page.reload();
    await openUserSettings(page, "models");
    const reopened = await openTiers(page);
    await expect(reopened.getByRole("combobox", { name: "Code", exact: true })).toHaveValue(
      `${router!.id}::router-code`,
    );
    await reopened.getByRole("combobox", { name: "Fast", exact: true }).selectOption("");
    await expect.poll(() => mappedTiers(page)).toEqual(["code:router-code"]);
    await page.getByRole("button", { name: "Close model settings" }).click();

    // A mapped tier is a model choice for a bot; an unmapped one is not offered.
    const { settings, model } = await openBotModel(page);
    await expect(model.locator('option[value="tier:code"]')).toHaveText("Code (router-code)");
    await expect(model.locator('option[value="tier:fast"]')).toHaveCount(0);
    await model.selectOption("tier:code");
    await model.scrollIntoViewIfNeeded();
    await captureScreenshot(page, testInfo, "model-tiers-bot-settings");
    const saved = page.waitForResponse(
      (response) => response.url().includes("/rpc/bots/update") && response.ok(),
    );
    await settings.getByRole("button", { name: "Save", exact: true }).click();
    await saved;
    const bots = await rpc<
      Array<{ name: string; modelTier: string | null; modelProvider: string | null }>
    >(page, "bots/list", {});
    expect(bots.find((bot) => bot.name === "Chief")).toMatchObject({
      modelTier: "code",
      modelProvider: null,
    });

    await page.reload();
    const after = await openBotModel(page);
    await expect(after.model).toHaveValue("tier:code");
  } finally {
    await new Promise<void>((resolve) => server.close(() => resolve()));
  }
});