import { createProvider, type Model, type MutableModels, type Provider, } from "@earendil-works/pi-ai"; import { openAICompletionsApi } from "@earendil-works/pi-ai/api/openai-completions.lazy"; import { type ModelTier, ModelTierSchema } from "@rakazo/contracts"; import { declaredVisionModelIds, inputModalities } from "./model-modalities.js"; /** * Local OpenAI-compatible model server (Ollama, LM Studio, llama.cpp, MLX). * * Pi's built-in catalog only ships hosted providers, so a model running on the * operator's own machine has no catalog entry to select. This registers one * from environment configuration. The server is keyless: `resolve` returns a * placeholder because OpenAI-compatible local servers ignore the header, but * Models treats a provider with no resolvable auth as unconfigured and hides * its models. */ export const LOCAL_PROVIDER_ID = "local"; /** Placeholder sent as the key: local servers ignore it, and it is distinct enough to redact safely. */ export const LOCAL_API_KEY = "rakazo-local-model-server"; /** Model ids the local server serves with vision, declared by the operator. */ export const LOCAL_VISION_MODELS_ENV = "RAKAZO_LOCAL_VISION_MODELS"; export function localVisionModelIds(): ReadonlySet { return declaredVisionModelIds(LOCAL_VISION_MODELS_ENV); } const DEFAULT_BASE_URL = "http://127.0.0.1:11434/v1"; const DEFAULT_CONTEXT_WINDOW = 32_768; const DEFAULT_MAX_TOKENS = 4_096; export function localBaseUrl(): string { return checkedUrl(process.env.RAKAZO_LOCAL_MODELS_URL?.trim() || DEFAULT_BASE_URL); } function checkedUrl(value: string, name = "RAKAZO_LOCAL_MODELS_URL"): string { let url: URL; try { url = new URL(value); } catch { throw new Error(`${name} must be an absolute HTTP(S) URL`); } if (url.protocol !== "http:" && url.protocol !== "https:") { throw new Error(`${name} must be an absolute HTTP(S) URL`); } return value; } /** * A token count from the environment, or the default when unset. * * Token limits are only meaningful as finite positive integers, so anything * else is a configuration mistake. Throwing beats `Number(x) || default`, which * would accept a negative window and silently swallow a typo as the default. */ function tokenLimit(name: string, fallback: number): number { const raw = process.env[name]?.trim(); if (!raw) return fallback; const value = Number(raw); if (!Number.isSafeInteger(value) || value <= 0) { throw new Error(`${name} must be a positive integer, received "${raw}"`); } return value; } /** * Comma-separated model ids exactly as the local server names them. An entry may name its own * server as `id=url` when the models run behind different endpoints. */ function localModelEntries(env: NodeJS.ProcessEnv): { id: string; baseUrl?: string }[] { return (env.RAKAZO_LOCAL_MODELS ?? "") .split(",") .map((entry) => entry.trim()) .filter((entry) => entry.length > 0) .map((entry) => { const separator = entry.indexOf("="); if (separator < 0) return { id: entry }; return { id: entry.slice(0, separator).trim(), baseUrl: entry.slice(separator + 1).trim() }; }); } export function localModelIds(env: NodeJS.ProcessEnv = process.env): string[] { return localModelEntries(env).map((entry) => entry.id); } /** * The deployment's tiers served by local models, from `RAKAZO_LOCAL_TIERS` (`tier=id,…`). * Entries naming an unknown tier or a model outside `RAKAZO_LOCAL_MODELS` are ignored. */ export function localTierModels( env: NodeJS.ProcessEnv = process.env, ): Partial> { const ids = new Set(localModelIds(env)); const tiers: Partial> = {}; for (const entry of (env.RAKAZO_LOCAL_TIERS ?? "").split(",")) { const [tier, id] = entry.split("=").map((part) => part.trim()); const parsed = ModelTierSchema.safeParse(tier); if (parsed.success && id && ids.has(id)) tiers[parsed.data] = id; } return tiers; } function localModel(entry: { id: string; baseUrl?: string }): Model<"openai-completions"> { const { id } = entry; return { id, name: id, api: "openai-completions", provider: LOCAL_PROVIDER_ID, baseUrl: entry.baseUrl ? checkedUrl(entry.baseUrl, `The RAKAZO_LOCAL_MODELS URL for ${id}`) : localBaseUrl(), reasoning: false, compat: { supportsDeveloperRole: false, supportsReasoningEffort: false }, input: inputModalities(localVisionModelIds().has(id)), // Runs on the operator's own hardware, so there is nothing to bill. cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: tokenLimit("RAKAZO_LOCAL_CONTEXT_WINDOW", DEFAULT_CONTEXT_WINDOW), maxTokens: tokenLimit("RAKAZO_LOCAL_MAX_TOKENS", DEFAULT_MAX_TOKENS), }; } /** The provider, or undefined when no local models are configured. */ export function localProvider(): Provider | undefined { const entries = localModelEntries(process.env); if (!entries.length) return undefined; return createProvider({ id: LOCAL_PROVIDER_ID, name: "Local (Ollama / LM Studio)", baseUrl: localBaseUrl(), auth: { apiKey: { name: "Local model server", resolve: async () => ({ auth: { apiKey: LOCAL_API_KEY, baseUrl: localBaseUrl() }, source: "local model server", }), }, }, models: entries.map(localModel), api: openAICompletionsApi(), }); } /** Register the local provider on a Models collection. No-op when unconfigured. */ export function registerLocalProvider(models: MutableModels): MutableModels { const provider = localProvider(); if (provider) models.setProvider(provider); return models; }