diff --git a/apps/api/src/ai-consent.test.ts b/apps/api/src/ai-consent.test.ts index caf5be5..38f0a02 100644 --- a/apps/api/src/ai-consent.test.ts +++ b/apps/api/src/ai-consent.test.ts @@ -29,6 +29,7 @@ function setup() { }, spaceMember: { findUnique: vi.fn(async () => ({ role: "owner" })) }, deploymentSettings: { findUnique: vi.fn(async () => null) }, + integrationProviderConfig: { findUnique: vi.fn(async () => null) }, aiDataConsent: { findMany: vi.fn(async () => []), upsert }, $transaction: vi.fn(async (queries) => Promise.all(queries)), }, @@ -134,6 +135,28 @@ describe("consent grants", () => { expect(JSON.stringify(status.recipients)).not.toContain("supermemory"); expect(deps.prisma.spaceMemoryConfig.findUnique).not.toHaveBeenCalled(); }); + it("discloses the deployment's media servers before model use, keyed by server", async () => { + const { deps, actor } = setup(); + const settings = { + comfyui: { baseUrl: "http://127.0.0.1:8188", apiKey: "comfy-key", imageModel: "sdxl" }, + transcription: { baseUrl: "http://127.0.0.1:8080/v1", model: "whisper-1" }, + }; + vi.mocked(deps.prisma.integrationProviderConfig.findUnique).mockResolvedValue({ + id: "media", + ciphertext: "fake-media-ciphertext", + } as never); + deps.secrets = { + load: vi.fn(() => JSON.stringify(settings)), + } as unknown as RouterDeps["secrets"]; + const status = await aiConsentStatus(deps, actor, { botId: "bot", uses: ["model"] }); + expect(status.recipients.map((recipient) => recipient.name)).toEqual([ + "ComfyUI (http://127.0.0.1:8188)", + "Transcription (http://127.0.0.1:8080)", + ]); + expect(status.recipients[0]?.detail).toBe("Model: sdxl."); + expect(JSON.stringify(status)).not.toContain("comfy-key"); + expect((await aiConsentStatus(deps, actor, { uses: ["voice"] })).recipients).toHaveLength(1); + }); it("returns an operator policy without requiring a hosted provider", async () => { const { deps, actor } = setup(); deps.env.privacyPolicyUrl = "https://example.com/privacy"; diff --git a/apps/api/src/ai-consent.ts b/apps/api/src/ai-consent.ts index 8c595b5..076eff7 100644 --- a/apps/api/src/ai-consent.ts +++ b/apps/api/src/ai-consent.ts @@ -3,6 +3,9 @@ import { ORPCError } from "@orpc/server"; import { aiRecipient, cloudAgentsEnabled, + loadMediaSettings, + MEDIA_COMFYUI_PROVIDER_ID, + MEDIA_TRANSCRIPTION_PROVIDER_ID, parseModelSecret, selectConfiguredModel, toStringRecord, @@ -210,6 +213,30 @@ export async function aiConsentStatus( ) { add(aiRecipient({ provider: deps.cloudAgent.provider.describe().id, use: "model" })); } + if (uses.includes("model")) { + // Media tools and card actions send prompts, images, video and audio to the owner's servers. + const media = await loadMediaSettings(deps); + if (media.comfyui) { + add( + aiRecipient({ + provider: MEDIA_COMFYUI_PROVIDER_ID, + use: "model", + baseUrl: media.comfyui.baseUrl, + modelId: media.comfyui.imageModel, + }), + ); + } + if (media.transcription) { + add( + aiRecipient({ + provider: MEDIA_TRANSCRIPTION_PROVIDER_ID, + use: "model", + baseUrl: media.transcription.baseUrl, + modelId: media.transcription.model, + }), + ); + } + } return { scope: createHash("sha256") .update(JSON.stringify([actor.userId, actor.spaceId])) diff --git a/packages/adapters/src/ai-consent.test.ts b/packages/adapters/src/ai-consent.test.ts index bb9bb7e..eedb1bb 100644 --- a/packages/adapters/src/ai-consent.test.ts +++ b/packages/adapters/src/ai-consent.test.ts @@ -41,6 +41,18 @@ describe("mobile AI recipient disclosure", () => { expect(recipient.name).toBe("Embeddings (http://192.168.1.20:8080)"); expect(recipient.privacyUrl).toBeUndefined(); }); + it("names the media servers that media jobs send files to", () => { + expect( + aiRecipient({ provider: "comfyui", use: "model", baseUrl: "http://127.0.0.1:8188" })?.name, + ).toBe("ComfyUI (http://127.0.0.1:8188)"); + expect( + aiRecipient({ + provider: "media-transcription", + use: "model", + baseUrl: "http://127.0.0.1:8080/v1", + })?.name, + ).toBe("Transcription (http://127.0.0.1:8080)"); + }); it("supports local models without a hosted dependency", () => { const fetch = vi.fn(); vi.stubGlobal("fetch", fetch); diff --git a/packages/adapters/src/ai-consent.ts b/packages/adapters/src/ai-consent.ts index 36a7981..d633f84 100644 --- a/packages/adapters/src/ai-consent.ts +++ b/packages/adapters/src/ai-consent.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import type { AiDataUse, AiRecipient } from "@rakazo/contracts"; import { BUILTIN_MEMORY_PROVIDER_ID } from "./builtin-memory-provider.js"; +import { MEDIA_COMFYUI_PROVIDER_ID, MEDIA_TRANSCRIPTION_PROVIDER_ID } from "./media-settings.js"; import { localBaseUrl } from "./pi-local-provider.js"; import { listPiCatalog } from "./pi-models.js"; import { voiceCatalogEntry } from "./voice-factory.js"; @@ -59,5 +60,7 @@ function recipientName(provider: string, use: AiDataUse) { return provider; } if (provider === "cursor") return "Cursor"; + if (provider === MEDIA_COMFYUI_PROVIDER_ID) return "ComfyUI"; + if (provider === MEDIA_TRANSCRIPTION_PROVIDER_ID) return "Transcription"; return listPiCatalog().find((entry) => entry.provider === provider)?.providerName ?? provider; } diff --git a/packages/adapters/src/media-settings.test.ts b/packages/adapters/src/media-settings.test.ts new file mode 100644 index 0000000..2040aa9 --- /dev/null +++ b/packages/adapters/src/media-settings.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it, vi } from "vitest"; +import { + loadMediaSettings, + MEDIA_SETTINGS_ID, + MediaSettingsInvalidError, + mediaSettingsView, + normalizeComfyUiBaseUrl, + normalizeTranscriptionBaseUrl, + saveMediaSettings, +} from "./media-settings.js"; +import { EncryptedSecretStore } from "./secrets.js"; + +function store() { + const rows = new Map(); + const prisma = { + integrationProviderConfig: { + findUnique: vi.fn(async ({ where }: { where: { id: string } }) => rows.get(where.id) ?? null), + upsert: vi.fn( + async ({ where, create }: { where: { id: string }; create: { ciphertext: string } }) => { + const row = { id: where.id, ciphertext: create.ciphertext }; + rows.set(where.id, row); + return row; + }, + ), + deleteMany: vi.fn(async ({ where }: { where: { id: string } }) => { + rows.delete(where.id); + return { count: 1 }; + }), + }, + }; + const secrets = new EncryptedSecretStore("test-encryption-key-for-media-settings"); + return { deps: { prisma: prisma as never, secrets }, rows, prisma, secrets }; +} + +describe("media settings", () => { + it("normalizes server URLs and refuses ones that are not allowed", () => { + expect(normalizeComfyUiBaseUrl(" http://127.0.0.1:8188/ ")).toBe("http://127.0.0.1:8188"); + expect(normalizeComfyUiBaseUrl("http://192.168.1.20:8188/comfy/?x=1#y")).toBe( + "http://192.168.1.20:8188/comfy", + ); + expect(normalizeTranscriptionBaseUrl("http://127.0.0.1:8080")).toBe("http://127.0.0.1:8080/v1"); + expect(normalizeTranscriptionBaseUrl("http://127.0.0.1:8080/v1/")).toBe( + "http://127.0.0.1:8080/v1", + ); + for (const url of ["ftp://127.0.0.1", "http://169.254.169.254", "not a url"]) { + expect(() => normalizeComfyUiBaseUrl(url), url).toThrow(); + } + expect(() => normalizeComfyUiBaseUrl("http://user:pass@127.0.0.1:8188")).toThrow("credentials"); + }); + + it("stores the settings encrypted and returns a view without keys", async () => { + const { deps, rows } = store(); + const view = await saveMediaSettings(deps, { + comfyui: { baseUrl: "http://127.0.0.1:8188", apiKey: "comfy-key" }, + transcription: null, + }); + expect(view).toEqual({ + comfyui: { baseUrl: "http://127.0.0.1:8188", hasApiKey: true }, + transcription: null, + }); + expect(rows.get(MEDIA_SETTINGS_ID)?.ciphertext).not.toContain("comfy-key"); + expect(await loadMediaSettings(deps)).toEqual({ + comfyui: { baseUrl: "http://127.0.0.1:8188", apiKey: "comfy-key" }, + transcription: null, + }); + expect( + mediaSettingsView({ + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:8080/v1", apiKey: "k", model: "m" }, + }), + ).toEqual({ + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:8080/v1", model: "m", hasApiKey: true }, + }); + }); + + it("decrypts once per saved version", async () => { + const { deps, secrets } = store(); + await saveMediaSettings(deps, { + comfyui: { baseUrl: "http://127.0.0.1:8188" }, + transcription: null, + }); + const load = vi.spyOn(secrets, "load"); + await loadMediaSettings(deps); + await loadMediaSettings(deps); + expect(load).not.toHaveBeenCalled(); + }); + + it("keeps a saved key only for the same server, removes it when cleared", async () => { + const { deps } = store(); + await saveMediaSettings(deps, { + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:8080", apiKey: "t-key" }, + }); + await saveMediaSettings(deps, { + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:8080/v1", model: "whisper-1" }, + }); + expect((await loadMediaSettings(deps)).transcription?.apiKey).toBe("t-key"); + await saveMediaSettings(deps, { + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:9090" }, + }); + expect((await loadMediaSettings(deps)).transcription?.apiKey).toBeUndefined(); + await saveMediaSettings(deps, { + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:9090", apiKey: "new" }, + }); + await saveMediaSettings(deps, { + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:9090", apiKey: "" }, + }); + expect((await loadMediaSettings(deps)).transcription?.apiKey).toBeUndefined(); + }); + + it("refuses invalid settings with a message for the owner and removes empty ones", async () => { + const { deps, rows } = store(); + await expect( + saveMediaSettings(deps, { + comfyui: { baseUrl: "http://169.254.169.254" }, + transcription: null, + }), + ).rejects.toBeInstanceOf(MediaSettingsInvalidError); + expect(rows.size).toBe(0); + await saveMediaSettings(deps, { + comfyui: { baseUrl: "http://127.0.0.1:8188" }, + transcription: null, + }); + expect(await saveMediaSettings(deps, { comfyui: null, transcription: null })).toEqual({ + comfyui: null, + transcription: null, + }); + expect(rows.size).toBe(0); + expect(await loadMediaSettings(deps)).toEqual({ comfyui: null, transcription: null }); + }); +}); diff --git a/packages/adapters/src/media-settings.ts b/packages/adapters/src/media-settings.ts index 325c45c..8f1078b 100644 --- a/packages/adapters/src/media-settings.ts +++ b/packages/adapters/src/media-settings.ts @@ -13,6 +13,10 @@ import { } from "./openai-compatible-url.js"; import type { EncryptedSecretStore } from "./secrets.js"; +/** Recipient ids for AI data-sharing disclosures of the media servers. */ +export const MEDIA_COMFYUI_PROVIDER_ID = "comfyui"; +export const MEDIA_TRANSCRIPTION_PROVIDER_ID = "media-transcription"; + /** The IntegrationProviderConfig row that stores the media backends, encrypted as one record. */ export const MEDIA_SETTINGS_ID = "media"; const RECORD_ID = `integration-provider:${MEDIA_SETTINGS_ID}`; diff --git a/packages/core/src/ai-consent.test.ts b/packages/core/src/ai-consent.test.ts index 18827fe..2359d66 100644 --- a/packages/core/src/ai-consent.test.ts +++ b/packages/core/src/ai-consent.test.ts @@ -86,6 +86,7 @@ describe("foreground AI consent", () => { expect(aiDataUsesForProcedure("threads/get")).toEqual([]); expect(aiDataUsesForProcedure("threads/send")).toEqual(["model", "memory"]); expect(aiDataUsesForProcedure("voice/prepare")).toEqual(["voice"]); + expect(aiDataUsesForProcedure("media/jobs/start")).toEqual(["model"]); expect( aiDataUsesForProcedure("routines/update", { routineId: "routine", active: false }), ).toEqual([]); diff --git a/packages/core/src/ai-consent.ts b/packages/core/src/ai-consent.ts index 5adca81..0223804 100644 --- a/packages/core/src/ai-consent.ts +++ b/packages/core/src/ai-consent.ts @@ -32,6 +32,7 @@ export function aiDataUsesForProcedure(procedure: string, input?: unknown): AiDa ].includes(path) ) return ["model", "memory"]; + if (path === "media/jobs/start") return ["model"]; if (["voice/prepare", "voice/speak", "voice/transcribe"].includes(path)) return ["voice"]; return []; }