From 533f82d856d68ab658fd012b8cbe8e8238570158 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 13:51:13 +0000 Subject: [PATCH] feat(media): disclose media servers in AI data sharing Media tools and card actions send prompts, images, video and audio to the deployment owner's ComfyUI and transcription servers. Like the embeddings server behind built-in memory, those self-hosted endpoints are recipients in the AI data-sharing disclosure: they appear with the model uses ("ComfyUI ()", "Transcription ()", keyed by the full endpoint so a new server asks again), and starting a card job asks for model consent on clients that run the foreground check. Also adds unit tests for the media settings store: URL normalization and refusals, encryption, the key-free view, one decryption per saved version, key retention per server and removing empty settings. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- apps/api/src/ai-consent.test.ts | 23 ++++ apps/api/src/ai-consent.ts | 27 ++++ packages/adapters/src/ai-consent.test.ts | 12 ++ packages/adapters/src/ai-consent.ts | 3 + packages/adapters/src/media-settings.test.ts | 136 +++++++++++++++++++ packages/adapters/src/media-settings.ts | 4 + packages/core/src/ai-consent.test.ts | 1 + packages/core/src/ai-consent.ts | 1 + 8 files changed, 207 insertions(+) create mode 100644 packages/adapters/src/media-settings.test.ts diff --git a/apps/api/src/ai-consent.test.ts b/apps/api/src/ai-consent.test.ts index caf5be5..38f0a02 100644 --- a/apps/api/src/ai-consent.test.ts +++ b/apps/api/src/ai-consent.test.ts @@ -29,6 +29,7 @@ function setup() { }, spaceMember: { findUnique: vi.fn(async () => ({ role: "owner" })) }, deploymentSettings: { findUnique: vi.fn(async () => null) }, + integrationProviderConfig: { findUnique: vi.fn(async () => null) }, aiDataConsent: { findMany: vi.fn(async () => []), upsert }, $transaction: vi.fn(async (queries) => Promise.all(queries)), }, @@ -134,6 +135,28 @@ describe("consent grants", () => { expect(JSON.stringify(status.recipients)).not.toContain("supermemory"); expect(deps.prisma.spaceMemoryConfig.findUnique).not.toHaveBeenCalled(); }); + it("discloses the deployment's media servers before model use, keyed by server", async () => { + const { deps, actor } = setup(); + const settings = { + comfyui: { baseUrl: "http://127.0.0.1:8188", apiKey: "comfy-key", imageModel: "sdxl" }, + transcription: { baseUrl: "http://127.0.0.1:8080/v1", model: "whisper-1" }, + }; + vi.mocked(deps.prisma.integrationProviderConfig.findUnique).mockResolvedValue({ + id: "media", + ciphertext: "fake-media-ciphertext", + } as never); + deps.secrets = { + load: vi.fn(() => JSON.stringify(settings)), + } as unknown as RouterDeps["secrets"]; + const status = await aiConsentStatus(deps, actor, { botId: "bot", uses: ["model"] }); + expect(status.recipients.map((recipient) => recipient.name)).toEqual([ + "ComfyUI (http://127.0.0.1:8188)", + "Transcription (http://127.0.0.1:8080)", + ]); + expect(status.recipients[0]?.detail).toBe("Model: sdxl."); + expect(JSON.stringify(status)).not.toContain("comfy-key"); + expect((await aiConsentStatus(deps, actor, { uses: ["voice"] })).recipients).toHaveLength(1); + }); it("returns an operator policy without requiring a hosted provider", async () => { const { deps, actor } = setup(); deps.env.privacyPolicyUrl = "https://example.com/privacy"; diff --git a/apps/api/src/ai-consent.ts b/apps/api/src/ai-consent.ts index 8c595b5..076eff7 100644 --- a/apps/api/src/ai-consent.ts +++ b/apps/api/src/ai-consent.ts @@ -3,6 +3,9 @@ import { ORPCError } from "@orpc/server"; import { aiRecipient, cloudAgentsEnabled, + loadMediaSettings, + MEDIA_COMFYUI_PROVIDER_ID, + MEDIA_TRANSCRIPTION_PROVIDER_ID, parseModelSecret, selectConfiguredModel, toStringRecord, @@ -210,6 +213,30 @@ export async function aiConsentStatus( ) { add(aiRecipient({ provider: deps.cloudAgent.provider.describe().id, use: "model" })); } + if (uses.includes("model")) { + // Media tools and card actions send prompts, images, video and audio to the owner's servers. + const media = await loadMediaSettings(deps); + if (media.comfyui) { + add( + aiRecipient({ + provider: MEDIA_COMFYUI_PROVIDER_ID, + use: "model", + baseUrl: media.comfyui.baseUrl, + modelId: media.comfyui.imageModel, + }), + ); + } + if (media.transcription) { + add( + aiRecipient({ + provider: MEDIA_TRANSCRIPTION_PROVIDER_ID, + use: "model", + baseUrl: media.transcription.baseUrl, + modelId: media.transcription.model, + }), + ); + } + } return { scope: createHash("sha256") .update(JSON.stringify([actor.userId, actor.spaceId])) diff --git a/packages/adapters/src/ai-consent.test.ts b/packages/adapters/src/ai-consent.test.ts index bb9bb7e..eedb1bb 100644 --- a/packages/adapters/src/ai-consent.test.ts +++ b/packages/adapters/src/ai-consent.test.ts @@ -41,6 +41,18 @@ describe("mobile AI recipient disclosure", () => { expect(recipient.name).toBe("Embeddings (http://192.168.1.20:8080)"); expect(recipient.privacyUrl).toBeUndefined(); }); + it("names the media servers that media jobs send files to", () => { + expect( + aiRecipient({ provider: "comfyui", use: "model", baseUrl: "http://127.0.0.1:8188" })?.name, + ).toBe("ComfyUI (http://127.0.0.1:8188)"); + expect( + aiRecipient({ + provider: "media-transcription", + use: "model", + baseUrl: "http://127.0.0.1:8080/v1", + })?.name, + ).toBe("Transcription (http://127.0.0.1:8080)"); + }); it("supports local models without a hosted dependency", () => { const fetch = vi.fn(); vi.stubGlobal("fetch", fetch); diff --git a/packages/adapters/src/ai-consent.ts b/packages/adapters/src/ai-consent.ts index 36a7981..d633f84 100644 --- a/packages/adapters/src/ai-consent.ts +++ b/packages/adapters/src/ai-consent.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import type { AiDataUse, AiRecipient } from "@rakazo/contracts"; import { BUILTIN_MEMORY_PROVIDER_ID } from "./builtin-memory-provider.js"; +import { MEDIA_COMFYUI_PROVIDER_ID, MEDIA_TRANSCRIPTION_PROVIDER_ID } from "./media-settings.js"; import { localBaseUrl } from "./pi-local-provider.js"; import { listPiCatalog } from "./pi-models.js"; import { voiceCatalogEntry } from "./voice-factory.js"; @@ -59,5 +60,7 @@ function recipientName(provider: string, use: AiDataUse) { return provider; } if (provider === "cursor") return "Cursor"; + if (provider === MEDIA_COMFYUI_PROVIDER_ID) return "ComfyUI"; + if (provider === MEDIA_TRANSCRIPTION_PROVIDER_ID) return "Transcription"; return listPiCatalog().find((entry) => entry.provider === provider)?.providerName ?? provider; } diff --git a/packages/adapters/src/media-settings.test.ts b/packages/adapters/src/media-settings.test.ts new file mode 100644 index 0000000..2040aa9 --- /dev/null +++ b/packages/adapters/src/media-settings.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it, vi } from "vitest"; +import { + loadMediaSettings, + MEDIA_SETTINGS_ID, + MediaSettingsInvalidError, + mediaSettingsView, + normalizeComfyUiBaseUrl, + normalizeTranscriptionBaseUrl, + saveMediaSettings, +} from "./media-settings.js"; +import { EncryptedSecretStore } from "./secrets.js"; + +function store() { + const rows = new Map(); + const prisma = { + integrationProviderConfig: { + findUnique: vi.fn(async ({ where }: { where: { id: string } }) => rows.get(where.id) ?? null), + upsert: vi.fn( + async ({ where, create }: { where: { id: string }; create: { ciphertext: string } }) => { + const row = { id: where.id, ciphertext: create.ciphertext }; + rows.set(where.id, row); + return row; + }, + ), + deleteMany: vi.fn(async ({ where }: { where: { id: string } }) => { + rows.delete(where.id); + return { count: 1 }; + }), + }, + }; + const secrets = new EncryptedSecretStore("test-encryption-key-for-media-settings"); + return { deps: { prisma: prisma as never, secrets }, rows, prisma, secrets }; +} + +describe("media settings", () => { + it("normalizes server URLs and refuses ones that are not allowed", () => { + expect(normalizeComfyUiBaseUrl(" http://127.0.0.1:8188/ ")).toBe("http://127.0.0.1:8188"); + expect(normalizeComfyUiBaseUrl("http://192.168.1.20:8188/comfy/?x=1#y")).toBe( + "http://192.168.1.20:8188/comfy", + ); + expect(normalizeTranscriptionBaseUrl("http://127.0.0.1:8080")).toBe("http://127.0.0.1:8080/v1"); + expect(normalizeTranscriptionBaseUrl("http://127.0.0.1:8080/v1/")).toBe( + "http://127.0.0.1:8080/v1", + ); + for (const url of ["ftp://127.0.0.1", "http://169.254.169.254", "not a url"]) { + expect(() => normalizeComfyUiBaseUrl(url), url).toThrow(); + } + expect(() => normalizeComfyUiBaseUrl("http://user:pass@127.0.0.1:8188")).toThrow("credentials"); + }); + + it("stores the settings encrypted and returns a view without keys", async () => { + const { deps, rows } = store(); + const view = await saveMediaSettings(deps, { + comfyui: { baseUrl: "http://127.0.0.1:8188", apiKey: "comfy-key" }, + transcription: null, + }); + expect(view).toEqual({ + comfyui: { baseUrl: "http://127.0.0.1:8188", hasApiKey: true }, + transcription: null, + }); + expect(rows.get(MEDIA_SETTINGS_ID)?.ciphertext).not.toContain("comfy-key"); + expect(await loadMediaSettings(deps)).toEqual({ + comfyui: { baseUrl: "http://127.0.0.1:8188", apiKey: "comfy-key" }, + transcription: null, + }); + expect( + mediaSettingsView({ + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:8080/v1", apiKey: "k", model: "m" }, + }), + ).toEqual({ + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:8080/v1", model: "m", hasApiKey: true }, + }); + }); + + it("decrypts once per saved version", async () => { + const { deps, secrets } = store(); + await saveMediaSettings(deps, { + comfyui: { baseUrl: "http://127.0.0.1:8188" }, + transcription: null, + }); + const load = vi.spyOn(secrets, "load"); + await loadMediaSettings(deps); + await loadMediaSettings(deps); + expect(load).not.toHaveBeenCalled(); + }); + + it("keeps a saved key only for the same server, removes it when cleared", async () => { + const { deps } = store(); + await saveMediaSettings(deps, { + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:8080", apiKey: "t-key" }, + }); + await saveMediaSettings(deps, { + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:8080/v1", model: "whisper-1" }, + }); + expect((await loadMediaSettings(deps)).transcription?.apiKey).toBe("t-key"); + await saveMediaSettings(deps, { + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:9090" }, + }); + expect((await loadMediaSettings(deps)).transcription?.apiKey).toBeUndefined(); + await saveMediaSettings(deps, { + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:9090", apiKey: "new" }, + }); + await saveMediaSettings(deps, { + comfyui: null, + transcription: { baseUrl: "http://127.0.0.1:9090", apiKey: "" }, + }); + expect((await loadMediaSettings(deps)).transcription?.apiKey).toBeUndefined(); + }); + + it("refuses invalid settings with a message for the owner and removes empty ones", async () => { + const { deps, rows } = store(); + await expect( + saveMediaSettings(deps, { + comfyui: { baseUrl: "http://169.254.169.254" }, + transcription: null, + }), + ).rejects.toBeInstanceOf(MediaSettingsInvalidError); + expect(rows.size).toBe(0); + await saveMediaSettings(deps, { + comfyui: { baseUrl: "http://127.0.0.1:8188" }, + transcription: null, + }); + expect(await saveMediaSettings(deps, { comfyui: null, transcription: null })).toEqual({ + comfyui: null, + transcription: null, + }); + expect(rows.size).toBe(0); + expect(await loadMediaSettings(deps)).toEqual({ comfyui: null, transcription: null }); + }); +}); diff --git a/packages/adapters/src/media-settings.ts b/packages/adapters/src/media-settings.ts index 325c45c..8f1078b 100644 --- a/packages/adapters/src/media-settings.ts +++ b/packages/adapters/src/media-settings.ts @@ -13,6 +13,10 @@ import { } from "./openai-compatible-url.js"; import type { EncryptedSecretStore } from "./secrets.js"; +/** Recipient ids for AI data-sharing disclosures of the media servers. */ +export const MEDIA_COMFYUI_PROVIDER_ID = "comfyui"; +export const MEDIA_TRANSCRIPTION_PROVIDER_ID = "media-transcription"; + /** The IntegrationProviderConfig row that stores the media backends, encrypted as one record. */ export const MEDIA_SETTINGS_ID = "media"; const RECORD_ID = `integration-provider:${MEDIA_SETTINGS_ID}`; diff --git a/packages/core/src/ai-consent.test.ts b/packages/core/src/ai-consent.test.ts index 18827fe..2359d66 100644 --- a/packages/core/src/ai-consent.test.ts +++ b/packages/core/src/ai-consent.test.ts @@ -86,6 +86,7 @@ describe("foreground AI consent", () => { expect(aiDataUsesForProcedure("threads/get")).toEqual([]); expect(aiDataUsesForProcedure("threads/send")).toEqual(["model", "memory"]); expect(aiDataUsesForProcedure("voice/prepare")).toEqual(["voice"]); + expect(aiDataUsesForProcedure("media/jobs/start")).toEqual(["model"]); expect( aiDataUsesForProcedure("routines/update", { routineId: "routine", active: false }), ).toEqual([]); diff --git a/packages/core/src/ai-consent.ts b/packages/core/src/ai-consent.ts index 5adca81..0223804 100644 --- a/packages/core/src/ai-consent.ts +++ b/packages/core/src/ai-consent.ts @@ -32,6 +32,7 @@ export function aiDataUsesForProcedure(procedure: string, input?: unknown): AiDa ].includes(path) ) return ["model", "memory"]; + if (path === "media/jobs/start") return ["model"]; if (["voice/prepare", "voice/speak", "voice/transcribe"].includes(path)) return ["voice"]; return []; }