From 044669edfc44772b1003c3166d192204f4b04bba Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 08:45:30 +0000 Subject: [PATCH 1/2] Add a media library: owner folders that bots can search and import from The deployment owner adds folders of photos and videos in the media settings. A per-folder background job indexes them (relative path, kind, size, capture date, dimensions, duration), re-indexes every six hours, and drops files that are gone. The owner's own runs get two tools: search_media (words in folder and file names, kind, capture date range, newest first) and import_media (copies one file into the conversation as an artifact). Files are never changed; hidden entries and symlinks are skipped, and an import must resolve inside its folder. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Wy1md9QE1ayJbDCwJ4vAQs --- apps/api/src/media-library.ts | 82 +++++ apps/api/src/router.ts | 20 + apps/web/e2e/media-settings.spec.ts | 112 ++++++ .../src/components/MediaLibraryFolders.tsx | 162 +++++++++ apps/web/src/components/MediaSettings.tsx | 6 +- apps/web/src/locales/de/messages.po | 28 ++ apps/web/src/locales/en/messages.po | 28 ++ apps/web/src/locales/es/messages.po | 28 ++ apps/web/src/locales/hi/messages.po | 28 ++ apps/web/src/locales/ko/messages.po | 28 ++ apps/web/src/locales/pt-BR/messages.po | 28 ++ apps/web/src/locales/ru/messages.po | 28 ++ apps/web/src/locales/tr/messages.po | 28 ++ apps/web/src/locales/zh-CN/messages.po | 28 ++ docs/self-host.md | 20 + .../adapter-kit/src/background-jobs.test.ts | 22 ++ packages/adapter-kit/src/background-jobs.ts | 18 + packages/adapter-kit/src/types.ts | 2 + .../adapters/src/background-job-handlers.ts | 4 + packages/adapters/src/builtin-tools.ts | 25 ++ packages/adapters/src/executor-tools.test.ts | 24 +- packages/adapters/src/executor-tools.ts | 10 + packages/adapters/src/executor.ts | 45 +++ packages/adapters/src/index.ts | 1 + packages/adapters/src/media-library.test.ts | 63 ++++ packages/adapters/src/media-library.ts | 343 ++++++++++++++++++ packages/adapters/src/media/ffmpeg.ts | 11 +- packages/adapters/src/pi-runtime.ts | 2 + .../adapters/src/wakeup-graphile-host.test.ts | 1 + packages/adapters/src/wakeup.postgres.test.ts | 1 + packages/adapters/src/wakeup.test.ts | 1 + packages/contracts/src/index.ts | 1 + packages/contracts/src/media-library.ts | 39 ++ packages/contracts/src/rpc.ts | 8 + .../migration.sql | 52 +++ packages/db/prisma/schema.prisma | 48 +++ packages/testkit/src/cli/harness.ts | 1 + .../src/media-library.postgres.test.ts | 204 +++++++++++ 38 files changed, 1576 insertions(+), 4 deletions(-) create mode 100644 apps/api/src/media-library.ts create mode 100644 apps/web/src/components/MediaLibraryFolders.tsx create mode 100644 packages/adapters/src/media-library.test.ts create mode 100644 packages/adapters/src/media-library.ts create mode 100644 packages/contracts/src/media-library.ts create mode 100644 packages/db/prisma/migrations/20261005090000_media_library/migration.sql create mode 100644 packages/testkit/src/media-library.postgres.test.ts diff --git a/apps/api/src/media-library.ts b/apps/api/src/media-library.ts new file mode 100644 index 0000000..3d3c9b6 --- /dev/null +++ b/apps/api/src/media-library.ts @@ -0,0 +1,82 @@ +import path from "node:path"; +import { ORPCError } from "@orpc/server"; +import type { JobPublisher } from "@rakazo/adapter-kit"; +import { mediaLibrarySyncJob } from "@rakazo/adapter-kit"; +import { checkMediaLibraryFolder, mediaLibrarySourceView } from "@rakazo/adapters"; +import type { Actor, MediaLibrarySourceAddInput } from "@rakazo/contracts"; +import type { PrismaClient } from "@rakazo/db"; + +export interface MediaLibraryRouteDeps { + prisma: PrismaClient; + jobs: JobPublisher; +} + +/** Library folders are paths on the server, so only its owner adds them, like host folders. */ +function requireDeploymentOwner(actor: Actor) { + if (!actor.isDeploymentOwner) throw new ORPCError("FORBIDDEN"); +} + +async function ownSource(deps: MediaLibraryRouteDeps, actor: Actor, sourceId: string) { + const source = await deps.prisma.mediaLibrarySource.findFirst({ + where: { id: sourceId, userId: actor.userId }, + }); + if (!source) throw new ORPCError("NOT_FOUND"); + return source; +} + +export async function listMediaLibrarySources(deps: MediaLibraryRouteDeps, actor: Actor) { + requireDeploymentOwner(actor); + const rows = await deps.prisma.mediaLibrarySource.findMany({ + where: { userId: actor.userId }, + orderBy: { createdAt: "asc" }, + }); + return rows.map(mediaLibrarySourceView); +} + +export async function addMediaLibrarySource( + deps: MediaLibraryRouteDeps, + actor: Actor, + input: MediaLibrarySourceAddInput, +) { + requireDeploymentOwner(actor); + const problem = await checkMediaLibraryFolder(input.path); + if (problem) throw new ORPCError("BAD_REQUEST", { message: problem }); + const folder = path.resolve(input.path); + const existing = await deps.prisma.mediaLibrarySource.findUnique({ + where: { userId_path: { userId: actor.userId, path: folder } }, + }); + if (existing) return mediaLibrarySourceView(existing); + const source = await deps.prisma.mediaLibrarySource.create({ + data: { userId: actor.userId, path: folder, status: "syncing" }, + }); + await deps.jobs.enqueue(mediaLibrarySyncJob(source.id)); + return mediaLibrarySourceView(source); +} + +export async function removeMediaLibrarySource( + deps: MediaLibraryRouteDeps, + actor: Actor, + sourceId: string, +) { + requireDeploymentOwner(actor); + await ownSource(deps, actor, sourceId); + await deps.prisma.mediaLibrarySource.deleteMany({ + where: { id: sourceId, userId: actor.userId }, + }); + return { ok: true as const }; +} + +export async function syncMediaLibrarySourceNow( + deps: MediaLibraryRouteDeps, + actor: Actor, + sourceId: string, +) { + requireDeploymentOwner(actor); + await ownSource(deps, actor, sourceId); + const source = await deps.prisma.mediaLibrarySource.update({ + where: { id: sourceId }, + data: { status: "syncing", error: null }, + }); + await deps.jobs.enqueue(mediaLibrarySyncJob(sourceId)); + return mediaLibrarySourceView(source); +} diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index 6fc50df..0c19853 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -205,6 +205,12 @@ import { startMediaJob, updateMediaSettings, } from "./media-jobs.js"; +import { + addMediaLibrarySource, + listMediaLibrarySources, + removeMediaLibrarySource, + syncMediaLibrarySourceNow, +} from "./media-library.js"; import { disconnectMemoryProvider, getMemoryProviderConfig, @@ -3797,6 +3803,20 @@ export function createRouter(deps: RouterDeps) { cancelMediaJob(deps, context.actor, input.jobId), ), }, + library: { + list: authed.media.library.list.handler(({ context }) => + listMediaLibrarySources(deps, context.actor), + ), + add: authed.media.library.add.handler(({ context, input }) => + addMediaLibrarySource(deps, context.actor, input), + ), + remove: authed.media.library.remove.handler(({ context, input }) => + removeMediaLibrarySource(deps, context.actor, input.sourceId), + ), + sync: authed.media.library.sync.handler(({ context, input }) => + syncMediaLibrarySourceNow(deps, context.actor, input.sourceId), + ), + }, }, connections: { catalog: authed.connections.catalog.handler(async ({ context, input }) => { diff --git a/apps/web/e2e/media-settings.spec.ts b/apps/web/e2e/media-settings.spec.ts index bf606b7..b579236 100644 --- a/apps/web/e2e/media-settings.spec.ts +++ b/apps/web/e2e/media-settings.spec.ts @@ -37,6 +37,68 @@ function view(draft: Draft, previous: { comfyui: boolean; transcription: boolean }; } +type Source = { + id: string; + path: string; + status: "idle" | "syncing" | "failed"; + error: string | null; + assetCount: number; + lastSyncedAt: string | null; +}; + +/** Library folders on the fake server: a new folder indexes on its next list. */ +async function fakeLibrary(page: Page, initial: Source[] = []) { + let sources = initial; + const calls: { proc: string; input: unknown }[] = []; + await page.route("**/rpc/media/library/**", (route) => { + const proc = new URL(route.request().url()).pathname.split("/").at(-1) ?? ""; + const input = (route.request().postDataJSON() as { json?: unknown } | null)?.json; + calls.push({ proc, input }); + if (proc === "list") { + const listed = sources; + sources = sources.map((source) => + source.status === "syncing" + ? { ...source, status: "idle", assetCount: 1284, lastSyncedAt: new Date().toISOString() } + : source, + ); + return route.fulfill({ json: { json: listed } }); + } + if (proc === "add") { + const { path } = input as { path: string }; + if (!path.startsWith("/")) { + return route.fulfill({ + status: 400, + json: { + json: { + defined: false, + code: "BAD_REQUEST", + status: 400, + message: "Use an absolute folder path.", + }, + }, + }); + } + const source: Source = { + id: `source-${sources.length + 1}`, + path, + status: "syncing", + error: null, + assetCount: 0, + lastSyncedAt: null, + }; + sources = [...sources, source]; + return route.fulfill({ json: { json: source } }); + } + if (proc === "remove") { + const { sourceId } = input as { sourceId: string }; + sources = sources.filter((source) => source.id !== sourceId); + return route.fulfill({ json: { json: { ok: true } } }); + } + return route.fallback(); + }); + return calls; +} + async function asDeploymentOwner(page: Page, isDeploymentOwner: boolean) { await page.route("**/rpc/bootstrap", async (route) => { const response = await route.fetch(); @@ -50,6 +112,7 @@ async function asDeploymentOwner(page: Page, isDeploymentOwner: boolean) { test("the deployment owner points media at their own servers", async ({ page }, testInfo) => { await asDeploymentOwner(page, true); + await fakeLibrary(page); let current: ReturnType = { comfyui: null, transcription: null }; const saved: Draft[] = []; await page.route("**/rpc/media/settings/get", (route) => @@ -113,6 +176,55 @@ test("the deployment owner points media at their own servers", async ({ page }, await expect(comfyKey).toHaveAttribute("placeholder", "Optional"); }); +test("the deployment owner adds a photo folder to the library", async ({ page }, testInfo) => { + await asDeploymentOwner(page, true); + const calls = await fakeLibrary(page, [ + { + id: "source-0", + path: "/srv/media/Videos", + status: "failed", + error: "Folder not found.", + assetCount: 0, + lastSyncedAt: null, + }, + ]); + await page.route("**/rpc/media/settings/get", (route) => + route.fulfill({ json: { json: { comfyui: null, transcription: null } } }), + ); + + await signup(page, `media-library-${Date.now()}@rakazo.test`, "password12", "Library Owner"); + await completeOnboarding(page); + const settings = await openUserSettings(page); + await settings.getByTestId("settings-nav-media").click(); + const library = settings.getByTestId("media-library"); + await expect(library.getByText("Folder not found.")).toBeVisible(); + + const folder = library.getByLabel("Folder", { exact: true }); + await folder.fill("photos"); + await library.getByRole("button", { name: "Add", exact: true }).click(); + await expect(library.getByRole("alert")).toHaveText("Use an absolute folder path."); + + await folder.fill("/srv/media/Photos"); + await library.getByRole("button", { name: "Add", exact: true }).click(); + await expect(folder).toHaveValue(""); + await expect(library.getByText("/srv/media/Photos")).toBeVisible(); + // The new folder indexes in the background; the list catches up on its own. + await expect(library.getByText("1,284 files")).toBeVisible({ timeout: 10_000 }); + await captureScreenshot(page, testInfo, "media-library"); + + await library + .getByRole("listitem") + .filter({ hasText: "/srv/media/Videos" }) + .getByRole("button", { name: "Remove folder" }) + .click(); + await expect(library.getByText("/srv/media/Videos")).toHaveCount(0); + expect(calls.filter((call) => call.proc !== "list")).toEqual([ + { proc: "add", input: { path: "photos" } }, + { proc: "add", input: { path: "/srv/media/Photos" } }, + { proc: "remove", input: { sourceId: "source-0" } }, + ]); +}); + test("members who do not own the deployment have no media settings", async ({ page }) => { await asDeploymentOwner(page, false); await signup(page, `media-member-${Date.now()}@rakazo.test`, "password12", "Media Member"); diff --git a/apps/web/src/components/MediaLibraryFolders.tsx b/apps/web/src/components/MediaLibraryFolders.tsx new file mode 100644 index 0000000..7878daf --- /dev/null +++ b/apps/web/src/components/MediaLibraryFolders.tsx @@ -0,0 +1,162 @@ +import { Plural, Trans, useLingui } from "@lingui/react/macro"; +import type { MediaLibrarySource } from "@rakazo/contracts"; +import { Button, Input } from "@rakazo/ui-web"; +import { RefreshCw, X } from "lucide-react"; +import { useEffect, useId, useState } from "react"; +import { rpc } from "../lib/rpc"; +import { Shimmer } from "./ai/primitives"; + +const POLL_MS = 2000; + +/** Folders on the server whose photos and videos the owner's bots can search and use. */ +export function MediaLibraryFolders() { + const { t } = useLingui(); + const id = useId(); + const [sources, setSources] = useState(null); + const [folder, setFolder] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + rpc.media.library + .list() + .then((next) => { + if (!cancelled) setSources(next); + }) + .catch((err: unknown) => { + if (!cancelled) setError(err instanceof Error ? err.message : t`Couldn't load`); + }); + return () => { + cancelled = true; + }; + }, [t]); + + const syncing = sources?.some((source) => source.status === "syncing") ?? false; + useEffect(() => { + if (!syncing) return; + const timer = setInterval(() => { + rpc.media.library + .list() + .then(setSources) + .catch(() => undefined); + }, POLL_MS); + return () => clearInterval(timer); + }, [syncing]); + + async function act(action: () => Promise) { + if (busy) return; + setBusy(true); + setError(null); + try { + await action(); + } catch (err) { + setError(err instanceof Error ? err.message : t`Couldn't save`); + } finally { + setBusy(false); + } + } + + function replace(next: MediaLibrarySource) { + setSources((current) => [...(current ?? []).filter((source) => source.id !== next.id), next]); + } + + const disabled = busy || sources === null; + + return ( +
+

+ Library +

+ {sources?.length ? ( +
    + {sources.map((source) => ( +
  • +
    +

    {source.path}

    + {source.status === "syncing" ? ( +

    + + Indexing… + +

    + ) : source.status === "failed" ? ( +

    {source.error}

    + ) : ( +

    + +

    + )} +
    + + +
    • + ))} +
+ ) : null} +
{ + event.preventDefault(); + const path = folder.trim(); + if (!path) return; + void act(async () => { + replace(await rpc.media.library.add({ path })); + setFolder(""); + }); + }} + > + setFolder(event.target.value)} + /> + +
+ {error ? ( +

+ {error} +

+ ) : null} +
+ ); +} diff --git a/apps/web/src/components/MediaSettings.tsx b/apps/web/src/components/MediaSettings.tsx index 2ff24d2..0e667e1 100644 --- a/apps/web/src/components/MediaSettings.tsx +++ b/apps/web/src/components/MediaSettings.tsx @@ -4,6 +4,7 @@ import { Button, Field, FieldLabel, Input, Textarea } from "@rakazo/ui-web"; import { useEffect, useId, useRef, useState } from "react"; import { rpc } from "../lib/rpc"; import { SuccessPop } from "./ai/primitives"; +import { MediaLibraryFolders } from "./MediaLibraryFolders"; type KeyDraft = { value: string; remove: boolean }; @@ -67,7 +68,9 @@ function ApiKeyField({ ); } -/** The deployment owner's media servers: ComfyUI for generation, a transcription server. */ +/** + * The deployment owner's media: library folders, ComfyUI for generation, a transcription server. + */ export function MediaSettings() { const { t } = useLingui(); const ids = useId(); @@ -152,6 +155,7 @@ export function MediaSettings() { return (
+

ComfyUI

diff --git a/apps/web/src/locales/de/messages.po b/apps/web/src/locales/de/messages.po index 3a4c2c7..b497e5b 100644 --- a/apps/web/src/locales/de/messages.po +++ b/apps/web/src/locales/de/messages.po @@ -27,6 +27,11 @@ msgstr "(optional)" msgid "{0, plural, one {# day} other {# days}}" msgstr "{0, plural, one {# Tag} other {# Tage}}" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "{0, plural, one {# Datei} other {# Dateien}}" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "Aktives Modell" msgid "Activity" msgstr "Aktivität" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "Stimme konnte nicht aktualisiert werden" msgid "Could not upload {name}. Try again." msgstr "{name} konnte nicht hochgeladen werden. Versuche es erneut." +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "Konnte nicht geladen werden" msgid "Couldn't load messaging settings" msgstr "Messaging-Einstellungen konnten nicht geladen werden" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "Schwung" msgid "Flower" msgstr "Blume" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "Ordner" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "Schriften lokal geladen" @@ -2813,6 +2825,14 @@ msgstr "In Arbeit" msgid "Inbox & email" msgstr "Posteingang & E-Mail" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "Neu einlesen" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "Wird eingelesen…" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "Standard übernehmen" @@ -2919,6 +2939,10 @@ msgstr "Später" msgid "Leave" msgstr "Verlassen" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "Mediathek" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "Privatkanal — ich bin da." @@ -4145,6 +4169,10 @@ msgstr "Entfernen" msgid "Remove {0}" msgstr "{0} entfernen" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "Ordner entfernen" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "Git-Ereignis entfernen" diff --git a/apps/web/src/locales/en/messages.po b/apps/web/src/locales/en/messages.po index 85a0267..b1d5bef 100644 --- a/apps/web/src/locales/en/messages.po +++ b/apps/web/src/locales/en/messages.po @@ -27,6 +27,11 @@ msgstr "(optional)" msgid "{0, plural, one {# day} other {# days}}" msgstr "{0, plural, one {# day} other {# days}}" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "{0, plural, one {# file} other {# files}}" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "Active model" msgid "Activity" msgstr "Activity" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "Could not update voice" msgid "Could not upload {name}. Try again." msgstr "Could not upload {name}. Try again." +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "Couldn't load" msgid "Couldn't load messaging settings" msgstr "Couldn't load messaging settings" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "Flick" msgid "Flower" msgstr "Flower" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "Folder" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "Fonts loaded locally" @@ -2813,6 +2825,14 @@ msgstr "In progress" msgid "Inbox & email" msgstr "Inbox & email" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "Index again" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "Indexing…" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "Inherit default" @@ -2919,6 +2939,10 @@ msgstr "Later" msgid "Leave" msgstr "Leave" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "Library" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "Life channel — I'm here." @@ -4145,6 +4169,10 @@ msgstr "Remove" msgid "Remove {0}" msgstr "Remove {0}" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "Remove folder" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "Remove Git event" diff --git a/apps/web/src/locales/es/messages.po b/apps/web/src/locales/es/messages.po index 214143b..69cc77e 100644 --- a/apps/web/src/locales/es/messages.po +++ b/apps/web/src/locales/es/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "Modelo activo" msgid "Activity" msgstr "Actividad" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "No se pudo cargar la configuración de mensajería" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "Heredar predeterminado" @@ -2919,6 +2939,10 @@ msgstr "" msgid "Leave" msgstr "Salir" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "Quitar" msgid "Remove {0}" msgstr "Quitar {0}" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "" diff --git a/apps/web/src/locales/hi/messages.po b/apps/web/src/locales/hi/messages.po index 51e1b09..e5e0917 100644 --- a/apps/web/src/locales/hi/messages.po +++ b/apps/web/src/locales/hi/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "सक्रिय मॉडल" msgid "Activity" msgstr "गतिविधि" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "डिफ़ॉल्ट का पालन करें" @@ -2919,6 +2939,10 @@ msgstr "" msgid "Leave" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "हटाएं" msgid "Remove {0}" msgstr "{0} हटाएं" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "" diff --git a/apps/web/src/locales/ko/messages.po b/apps/web/src/locales/ko/messages.po index 478118c..5d80cb6 100644 --- a/apps/web/src/locales/ko/messages.po +++ b/apps/web/src/locales/ko/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "현재 모델" msgid "Activity" msgstr "활동" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "메시징 설정을 불러올 수 없습니다" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "기본 설정 사용" @@ -2919,6 +2939,10 @@ msgstr "" msgid "Leave" msgstr "나가기" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "삭제" msgid "Remove {0}" msgstr "{0} 삭제" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "" diff --git a/apps/web/src/locales/pt-BR/messages.po b/apps/web/src/locales/pt-BR/messages.po index 0b3193e..48cf91b 100644 --- a/apps/web/src/locales/pt-BR/messages.po +++ b/apps/web/src/locales/pt-BR/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "Modelo ativo" msgid "Activity" msgstr "Atividade" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "Herdar (padrão)" @@ -2919,6 +2939,10 @@ msgstr "" msgid "Leave" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "Remover" msgid "Remove {0}" msgstr "Remover {0}" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "" diff --git a/apps/web/src/locales/ru/messages.po b/apps/web/src/locales/ru/messages.po index 026f751..426b602 100644 --- a/apps/web/src/locales/ru/messages.po +++ b/apps/web/src/locales/ru/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "Активная модель" msgid "Activity" msgstr "Активность" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "Не удалось загрузить настройки обмена сообщениями." +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "Наследовать значение по умолчанию" @@ -2919,6 +2939,10 @@ msgstr "Позже" msgid "Leave" msgstr "Выйти" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "Удалить" msgid "Remove {0}" msgstr "Удалить {0}" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "Удалить событие Git" diff --git a/apps/web/src/locales/tr/messages.po b/apps/web/src/locales/tr/messages.po index 9b782c4..bbc9266 100644 --- a/apps/web/src/locales/tr/messages.po +++ b/apps/web/src/locales/tr/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "Etkin model" msgid "Activity" msgstr "Etkinlik" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "Varsayılanı devral" @@ -2919,6 +2939,10 @@ msgstr "" msgid "Leave" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "Kaldır" msgid "Remove {0}" msgstr "{0} kaldır" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "" diff --git a/apps/web/src/locales/zh-CN/messages.po b/apps/web/src/locales/zh-CN/messages.po index 4932ef1..4d1f650 100644 --- a/apps/web/src/locales/zh-CN/messages.po +++ b/apps/web/src/locales/zh-CN/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "当前模型" msgid "Activity" msgstr "动态" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "无法加载消息设置" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "沿用默认" @@ -2919,6 +2939,10 @@ msgstr "" msgid "Leave" msgstr "退出" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "移除" msgid "Remove {0}" msgstr "移除 {0}" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "" diff --git a/docs/self-host.md b/docs/self-host.md index 8349f41..a0e44d0 100644 --- a/docs/self-host.md +++ b/docs/self-host.md @@ -80,6 +80,26 @@ Docker computer topology: Set any of them to `0`, `none` or `unlimited` to remove that ceiling. A malformed value fails the supervisor at startup naming the variable, rather than surfacing later as a failed bot. +### Media library + +The deployment owner can add folders of photos and videos under Settings, Media. The worker +indexes them (names, dates, sizes; never moving or changing a file), re-indexes every six hours, +and the owner's bots can then search the library and copy a file into a conversation. Paths are +read on the server, so under Compose mount each folder read-only into both `api` and `worker` at +the same path, for example in an overlay: + +```yaml +services: + api: + volumes: ["/srv/media/Photos:/media/Photos:ro"] + worker: + volumes: ["/srv/media/Photos:/media/Photos:ro"] +``` + +and add `/media/Photos` in the settings. Indexed are the photo and video types a conversation can +hold; capture dates come from video metadata when ffmpeg is installed, otherwise from the file's +modification time. + ## Docker Compose (single machine) 1. Copy `.env.example` to `.env` and set `POSTGRES_PASSWORD` (`openssl rand -hex 16`), plus `BETTER_AUTH_SECRET`, `ENCRYPTION_KEY`, and `SCREEN_PROXY_SECRET` to independent long random strings (32+ characters; 64 hex for `ENCRYPTION_KEY`). Docker sandboxes also need a dedicated `SANDBOX_SUPERVISOR_TOKEN`. Keep existing `ENCRYPTION_KEY` values so stored credentials stay decryptable. diff --git a/packages/adapter-kit/src/background-jobs.test.ts b/packages/adapter-kit/src/background-jobs.test.ts index 1846253..e7eee9f 100644 --- a/packages/adapter-kit/src/background-jobs.test.ts +++ b/packages/adapter-kit/src/background-jobs.test.ts @@ -4,6 +4,9 @@ import { historyCompactJob, historyCompactJobKey, MEDIA_JOB_QUEUE, + MEDIA_LIBRARY_JOB_QUEUE, + mediaLibrarySyncJob, + mediaLibrarySyncJobKey, mediaRunJob, mediaRunJobKey, messagingDeliverJob, @@ -26,6 +29,7 @@ function handlers(): BackgroundJobHandlers { "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), "media.run": vi.fn(async () => undefined), + "media.library.sync": vi.fn(async () => undefined), }; } @@ -134,3 +138,21 @@ describe("mediaRunJob", () => { expect(mediaRunJobKey("job-1")).not.toBe(mediaRunJobKey("job-2")); }); }); + +describe("mediaLibrarySyncJob", () => { + it("validates, dispatches, and keeps one pending sync per folder on the library queue", async () => { + const target = handlers(); + await dispatchBackgroundJob(target, "media.library.sync", { sourceId: "source-1" }); + expect(target["media.library.sync"]).toHaveBeenCalledWith({ sourceId: "source-1" }); + expect(() => parseBackgroundJob("media.library.sync", { sourceId: "" })).toThrow(); + const at = new Date("2026-10-05T12:00:00.000Z"); + expect(mediaLibrarySyncJob("source-1", at)).toEqual({ + name: "media.library.sync", + payload: { sourceId: "source-1" }, + replaceKey: mediaLibrarySyncJobKey("source-1"), + queue: MEDIA_LIBRARY_JOB_QUEUE, + availableAt: at, + }); + expect(mediaLibrarySyncJob("source-1").availableAt).toBeUndefined(); + }); +}); diff --git a/packages/adapter-kit/src/background-jobs.ts b/packages/adapter-kit/src/background-jobs.ts index 6705ad5..a2a456f 100644 --- a/packages/adapter-kit/src/background-jobs.ts +++ b/packages/adapter-kit/src/background-jobs.ts @@ -24,6 +24,7 @@ const payloadSchemas = { "cloud_agent.poll": z.object({ agentId: z.string().min(1) }), "profile.consolidate": z.object({ userId: z.string().min(1) }), "media.run": z.object({ jobId: z.string().min(1) }), + "media.library.sync": z.object({ sourceId: z.string().min(1) }), } satisfies { [Name in BackgroundJobName]: z.ZodType }; export function parseBackgroundJob(name: string, payload: unknown): BackgroundJob { @@ -178,3 +179,20 @@ export function mediaRunJob(jobId: string, availableAt?: Date): BackgroundJob { ...(availableAt ? { availableAt } : {}), }; } + +/** Library syncs walk disks; one at a time keeps them from competing for the same drive. */ +export const MEDIA_LIBRARY_JOB_QUEUE = "media-library"; + +export function mediaLibrarySyncJobKey(sourceId: string): string { + return `media.library.sync:${sourceId}`; +} + +export function mediaLibrarySyncJob(sourceId: string, availableAt?: Date): BackgroundJob { + return { + name: "media.library.sync", + payload: { sourceId }, + replaceKey: mediaLibrarySyncJobKey(sourceId), + queue: MEDIA_LIBRARY_JOB_QUEUE, + ...(availableAt ? { availableAt } : {}), + }; +} diff --git a/packages/adapter-kit/src/types.ts b/packages/adapter-kit/src/types.ts index 8ac55ec..340d772 100644 --- a/packages/adapter-kit/src/types.ts +++ b/packages/adapter-kit/src/types.ts @@ -535,6 +535,8 @@ export interface BackgroundJobPayloads { "profile.consolidate": { userId: string }; /** Run one media job; scope, input and lease live on the job row. */ "media.run": { jobId: string }; + /** Re-index one media library folder; the source row holds the path and owner. */ + "media.library.sync": { sourceId: string }; } export type BackgroundJobName = keyof BackgroundJobPayloads; diff --git a/packages/adapters/src/background-job-handlers.ts b/packages/adapters/src/background-job-handlers.ts index 1c45073..16fd28a 100644 --- a/packages/adapters/src/background-job-handlers.ts +++ b/packages/adapters/src/background-job-handlers.ts @@ -16,6 +16,7 @@ import { scheduleComputerSleep, sleepComputerIfIdle } from "./computer-idle.js"; import { performComputerUpdate } from "./computer-update.js"; import type { createRunExecutor } from "./executor.js"; import { compactHistory } from "./history-compaction.js"; +import { syncMediaLibrarySource } from "./media-library.js"; import type { MediaRuntimeHost } from "./media-runtime.js"; import type { MemoryProviderResolver } from "./memory-provider-factory.js"; import { deliverMessagingOutbound, mirrorMessagingOutbound } from "./messaging-delivery.js"; @@ -117,6 +118,9 @@ export function createBackgroundJobHandlers(deps: { "media.run": async (payload) => { await deps.media?.runJob(payload.jobId); }, + "media.library.sync": async (payload) => { + await syncMediaLibrarySource(deps, payload.sourceId); + }, "profile.consolidate": async (payload) => { await consolidateProfile( { diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index fb87c63..ddf8695 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -573,6 +573,31 @@ export const builtinAgentTools: ConnectorTool[] = [ "Transcribe speech in an audio or video artifact into SRT and VTT subtitle files, shown on a media card. Set continue_after to get the transcript back, e.g. to burn the .srt in with edit_video.", inputSchema: mediaParameters(TranscribeInputSchema), }, + { + name: "search_media", + description: + "Search the user's photo and video library, newest first. query matches words in folder and file names (trips, events, dates people put in names). Returns asset ids; import_media brings one into the conversation.", + inputSchema: { + type: "object", + properties: { + query: { type: "string" }, + kind: { type: "string", enum: ["photo", "video"] }, + taken_after: { type: "string", description: "ISO 8601 date-time." }, + taken_before: { type: "string", description: "ISO 8601 date-time." }, + limit: { type: "integer", minimum: 1, maximum: 50, description: "Default 20." }, + }, + }, + }, + { + name: "import_media", + description: + "Copy a photo or video from the user's library into this conversation as an artifact, so it can be shown or edited with the media tools. The original file is not changed.", + inputSchema: { + type: "object", + properties: { asset_id: { type: "string" } }, + required: ["asset_id"], + }, + }, { name: "add_mcp_server", description: diff --git a/packages/adapters/src/executor-tools.test.ts b/packages/adapters/src/executor-tools.test.ts index f1035d0..3b2cb4f 100644 --- a/packages/adapters/src/executor-tools.test.ts +++ b/packages/adapters/src/executor-tools.test.ts @@ -13,15 +13,23 @@ const unrestricted = { kind: "specialist", restrictBuiltinTools: false, allowedB function select( bot: { kind: string; restrictBuiltinTools: boolean; allowedBuiltinTools: string[] }, - options: { graphicalToolsAllowed?: boolean; agentConnectionsEnabled?: boolean } = {}, + options: { + graphicalToolsAllowed?: boolean; + agentConnectionsEnabled?: boolean; + mediaLibraryEnabled?: boolean; + foreignAgentRun?: boolean; + messagingChannelRun?: boolean; + } = {}, ) { return selectBuiltinToolsForRun({ graphicalToolsAllowed: options.graphicalToolsAllowed ?? true, groupId: null, trigger: "user", semanticMemoryEnabled: false, - messagingChannelRun: false, + messagingChannelRun: options.messagingChannelRun ?? false, agentConnectionsEnabled: options.agentConnectionsEnabled, + mediaLibraryEnabled: options.mediaLibraryEnabled, + foreignAgentRun: options.foreignAgentRun, bot, }).map((tool) => tool.name); } @@ -39,6 +47,18 @@ describe("builtin tools per bot", () => { ); }); + it("offers the media library tools only to the user's own runs once a folder exists", () => { + const library = ["search_media", "import_media"]; + expect(select(unrestricted)).not.toEqual(expect.arrayContaining(["search_media"])); + expect(select(unrestricted, { mediaLibraryEnabled: true })).toEqual( + expect.arrayContaining(library), + ); + for (const run of [{ foreignAgentRun: true }, { messagingChannelRun: true }]) { + const names = select(unrestricted, { mediaLibraryEnabled: true, ...run }); + expect(names.filter((name) => library.includes(name))).toEqual([]); + } + }); + it("applies the allowlist to agent connection tools too", () => { const bot = { kind: "specialist", diff --git a/packages/adapters/src/executor-tools.ts b/packages/adapters/src/executor-tools.ts index 34d5577..07ea74d 100644 --- a/packages/adapters/src/executor-tools.ts +++ b/packages/adapters/src/executor-tools.ts @@ -34,6 +34,9 @@ export const MAIN_ONLY_BUILTIN_TOOLS: ReadonlySet = new Set([ ...PERSONAL_AGENT_TOOLS, ]); +/** Tools over the user's own photo and video library. */ +export const MEDIA_LIBRARY_TOOLS: ReadonlySet = new Set(["search_media", "import_media"]); + /** Tools that write what the user's bots know about the user, so only the user may drive them. */ const USER_MODEL_TOOLS: ReadonlySet = new Set([ "update_user_profile", @@ -110,6 +113,8 @@ export function selectBuiltinToolsForRun(options: { cloudAgentEnabled?: boolean; /** Media tools appear only for operations the space can run. */ mediaCapabilities?: MediaCapabilities; + /** The run's user has a media library folder. */ + mediaLibraryEnabled?: boolean; messagingChannelRun: boolean; /** Cross-owner agent connections only exist for chat-linked bots. */ agentConnectionsEnabled?: boolean; @@ -138,6 +143,11 @@ export function selectBuiltinToolsForRun(options: { ).filter( (tool) => !(options.foreignAgentRun && USER_MODEL_TOOLS.has(tool.name)) && + // The library is the user's private media: not for other owners' agents or shared chats. + (!MEDIA_LIBRARY_TOOLS.has(tool.name) || + (options.mediaLibraryEnabled === true && + !options.foreignAgentRun && + !options.messagingChannelRun)) && (!options.messagingChannelRun || (![ "remember", diff --git a/packages/adapters/src/executor.ts b/packages/adapters/src/executor.ts index c6805a9..f1faf42 100644 --- a/packages/adapters/src/executor.ts +++ b/packages/adapters/src/executor.ts @@ -20,6 +20,7 @@ import type { WebProvider, } from "@rakazo/adapter-kit"; import { + ArtifactTooLargeError, historyCompactJob, routineJobKey, routineWakeupJob, @@ -34,6 +35,8 @@ import { BotSecretName, BotSecretSubmission, isAttachmentImageMimeType, + MediaLibraryImportInputSchema, + MediaLibrarySearchInputSchema, type MessageBlock, MessageBlock as MessageBlockSchema, OPENAI_COMPATIBLE_PROVIDER_ID, @@ -61,6 +64,7 @@ import { isMessagingChannelRun, isOneShotRoutineCrons, isTerminal, + messageBlockForArtifact, messagingChannelId, messagingChannelPrivacyBlock, messagingDmSurfaceNote, @@ -304,6 +308,12 @@ import { needsOAuthProbe, parseMcpServerToolArgs, } from "./mcp-server-tool.js"; +import { + importMediaLibraryAsset, + MediaLibraryError, + searchMediaLibrary, + userHasMediaLibrary, +} from "./media-library.js"; import { executeMediaTool, isMediaToolName } from "./media-tools.js"; import { loadAgentMemoryContext } from "./memory-context.js"; import type { MemoryProviderResolver } from "./memory-provider-factory.js"; @@ -1559,6 +1569,9 @@ export function createRunExecutor(deps: ExecutorDeps) { semanticMemoryEnabled, cloudAgentEnabled: cloudAgentsEnabled(cloudAgent, run.spaceId), mediaCapabilities: await deps.media?.capabilities(run.spaceId).catch(() => undefined), + mediaLibraryEnabled: await userHasMediaLibrary(deps.prisma, run.userId).catch( + () => false, + ), messagingChannelRun, agentConnectionsEnabled: hasMessagingIdentity, foreignAgentRun, @@ -2533,6 +2546,38 @@ export function createRunExecutor(deps: ExecutorDeps) { }); } } + if (name === "search_media") { + const parsed = MediaLibrarySearchInputSchema.safeParse(args); + if (!parsed.success) return finish({ error: parsed.error.issues[0]?.message }); + return finish(await searchMediaLibrary(deps.prisma, run.userId, parsed.data)); + } + if (name === "import_media") { + const parsed = MediaLibraryImportInputSchema.safeParse(args); + if (!parsed.success) return finish({ error: "asset_id is required" }); + if (!deps.artifacts) return finish({ error: "artifact storage unavailable" }); + try { + const row = await importMediaLibraryAsset( + { prisma: deps.prisma, artifacts: deps.artifacts }, + { + spaceId: run.spaceId, + userId: run.userId, + botId: bot.id, + groupId: thread.groupId, + runId: run.id, + signal: context.signal, + }, + parsed.data.asset_id, + ); + await publishMessage(deps, run, "bot", [messageBlockForArtifact(row)]); + return finish({ ok: true, artifactId: row.id, name: row.name }); + } catch (error) { + if (error instanceof ArtifactTooLargeError) { + return finish({ error: `file exceeds the ${formatMiB(error.maxBytes)} limit` }); + } + if (error instanceof MediaLibraryError) return finish({ error: error.message }); + throw error; + } + } if (name === "attach_file") { const filePath = String(args.path ?? ""); if (!deps.artifacts) { diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 62c10ce..3450600 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -91,6 +91,7 @@ export { runFfmpeg, } from "./media/ffmpeg.js"; export { createVideoOperations } from "./media/video-operations.js"; +export * from "./media-library.js"; export * from "./media-operations.js"; export * from "./media-runtime.js"; export * from "./media-settings.js"; diff --git a/packages/adapters/src/media-library.test.ts b/packages/adapters/src/media-library.test.ts new file mode 100644 index 0000000..b2c363e --- /dev/null +++ b/packages/adapters/src/media-library.test.ts @@ -0,0 +1,63 @@ +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + checkMediaLibraryFolder, + mediaLibraryFileType, + walkMediaLibrary, +} from "./media-library.js"; + +const dirs: string[] = []; +function tempDir() { + const dir = mkdtempSync(path.join(tmpdir(), "rakazo-library-unit-")); + dirs.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +async function walk(root: string) { + const out: string[] = []; + for await (const file of walkMediaLibrary(root)) out.push(file); + return out; +} + +describe("media library", () => { + it("indexes photos and videos only", () => { + expect(mediaLibraryFileType("IMG_0001.JPG")).toEqual({ kind: "photo", mimeType: "image/jpeg" }); + expect(mediaLibraryFileType("clip.mp4")).toEqual({ kind: "video", mimeType: "video/mp4" }); + expect(mediaLibraryFileType("song.mp3")).toBeNull(); + expect(mediaLibraryFileType("notes.txt")).toBeNull(); + }); + + it("walks below the folder without hidden entries or symlinks", async () => { + const root = tempDir(); + const outside = tempDir(); + mkdirSync(path.join(root, "2024", "Summer"), { recursive: true }); + mkdirSync(path.join(root, ".cache")); + writeFileSync(path.join(root, "2024", "Summer", "b.jpg"), "x"); + writeFileSync(path.join(root, "2024", "a.png"), "x"); + writeFileSync(path.join(root, "2024", "readme.txt"), "x"); + writeFileSync(path.join(root, ".cache", "c.jpg"), "x"); + writeFileSync(path.join(root, ".hidden.jpg"), "x"); + writeFileSync(path.join(outside, "d.jpg"), "x"); + symlinkSync(path.join(outside, "d.jpg"), path.join(root, "link.jpg")); + symlinkSync(outside, path.join(root, "linked-folder")); + + expect(await walk(root)).toEqual(["2024/Summer/b.jpg", "2024/a.png"]); + }); + + it("accepts only an absolute path to an existing folder", async () => { + const root = tempDir(); + writeFileSync(path.join(root, "file.jpg"), "x"); + expect(await checkMediaLibraryFolder(root)).toBeNull(); + expect(await checkMediaLibraryFolder("photos")).toBe("Use an absolute folder path."); + expect(await checkMediaLibraryFolder(path.join(root, "file.jpg"))).toBe( + "This is not a folder.", + ); + expect(await checkMediaLibraryFolder(path.join(root, "missing"))).toBe("Folder not found."); + }); +}); diff --git a/packages/adapters/src/media-library.ts b/packages/adapters/src/media-library.ts new file mode 100644 index 0000000..3030fe5 --- /dev/null +++ b/packages/adapters/src/media-library.ts @@ -0,0 +1,343 @@ +import type { Dirent } from "node:fs"; +import { lstat, readdir, realpath, stat } from "node:fs/promises"; +import path from "node:path"; +import type { ArtifactStore, JobPublisher } from "@rakazo/adapter-kit"; +import { mediaLibrarySyncJob } from "@rakazo/adapter-kit"; +import type { + MediaLibraryKind, + MediaLibrarySearchInput, + MediaLibrarySource, + MediaLibrarySourceStatus, +} from "@rakazo/contracts"; +import { attachmentKindOf } from "@rakazo/contracts"; +import { inferAttachmentMimeType } from "@rakazo/core"; +import type { PrismaClient } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; +import { storeArtifactFromFile } from "./artifact-files.js"; +import { detectFfmpeg, probeMedia } from "./media/ffmpeg.js"; + +/* + * Media library folders: the deployment owner points evOo at folders on the server, a sync job + * indexes the photos and videos below them, and the user's bots search the index and import a + * file into the conversation when they need its bytes. Files are never moved or changed. + */ + +/** A folder is re-indexed this long after its last sync, so new files show up on their own. */ +export const MEDIA_LIBRARY_RESYNC_MS = 6 * 60 * 60 * 1000; +const DELETE_BATCH = 500; +const DEFAULT_SEARCH_LIMIT = 20; + +export class MediaLibraryError extends Error { + constructor(message: string) { + super(message); + this.name = "MediaLibraryError"; + } +} + +/** Photo or video for files a conversation can hold; null for everything else. */ +export function mediaLibraryFileType( + name: string, +): { kind: MediaLibraryKind; mimeType: string } | null { + const mimeType = inferAttachmentMimeType(name); + if (!mimeType) return null; + const kind = attachmentKindOf(mimeType); + if (kind === "image") return { kind: "photo", mimeType }; + if (kind === "video") return { kind: "video", mimeType }; + return null; +} + +function visible(entry: Dirent): boolean { + return !entry.name.startsWith(".") && !entry.isSymbolicLink(); +} + +/** + * Relative POSIX paths of the media files below `root`. Hidden entries and symlinks are skipped + * and never followed, so the walk cannot leave the folder; unreadable subfolders are skipped. + */ +export async function* walkMediaLibrary( + root: string, + signal?: AbortSignal, + relativeDir = "", +): AsyncGenerator { + signal?.throwIfAborted(); + let entries: Dirent[]; + try { + entries = await readdir(path.join(root, relativeDir), { withFileTypes: true }); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (relativeDir && (code === "ENOENT" || code === "EACCES" || code === "ENOTDIR")) return; + throw error; + } + for (const entry of entries.filter(visible).sort((a, b) => (a.name < b.name ? -1 : 1))) { + const relative = relativeDir ? `${relativeDir}/${entry.name}` : entry.name; + if (entry.isDirectory()) yield* walkMediaLibrary(root, signal, relative); + else if (entry.isFile() && mediaLibraryFileType(entry.name)) yield relative; + } +} + +export function mediaLibrarySourceView(row: { + id: string; + path: string; + status: string; + error: string | null; + assetCount: number; + lastSyncedAt: Date | null; +}): MediaLibrarySource { + return { + id: row.id, + path: row.path, + status: row.status as MediaLibrarySourceStatus, + error: row.error, + assetCount: row.assetCount, + lastSyncedAt: row.lastSyncedAt?.toISOString() ?? null, + }; +} + +/** An absolute path to an existing folder, or a message saying what is wrong with it. */ +export async function checkMediaLibraryFolder(folder: string): Promise { + if (!path.isAbsolute(folder)) return "Use an absolute folder path."; + try { + if (!(await stat(folder)).isDirectory()) return "This is not a folder."; + } catch { + return "Folder not found."; + } + return null; +} + +type IndexedFile = { + relativePath: string; + name: string; + kind: MediaLibraryKind; + mimeType: string; + size: bigint; + modifiedAt: Date; + takenAt: Date; + width: number | null; + height: number | null; + durationMs: number | null; +}; + +async function indexFile( + root: string, + relativePath: string, + canProbe: boolean, + signal?: AbortSignal, +): Promise { + const name = path.posix.basename(relativePath); + const type = mediaLibraryFileType(name); + if (!type) return null; + const full = path.join(root, ...relativePath.split("/")); + const info = await lstat(full).catch(() => null); + if (!info?.isFile()) return null; + const probe = canProbe + ? await probeMedia(full, { signal, mimeType: type.mimeType }).catch(() => null) + : null; + return { + relativePath, + name, + ...type, + size: BigInt(info.size), + modifiedAt: info.mtime, + takenAt: probe?.createdAt ?? info.mtime, + width: probe?.video?.width ?? null, + height: probe?.video?.height ?? null, + durationMs: type.kind === "video" && probe?.durationMs ? probe.durationMs : null, + }; +} + +/** + * Re-indexes one folder: new and changed files are probed and stored, files that are gone are + * dropped, unchanged files cost one stat. A failure is recorded on the source instead of + * retried; the next sync is scheduled either way. + */ +export async function syncMediaLibrarySource( + deps: { prisma: PrismaClient; jobs: JobPublisher }, + sourceId: string, + options: { signal?: AbortSignal; now?: () => Date } = {}, +): Promise { + const now = options.now ?? (() => new Date()); + const source = await deps.prisma.mediaLibrarySource.findUnique({ where: { id: sourceId } }); + if (!source) return; + await deps.prisma.mediaLibrarySource.update({ + where: { id: sourceId }, + data: { status: "syncing", error: null }, + }); + let result: { status: MediaLibrarySourceStatus; error: string | null; synced: boolean }; + try { + const problem = await checkMediaLibraryFolder(source.path); + if (problem) throw new MediaLibraryError(problem); + await indexFolder(deps.prisma, source, options.signal); + result = { status: "idle", error: null, synced: true }; + } catch (error) { + const message = + error instanceof MediaLibraryError ? error.message : "Could not read this folder."; + if (!(error instanceof MediaLibraryError)) { + getLogger().error(`media library sync failed for source ${sourceId}`, error); + } + result = { status: "failed", error: message, synced: false }; + } + const assetCount = await deps.prisma.mediaLibraryAsset.count({ where: { sourceId } }); + const updated = await deps.prisma.mediaLibrarySource.updateMany({ + where: { id: sourceId }, + data: { + status: result.status, + error: result.error, + assetCount, + ...(result.synced ? { lastSyncedAt: now() } : {}), + }, + }); + // Removed while syncing: nothing left to schedule. + if (updated.count === 0) return; + await deps.jobs.enqueue( + mediaLibrarySyncJob(sourceId, new Date(now().getTime() + MEDIA_LIBRARY_RESYNC_MS)), + ); +} + +async function indexFolder( + prisma: PrismaClient, + source: { id: string; userId: string; path: string }, + signal?: AbortSignal, +) { + const canProbe = (await detectFfmpeg()) !== null; + const known = new Map( + ( + await prisma.mediaLibraryAsset.findMany({ + where: { sourceId: source.id }, + select: { id: true, relativePath: true, size: true, modifiedAt: true }, + }) + ).map((row) => [row.relativePath, row]), + ); + const seen = new Set(); + for await (const relativePath of walkMediaLibrary(source.path, signal)) { + seen.add(relativePath); + const row = known.get(relativePath); + if (row) { + const info = await lstat(path.join(source.path, ...relativePath.split("/"))).catch( + () => null, + ); + if ( + info && + BigInt(info.size) === row.size && + info.mtime.getTime() === row.modifiedAt.getTime() + ) { + continue; + } + } + const file = await indexFile(source.path, relativePath, canProbe, signal); + if (!file) continue; + await prisma.mediaLibraryAsset.upsert({ + where: { sourceId_relativePath: { sourceId: source.id, relativePath } }, + create: { ...file, sourceId: source.id, userId: source.userId }, + update: file, + }); + } + const gone = [...known.values()].filter((row) => !seen.has(row.relativePath)); + for (let index = 0; index < gone.length; index += DELETE_BATCH) { + await prisma.mediaLibraryAsset.deleteMany({ + where: { id: { in: gone.slice(index, index + DELETE_BATCH).map((row) => row.id) } }, + }); + } +} + +export async function userHasMediaLibrary(prisma: PrismaClient, userId: string) { + return (await prisma.mediaLibrarySource.count({ where: { userId } })) > 0; +} + +/** The `search_media` tool: the user's library, newest capture first. */ +export async function searchMediaLibrary( + prisma: PrismaClient, + userId: string, + input: MediaLibrarySearchInput, +) { + const words = input.query?.split(/\s+/).filter(Boolean) ?? []; + const where = { + userId, + ...(input.kind ? { kind: input.kind } : {}), + ...(input.taken_after || input.taken_before + ? { + takenAt: { + ...(input.taken_after ? { gte: new Date(input.taken_after) } : {}), + ...(input.taken_before ? { lt: new Date(input.taken_before) } : {}), + }, + } + : {}), + // Folder and file names carry most of what people know about their media. + AND: words.map((word) => ({ + relativePath: { contains: word, mode: "insensitive" as const }, + })), + }; + const [total, rows] = await Promise.all([ + prisma.mediaLibraryAsset.count({ where }), + prisma.mediaLibraryAsset.findMany({ + where, + orderBy: [{ takenAt: "desc" }, { id: "asc" }], + take: input.limit ?? DEFAULT_SEARCH_LIMIT, + }), + ]); + return { + total, + items: rows.map((row) => ({ + asset_id: row.id, + path: row.relativePath, + kind: row.kind, + taken_at: row.takenAt.toISOString(), + size_mb: Math.round((Number(row.size) / (1024 * 1024)) * 10) / 10, + ...(row.width && row.height ? { width: row.width, height: row.height } : {}), + ...(row.durationMs ? { duration_s: Math.round(row.durationMs / 100) / 10 } : {}), + })), + }; +} + +/** + * The asset's file, or a MediaLibraryError. The resolved path must still lie inside the folder: + * a folder swapped for a symlink after indexing does not lead anywhere else. + */ +export async function resolveMediaLibraryFile( + prisma: PrismaClient, + userId: string, + assetId: string, +) { + const asset = await prisma.mediaLibraryAsset.findFirst({ + where: { id: assetId, userId }, + include: { source: { select: { path: true } } }, + }); + if (!asset) throw new MediaLibraryError("No media with this id in the library."); + const root = await realpath(asset.source.path).catch(() => null); + const file = await realpath(path.join(asset.source.path, ...asset.relativePath.split("/"))).catch( + () => null, + ); + if (!root || !file?.startsWith(`${root}${path.sep}`)) { + throw new MediaLibraryError("This file is no longer in the library folder."); + } + return { asset, file }; +} + +/** The `import_media` tool: copies a library file into the conversation as an artifact. */ +export async function importMediaLibraryAsset( + deps: { prisma: PrismaClient; artifacts: ArtifactStore }, + scope: { + spaceId: string; + userId: string; + botId: string | null; + groupId: string | null; + runId: string | null; + signal?: AbortSignal; + }, + assetId: string, +) { + const { asset, file } = await resolveMediaLibraryFile(deps.prisma, scope.userId, assetId); + return storeArtifactFromFile(deps, { + spaceId: scope.spaceId, + userId: scope.userId, + botId: scope.botId, + groupId: scope.groupId, + runId: scope.runId, + name: asset.name, + mimeType: asset.mimeType, + path: file, + signal: scope.signal, + ...(asset.width ? { width: asset.width } : {}), + ...(asset.height ? { height: asset.height } : {}), + ...(asset.durationMs ? { durationMs: asset.durationMs } : {}), + }); +} diff --git a/packages/adapters/src/media/ffmpeg.ts b/packages/adapters/src/media/ffmpeg.ts index dba7930..c184eb7 100644 --- a/packages/adapters/src/media/ffmpeg.ts +++ b/packages/adapters/src/media/ffmpeg.ts @@ -30,6 +30,8 @@ export interface MediaProbe { }; audio?: { codec: string; sampleRate: number; channels: number }; hasAudio: boolean; + /** When the recording was made, from the container's `creation_time` tag. */ + createdAt?: Date; } export interface FfmpegProgress { @@ -218,7 +220,7 @@ interface ProbeStream { /** Parses `ffprobe -print_format json -show_format -show_streams` output. */ export function parseProbeOutput(json: string): MediaProbe { const data = JSON.parse(json) as { - format?: { duration?: string }; + format?: { duration?: string; tags?: { creation_time?: string } }; streams?: ProbeStream[]; }; const streams = data.streams ?? []; @@ -245,9 +247,16 @@ export function parseProbeOutput(json: string): MediaProbe { } : {}), hasAudio: Boolean(audio), + ...creationTime(data.format?.tags?.creation_time), }; } +/** Cameras without a clock write 1970 or 1904; those are no capture time. */ +function creationTime(value: string | undefined): { createdAt?: Date } { + const time = value ? Date.parse(value) : Number.NaN; + return Number.isFinite(time) && time > Date.UTC(1990, 0, 1) ? { createdAt: new Date(time) } : {}; +} + function probeVideo(stream: ProbeStream, width: number, height: number) { const rotation = normalizeRotation( stream.side_data_list?.find((entry) => typeof entry.rotation === "number")?.rotation ?? diff --git a/packages/adapters/src/pi-runtime.ts b/packages/adapters/src/pi-runtime.ts index 95f295c..170d6eb 100644 --- a/packages/adapters/src/pi-runtime.ts +++ b/packages/adapters/src/pi-runtime.ts @@ -472,6 +472,8 @@ export function describeToolActivity(toolName: string, args: unknown): string { if (toolName === "write_file") return `Writing ${detail(record.path)}`; if (toolName === "list_files") return `Listing ${detail(record.path ?? ".")}`; if (toolName === "attach_file") return `Attaching ${detail(record.path)}`; + if (toolName === "search_media") return "Searching your photos and videos"; + if (toolName === "import_media") return "Bringing in a photo or video"; if (toolName === "open_path") return `Opening ${detail(record.path)}`; if (toolName === "render_plot") return "Rendering a chart"; if (toolName === "add_mcp_server") return `Connecting MCP server: ${detail(record.name)}`; diff --git a/packages/adapters/src/wakeup-graphile-host.test.ts b/packages/adapters/src/wakeup-graphile-host.test.ts index 7b955b9..a45ce67 100644 --- a/packages/adapters/src/wakeup-graphile-host.test.ts +++ b/packages/adapters/src/wakeup-graphile-host.test.ts @@ -25,6 +25,7 @@ function handlers(): BackgroundJobHandlers { "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), "media.run": vi.fn(async () => undefined), + "media.library.sync": vi.fn(async () => undefined), }; } diff --git a/packages/adapters/src/wakeup.postgres.test.ts b/packages/adapters/src/wakeup.postgres.test.ts index ad9d241..67965ca 100644 --- a/packages/adapters/src/wakeup.postgres.test.ts +++ b/packages/adapters/src/wakeup.postgres.test.ts @@ -30,6 +30,7 @@ function handlers(overrides: Partial = {}): BackgroundJob "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), "media.run": vi.fn(async () => undefined), + "media.library.sync": vi.fn(async () => undefined), ...overrides, }; } diff --git a/packages/adapters/src/wakeup.test.ts b/packages/adapters/src/wakeup.test.ts index 691f17a..14c57d9 100644 --- a/packages/adapters/src/wakeup.test.ts +++ b/packages/adapters/src/wakeup.test.ts @@ -15,6 +15,7 @@ function handlers(): BackgroundJobHandlers { "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), "media.run": vi.fn(async () => undefined), + "media.library.sync": vi.fn(async () => undefined), }; } diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 9f1ea52..5f265fe 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -17,6 +17,7 @@ export * from "./mcp.js"; export * from "./media.js"; export * from "./media-generation.js"; export * from "./media-jobs.js"; +export * from "./media-library.js"; export * from "./media-settings.js"; export * from "./media-transcription.js"; export * from "./media-video.js"; diff --git a/packages/contracts/src/media-library.ts b/packages/contracts/src/media-library.ts new file mode 100644 index 0000000..b03e745 --- /dev/null +++ b/packages/contracts/src/media-library.ts @@ -0,0 +1,39 @@ +import { z } from "zod"; + +/** Kinds a library folder indexes; everything else in the folder is ignored. */ +export const MediaLibraryKindSchema = z.enum(["photo", "video"]); +export type MediaLibraryKind = z.infer; + +export const MediaLibrarySourceStatusSchema = z.enum(["idle", "syncing", "failed"]); +export type MediaLibrarySourceStatus = z.infer; + +export const MediaLibrarySourceSchema = z.object({ + id: z.string(), + path: z.string(), + status: MediaLibrarySourceStatusSchema, + error: z.string().nullable(), + assetCount: z.number().int().nonnegative(), + lastSyncedAt: z.string().nullable(), +}); +export type MediaLibrarySource = z.infer; + +export const MediaLibrarySourceAddInputSchema = z.object({ + path: z.string().trim().min(1).max(4096), +}); +export type MediaLibrarySourceAddInput = z.infer; + +/** Arguments of the `search_media` agent tool. */ +export const MediaLibrarySearchInputSchema = z.object({ + query: z.string().trim().max(200).optional(), + kind: MediaLibraryKindSchema.optional(), + taken_after: z.string().datetime({ offset: true }).optional(), + taken_before: z.string().datetime({ offset: true }).optional(), + limit: z.number().int().min(1).max(50).optional(), +}); +export type MediaLibrarySearchInput = z.infer; + +/** Arguments of the `import_media` agent tool. */ +export const MediaLibraryImportInputSchema = z.object({ + asset_id: z.string().min(1).max(64), +}); +export type MediaLibraryImportInput = z.infer; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index db0cb0e..8b1ad34 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -108,6 +108,7 @@ import { MediaJobStartInputSchema, MediaJobStartOutputSchema, } from "./media-jobs.js"; +import { MediaLibrarySourceAddInputSchema, MediaLibrarySourceSchema } from "./media-library.js"; import { MediaSettingsUpdateSchema, MediaSettingsViewSchema } from "./media-settings.js"; import { ModelTierMappingSchema, SetModelTierInput } from "./model-tiers.js"; import { @@ -799,6 +800,13 @@ export const appContract = { get: oc.input(z.object({ jobId: Id })).output(MediaJobSnapshotSchema), cancel: oc.input(z.object({ jobId: Id })).output(MediaJobSnapshotSchema), }, + /** Server folders the user's bots can search and use; deployment owner only. */ + library: { + list: oc.output(z.array(MediaLibrarySourceSchema)), + add: oc.input(MediaLibrarySourceAddInputSchema).output(MediaLibrarySourceSchema), + remove: oc.input(z.object({ sourceId: Id })).output(z.object({ ok: z.literal(true) })), + sync: oc.input(z.object({ sourceId: Id })).output(MediaLibrarySourceSchema), + }, }, connections: { catalog: oc diff --git a/packages/db/prisma/migrations/20261005090000_media_library/migration.sql b/packages/db/prisma/migrations/20261005090000_media_library/migration.sql new file mode 100644 index 0000000..620e1b3 --- /dev/null +++ b/packages/db/prisma/migrations/20261005090000_media_library/migration.sql @@ -0,0 +1,52 @@ +-- Folders on the server whose photos and videos a user's bots can search and use, and their index. +-- CreateTable +CREATE TABLE "media_library_sources" ( + "id" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "path" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'idle', + "error" TEXT, + "assetCount" INTEGER NOT NULL DEFAULT 0, + "lastSyncedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_library_sources_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_library_assets" ( + "id" TEXT NOT NULL, + "sourceId" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "relativePath" TEXT NOT NULL, + "name" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "mimeType" TEXT NOT NULL, + "size" BIGINT NOT NULL, + "modifiedAt" TIMESTAMP(3) NOT NULL, + "takenAt" TIMESTAMP(3) NOT NULL, + "width" INTEGER, + "height" INTEGER, + "durationMs" INTEGER, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_library_assets_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "media_library_sources_userId_path_key" ON "media_library_sources"("userId", "path"); + +-- CreateIndex +CREATE INDEX "media_library_assets_userId_takenAt_idx" ON "media_library_assets"("userId", "takenAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "media_library_assets_sourceId_relativePath_key" ON "media_library_assets"("sourceId", "relativePath"); + +-- AddForeignKey +ALTER TABLE "media_library_sources" ADD CONSTRAINT "media_library_sources_userId_fkey" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_library_assets" ADD CONSTRAINT "media_library_assets_sourceId_fkey" FOREIGN KEY ("sourceId") REFERENCES "media_library_sources"("id") ON DELETE CASCADE ON UPDATE CASCADE; + diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 2a70125..2ba907b 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -54,6 +54,7 @@ model User { checkIns CheckIn[] dailyBriefs DailyBrief[] pushSubscriptions PushSubscription[] + mediaLibrarySources MediaLibrarySource[] notificationSettings UserNotificationSettings? @@unique([email]) @@ -1189,6 +1190,53 @@ model MediaJob { @@map("media_jobs") } +/// A folder on the server whose photos and videos the user's bots can search and use. +model MediaLibrarySource { + id String @id @default(cuid()) + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + /// Absolute folder path on the server the API and worker run on. + path String + /// idle, syncing or failed. + status String @default("idle") + error String? + assetCount Int @default(0) + lastSyncedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + assets MediaLibraryAsset[] + + @@unique([userId, path]) + @@map("media_library_sources") +} + +/// One photo or video in a library folder. Files stay where they are; this is the index. +model MediaLibraryAsset { + id String @id @default(cuid()) + sourceId String + source MediaLibrarySource @relation(fields: [sourceId], references: [id], onDelete: Cascade) + userId String + /// POSIX path below the source folder. + relativePath String + name String + /// photo or video. + kind String + mimeType String + size BigInt + modifiedAt DateTime + /// Capture time from the file's metadata, else its modification time. + takenAt DateTime + width Int? + height Int? + durationMs Int? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([sourceId, relativePath]) + @@index([userId, takenAt]) + @@map("media_library_assets") +} + model UsageRecord { id String @id @default(cuid()) spaceId String diff --git a/packages/testkit/src/cli/harness.ts b/packages/testkit/src/cli/harness.ts index cc051fe..13ab1dd 100644 --- a/packages/testkit/src/cli/harness.ts +++ b/packages/testkit/src/cli/harness.ts @@ -109,6 +109,7 @@ async function main() { "packages/testkit/src/bot-secrets.test.ts", "packages/testkit/src/delegation.postgres.test.ts", "packages/testkit/src/media.postgres.test.ts", + "packages/testkit/src/media-library.postgres.test.ts", "packages/testkit/src/user-profile.postgres.test.ts", "packages/testkit/src/onboarding-intake.postgres.test.ts", "packages/testkit/src/onboarding-discovery.postgres.test.ts", diff --git a/packages/testkit/src/media-library.postgres.test.ts b/packages/testkit/src/media-library.postgres.test.ts new file mode 100644 index 0000000..bf99963 --- /dev/null +++ b/packages/testkit/src/media-library.postgres.test.ts @@ -0,0 +1,204 @@ +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + ComposioEmulator, + importMediaLibraryAsset, + LocalArtifactStore, + MediaLibraryError, + searchMediaLibrary, +} from "@rakazo/adapters"; +import type { MediaLibrarySource } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { createApp } from "../../../apps/api/src/app.ts"; +import { sessionCookieHeader } from "./index.js"; + +type App = { request: (input: string, init?: RequestInit) => Response | Promise }; +type AppHandles = Awaited>; + +process.env.WAKEUP_DRIVER = "memory"; +process.env.SANDBOX_PROVIDER = "fake"; +process.env.AGENT_RUNTIME = "scripted"; + +const hasDb = process.env.VERIFY_DATABASE === "1" && Boolean(process.env.DATABASE_URL); +const describeWithDatabase = hasDb ? describe.sequential : describe.skip; + +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]); + +describeWithDatabase("media library (PostgreSQL)", () => { + let handles: AppHandles; + let app: App; + const stamp = `${Date.now()}-${Math.random().toString(36).slice(2)}`; + const dataDir = mkdtempSync(path.join(tmpdir(), "rakazo-library-data-")); + const library = mkdtempSync(path.join(tmpdir(), "rakazo-library-")); + const outside = mkdtempSync(path.join(tmpdir(), "rakazo-library-outside-")); + + beforeAll(async () => { + const { createApp } = await import("../../../apps/api/src/app.ts"); + handles = await createApp({ + databaseUrl: process.env.DATABASE_URL!, + dataDir, + sandboxProvider: "fake", + agentRuntime: "scripted", + wakeupDriver: "memory", + signupsEnabled: "true", + composio: new ComposioEmulator(), + }); + app = handles.app; + + mkdirSync(path.join(library, "Italy 2024")); + mkdirSync(path.join(library, ".thumbnails")); + writeFileSync(path.join(library, "Italy 2024", "beach.png"), PNG); + writeFileSync(path.join(library, "Italy 2024", "notes.txt"), "not media"); + writeFileSync(path.join(library, "birthday.png"), PNG); + writeFileSync(path.join(library, ".thumbnails", "hidden.png"), PNG); + writeFileSync(path.join(outside, "secret.png"), PNG); + symlinkSync(path.join(outside, "secret.png"), path.join(library, "linked.png")); + utimesSync( + path.join(library, "Italy 2024", "beach.png"), + new Date("2024-07-01"), + new Date("2024-07-01"), + ); + utimesSync(path.join(library, "birthday.png"), new Date("2025-03-01"), new Date("2025-03-01")); + }); + + afterAll(async () => { + await handles?.stop(); + for (const dir of [dataDir, library, outside]) rmSync(dir, { recursive: true, force: true }); + }); + + async function user(label: string) { + const cookie = await signup(app, `${label}-${stamp}@rakazo.test`, label); + const me = await rpc<{ userId: string; spaceId: string }>(app, cookie, "me"); + return { cookie, me }; + } + + async function waitForSync(cookie: string, sourceId: string) { + const start = Date.now(); + while (Date.now() - start < 20_000) { + const sources = await rpc(app, cookie, "media/library/list"); + const source = sources.find((entry) => entry.id === sourceId); + if (source && source.status !== "syncing") return source; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`library source ${sourceId} never finished syncing`); + } + + it("indexes, searches and imports only the owner's own library files", async () => { + const owner = await user("library-owner"); + const other = await user("library-other"); + await handles.prisma.deploymentSettings.upsert({ + where: { id: "default" }, + create: { id: "default", ownerUserId: owner.me.userId }, + update: { ownerUserId: owner.me.userId }, + }); + + expect((await raw(app, other.cookie, "media/library/list")).status).toBe(403); + expect((await raw(app, other.cookie, "media/library/add", { path: library })).status).toBe(403); + expect( + (await raw(app, owner.cookie, "media/library/add", { path: "relative/folder" })).status, + ).toBe(400); + expect( + (await raw(app, owner.cookie, "media/library/add", { path: path.join(library, "nope") })) + .status, + ).toBe(400); + + const added = await rpc(app, owner.cookie, "media/library/add", { + path: library, + }); + expect(added).toMatchObject({ path: library, status: "syncing" }); + // Adding the same folder again returns it instead of a duplicate. + expect( + (await rpc(app, owner.cookie, "media/library/add", { path: library })).id, + ).toBe(added.id); + const synced = await waitForSync(owner.cookie, added.id); + expect(synced).toMatchObject({ status: "idle", error: null, assetCount: 2 }); + expect(synced.lastSyncedAt).not.toBeNull(); + + const all = await searchMediaLibrary(handles.prisma, owner.me.userId, {}); + expect(all.total).toBe(2); + expect(all.items.map((item) => item.path)).toEqual(["birthday.png", "Italy 2024/beach.png"]); + const italy = await searchMediaLibrary(handles.prisma, owner.me.userId, { + query: "italy BEACH", + }); + expect(italy.items.map((item) => item.path)).toEqual(["Italy 2024/beach.png"]); + const before = await searchMediaLibrary(handles.prisma, owner.me.userId, { + taken_before: "2025-01-01T00:00:00Z", + }); + expect(before.items.map((item) => item.path)).toEqual(["Italy 2024/beach.png"]); + expect((await searchMediaLibrary(handles.prisma, other.me.userId, {})).total).toBe(0); + + const artifacts = new LocalArtifactStore(dataDir); + const assetId = italy.items[0]!.asset_id; + const imported = await importMediaLibraryAsset( + { prisma: handles.prisma, artifacts }, + { + spaceId: owner.me.spaceId, + userId: owner.me.userId, + botId: null, + groupId: null, + runId: null, + }, + assetId, + ); + expect(imported).toMatchObject({ name: "beach.png", mimeType: "image/png", size: PNG.length }); + await expect( + importMediaLibraryAsset( + { prisma: handles.prisma, artifacts }, + { + spaceId: other.me.spaceId, + userId: other.me.userId, + botId: null, + groupId: null, + runId: null, + }, + assetId, + ), + ).rejects.toBeInstanceOf(MediaLibraryError); + + // Another user can neither resync nor remove the owner's folder. + expect( + (await raw(app, other.cookie, "media/library/sync", { sourceId: added.id })).status, + ).toBe(403); + + // A deleted file leaves the index on the next sync. + rmSync(path.join(library, "birthday.png")); + await rpc(app, owner.cookie, "media/library/sync", { sourceId: added.id }); + expect(await waitForSync(owner.cookie, added.id)).toMatchObject({ assetCount: 1 }); + + await rpc(app, owner.cookie, "media/library/remove", { sourceId: added.id }); + expect(await rpc(app, owner.cookie, "media/library/list")).toEqual([]); + expect(await handles.prisma.mediaLibraryAsset.count({ where: { sourceId: added.id } })).toBe(0); + expect( + (await raw(app, owner.cookie, "media/library/sync", { sourceId: added.id })).status, + ).toBe(404); + }); +}); + +async function signup(app: App, email: string, name: string) { + const res = await app.request("/api/auth/sign-up/email", { + method: "POST", + headers: { "content-type": "application/json", origin: "http://127.0.0.1:5173" }, + body: JSON.stringify({ email, password: "password12", name }), + }); + if (res.status >= 400) throw new Error(`signup failed ${res.status}: ${await res.text()}`); + return sessionCookieHeader(res); +} + +function raw(app: App, cookie: string, proc: string, body: unknown = {}) { + return app.request(`/rpc/${proc}`, { + method: "POST", + headers: { "content-type": "application/json", cookie, origin: "http://127.0.0.1:5173" }, + body: JSON.stringify({ json: body }), + }); +} + +async function rpc(app: App, cookie: string, proc: string, body: unknown = {}): Promise { + const res = await raw(app, cookie, proc, body); + const text = await res.text(); + const parsed = JSON.parse(text) as { json?: T; error?: { message?: string } }; + if (res.status >= 400 || parsed.error) { + throw new Error(`${proc} ${res.status}: ${parsed.error?.message ?? text}`); + } + return parsed.json as T; +} From 738811dbe7d1b887005820826fbb6b91c1eb439a Mon Sep 17 00:00:00 2001 From: cynarAI <155274210+cynarAI@users.noreply.github.com> Date: Mon, 5 Oct 2026 12:42:30 +0200 Subject: [PATCH 2/2] Read the web app at 16px base size on phones (#155) Text sizes now follow a --font-scale variable that is 1 on wider screens and 8/7 below 768px, so the 14px base text reads at 16px on phones (Safari stops zooming into fields at that size) and every other size grows by the same ratio. One-off pixel sizes in the shell move onto the shared text tokens so they scale too. The stage subtitle now truncates inside the screen instead of spilling past both edges. Claude-Session: https://claude.ai/code/session_01PFEDgbyJm12ExL3Kn4Dk8B Co-authored-by: Claude --- apps/web/e2e/mobile-font-size.spec.ts | 41 +++++++++++++++++++ apps/web/src/components/stage/OoStage.tsx | 4 +- .../pages/ExternalConversationSettings.tsx | 5 ++- apps/web/src/pages/PeerMessagesOverlay.tsx | 2 +- apps/web/src/pages/PluginsOverlay.tsx | 4 +- apps/web/src/pages/RoutineEditor.tsx | 6 +-- apps/web/src/pages/RoutineSchedule.tsx | 4 +- apps/web/src/pages/ScratchpadSection.tsx | 2 +- apps/web/src/pages/Welcome.tsx | 2 +- apps/web/src/pages/shell/message-cards.tsx | 4 +- apps/web/src/pages/shell/transcript.tsx | 12 +++--- apps/web/src/pages/shell/user-menu.tsx | 2 +- apps/web/src/styles.css | 6 +-- packages/ui-web/src/styles.css | 34 +++++++++++---- 14 files changed, 96 insertions(+), 32 deletions(-) create mode 100644 apps/web/e2e/mobile-font-size.spec.ts diff --git a/apps/web/e2e/mobile-font-size.spec.ts b/apps/web/e2e/mobile-font-size.spec.ts new file mode 100644 index 0000000..759b6af --- /dev/null +++ b/apps/web/e2e/mobile-font-size.spec.ts @@ -0,0 +1,41 @@ +import { expect, type Locator, test } from "@playwright/test"; +import { captureScreenshot, completeOnboarding, signup } from "./helpers"; + +async function fontSize(locator: Locator) { + return locator.evaluate((element) => Number.parseFloat(getComputedStyle(element).fontSize)); +} + +test("phone widths read text at 16px and up while desktop keeps its sizes", async ({ + page, +}, testInfo) => { + const stamp = Date.now(); + await signup(page, `mobile-font-${stamp}@rakazo.test`, "password12", "Font QA"); + await completeOnboarding(page); + + const composer = page.getByRole("combobox", { name: /^Message/ }); + await composer.fill("Please review `shared/PROJECT_CHECKPOINT_WRAPUP.md`."); + await composer.press("Enter"); + const reply = page + .getByTestId("transcript") + .locator("[data-message-id]") + .filter({ hasText: "done. i handled:" }); + await expect(reply).toBeVisible({ timeout: 30_000 }); + + const smallText = page.locator(".text-ui-body-sm").first(); + const desktopSmall = await fontSize(smallText); + expect(desktopSmall).toBe(14); + + await page.setViewportSize({ width: 390, height: 844 }); + await expect.poll(() => fontSize(smallText)).toBeCloseTo(16, 1); + expect(await fontSize(composer)).toBeGreaterThanOrEqual(16); + + await page.emulateMedia({ colorScheme: "dark" }); + await expect(page.locator("html")).toHaveAttribute("data-theme", "dark"); + await captureScreenshot(page, testInfo, "mobile-font-size-dark"); + await page.emulateMedia({ colorScheme: "light" }); + await expect(page.locator("html")).toHaveAttribute("data-theme", "light"); + await captureScreenshot(page, testInfo, "mobile-font-size-light"); + + await page.setViewportSize({ width: 1280, height: 720 }); + await expect.poll(() => fontSize(smallText)).toBe(desktopSmall); +}); diff --git a/apps/web/src/components/stage/OoStage.tsx b/apps/web/src/components/stage/OoStage.tsx index e30c59d..c6b5d23 100644 --- a/apps/web/src/components/stage/OoStage.tsx +++ b/apps/web/src/components/stage/OoStage.tsx @@ -251,7 +251,7 @@ export function OoStage({ ) : null} ) : ( -

+

{subtitle}

)} @@ -548,7 +548,7 @@ export function OoGroupStage({ {subtitle}

) : ( -

+

{subtitle}

)} diff --git a/apps/web/src/pages/ExternalConversationSettings.tsx b/apps/web/src/pages/ExternalConversationSettings.tsx index 0b00102..396bb96 100644 --- a/apps/web/src/pages/ExternalConversationSettings.tsx +++ b/apps/web/src/pages/ExternalConversationSettings.tsx @@ -80,7 +80,10 @@ export function ExternalConversationSettings({ return (
-

+

{conversation.displayName || t`External conversation`}

diff --git a/apps/web/src/pages/PeerMessagesOverlay.tsx b/apps/web/src/pages/PeerMessagesOverlay.tsx index 5c61241..b8c8451 100644 --- a/apps/web/src/pages/PeerMessagesOverlay.tsx +++ b/apps/web/src/pages/PeerMessagesOverlay.tsx @@ -134,7 +134,7 @@ export function PeerMessagesOverlay({

{sent ? botName : peerBotName}
-
+
{peerMessage.text}
diff --git a/apps/web/src/pages/PluginsOverlay.tsx b/apps/web/src/pages/PluginsOverlay.tsx index 0c56fdc..01daf66 100644 --- a/apps/web/src/pages/PluginsOverlay.tsx +++ b/apps/web/src/pages/PluginsOverlay.tsx @@ -558,7 +558,9 @@ export function PluginsOverlay({ {item.name[0]}
)} -
{item.name}
+
+ {item.name} +