From 044669edfc44772b1003c3166d192204f4b04bba Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 08:45:30 +0000 Subject: [PATCH] Add a media library: owner folders that bots can search and import from The deployment owner adds folders of photos and videos in the media settings. A per-folder background job indexes them (relative path, kind, size, capture date, dimensions, duration), re-indexes every six hours, and drops files that are gone. The owner's own runs get two tools: search_media (words in folder and file names, kind, capture date range, newest first) and import_media (copies one file into the conversation as an artifact). Files are never changed; hidden entries and symlinks are skipped, and an import must resolve inside its folder. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Wy1md9QE1ayJbDCwJ4vAQs --- apps/api/src/media-library.ts | 82 +++++ apps/api/src/router.ts | 20 + apps/web/e2e/media-settings.spec.ts | 112 ++++++ .../src/components/MediaLibraryFolders.tsx | 162 +++++++++ apps/web/src/components/MediaSettings.tsx | 6 +- apps/web/src/locales/de/messages.po | 28 ++ apps/web/src/locales/en/messages.po | 28 ++ apps/web/src/locales/es/messages.po | 28 ++ apps/web/src/locales/hi/messages.po | 28 ++ apps/web/src/locales/ko/messages.po | 28 ++ apps/web/src/locales/pt-BR/messages.po | 28 ++ apps/web/src/locales/ru/messages.po | 28 ++ apps/web/src/locales/tr/messages.po | 28 ++ apps/web/src/locales/zh-CN/messages.po | 28 ++ docs/self-host.md | 20 + .../adapter-kit/src/background-jobs.test.ts | 22 ++ packages/adapter-kit/src/background-jobs.ts | 18 + packages/adapter-kit/src/types.ts | 2 + .../adapters/src/background-job-handlers.ts | 4 + packages/adapters/src/builtin-tools.ts | 25 ++ packages/adapters/src/executor-tools.test.ts | 24 +- packages/adapters/src/executor-tools.ts | 10 + packages/adapters/src/executor.ts | 45 +++ packages/adapters/src/index.ts | 1 + packages/adapters/src/media-library.test.ts | 63 ++++ packages/adapters/src/media-library.ts | 343 ++++++++++++++++++ packages/adapters/src/media/ffmpeg.ts | 11 +- packages/adapters/src/pi-runtime.ts | 2 + .../adapters/src/wakeup-graphile-host.test.ts | 1 + packages/adapters/src/wakeup.postgres.test.ts | 1 + packages/adapters/src/wakeup.test.ts | 1 + packages/contracts/src/index.ts | 1 + packages/contracts/src/media-library.ts | 39 ++ packages/contracts/src/rpc.ts | 8 + .../migration.sql | 52 +++ packages/db/prisma/schema.prisma | 48 +++ packages/testkit/src/cli/harness.ts | 1 + .../src/media-library.postgres.test.ts | 204 +++++++++++ 38 files changed, 1576 insertions(+), 4 deletions(-) create mode 100644 apps/api/src/media-library.ts create mode 100644 apps/web/src/components/MediaLibraryFolders.tsx create mode 100644 packages/adapters/src/media-library.test.ts create mode 100644 packages/adapters/src/media-library.ts create mode 100644 packages/contracts/src/media-library.ts create mode 100644 packages/db/prisma/migrations/20261005090000_media_library/migration.sql create mode 100644 packages/testkit/src/media-library.postgres.test.ts diff --git a/apps/api/src/media-library.ts b/apps/api/src/media-library.ts new file mode 100644 index 0000000..3d3c9b6 --- /dev/null +++ b/apps/api/src/media-library.ts @@ -0,0 +1,82 @@ +import path from "node:path"; +import { ORPCError } from "@orpc/server"; +import type { JobPublisher } from "@rakazo/adapter-kit"; +import { mediaLibrarySyncJob } from "@rakazo/adapter-kit"; +import { checkMediaLibraryFolder, mediaLibrarySourceView } from "@rakazo/adapters"; +import type { Actor, MediaLibrarySourceAddInput } from "@rakazo/contracts"; +import type { PrismaClient } from "@rakazo/db"; + +export interface MediaLibraryRouteDeps { + prisma: PrismaClient; + jobs: JobPublisher; +} + +/** Library folders are paths on the server, so only its owner adds them, like host folders. */ +function requireDeploymentOwner(actor: Actor) { + if (!actor.isDeploymentOwner) throw new ORPCError("FORBIDDEN"); +} + +async function ownSource(deps: MediaLibraryRouteDeps, actor: Actor, sourceId: string) { + const source = await deps.prisma.mediaLibrarySource.findFirst({ + where: { id: sourceId, userId: actor.userId }, + }); + if (!source) throw new ORPCError("NOT_FOUND"); + return source; +} + +export async function listMediaLibrarySources(deps: MediaLibraryRouteDeps, actor: Actor) { + requireDeploymentOwner(actor); + const rows = await deps.prisma.mediaLibrarySource.findMany({ + where: { userId: actor.userId }, + orderBy: { createdAt: "asc" }, + }); + return rows.map(mediaLibrarySourceView); +} + +export async function addMediaLibrarySource( + deps: MediaLibraryRouteDeps, + actor: Actor, + input: MediaLibrarySourceAddInput, +) { + requireDeploymentOwner(actor); + const problem = await checkMediaLibraryFolder(input.path); + if (problem) throw new ORPCError("BAD_REQUEST", { message: problem }); + const folder = path.resolve(input.path); + const existing = await deps.prisma.mediaLibrarySource.findUnique({ + where: { userId_path: { userId: actor.userId, path: folder } }, + }); + if (existing) return mediaLibrarySourceView(existing); + const source = await deps.prisma.mediaLibrarySource.create({ + data: { userId: actor.userId, path: folder, status: "syncing" }, + }); + await deps.jobs.enqueue(mediaLibrarySyncJob(source.id)); + return mediaLibrarySourceView(source); +} + +export async function removeMediaLibrarySource( + deps: MediaLibraryRouteDeps, + actor: Actor, + sourceId: string, +) { + requireDeploymentOwner(actor); + await ownSource(deps, actor, sourceId); + await deps.prisma.mediaLibrarySource.deleteMany({ + where: { id: sourceId, userId: actor.userId }, + }); + return { ok: true as const }; +} + +export async function syncMediaLibrarySourceNow( + deps: MediaLibraryRouteDeps, + actor: Actor, + sourceId: string, +) { + requireDeploymentOwner(actor); + await ownSource(deps, actor, sourceId); + const source = await deps.prisma.mediaLibrarySource.update({ + where: { id: sourceId }, + data: { status: "syncing", error: null }, + }); + await deps.jobs.enqueue(mediaLibrarySyncJob(sourceId)); + return mediaLibrarySourceView(source); +} diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index 6fc50df..0c19853 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -205,6 +205,12 @@ import { startMediaJob, updateMediaSettings, } from "./media-jobs.js"; +import { + addMediaLibrarySource, + listMediaLibrarySources, + removeMediaLibrarySource, + syncMediaLibrarySourceNow, +} from "./media-library.js"; import { disconnectMemoryProvider, getMemoryProviderConfig, @@ -3797,6 +3803,20 @@ export function createRouter(deps: RouterDeps) { cancelMediaJob(deps, context.actor, input.jobId), ), }, + library: { + list: authed.media.library.list.handler(({ context }) => + listMediaLibrarySources(deps, context.actor), + ), + add: authed.media.library.add.handler(({ context, input }) => + addMediaLibrarySource(deps, context.actor, input), + ), + remove: authed.media.library.remove.handler(({ context, input }) => + removeMediaLibrarySource(deps, context.actor, input.sourceId), + ), + sync: authed.media.library.sync.handler(({ context, input }) => + syncMediaLibrarySourceNow(deps, context.actor, input.sourceId), + ), + }, }, connections: { catalog: authed.connections.catalog.handler(async ({ context, input }) => { diff --git a/apps/web/e2e/media-settings.spec.ts b/apps/web/e2e/media-settings.spec.ts index bf606b7..b579236 100644 --- a/apps/web/e2e/media-settings.spec.ts +++ b/apps/web/e2e/media-settings.spec.ts @@ -37,6 +37,68 @@ function view(draft: Draft, previous: { comfyui: boolean; transcription: boolean }; } +type Source = { + id: string; + path: string; + status: "idle" | "syncing" | "failed"; + error: string | null; + assetCount: number; + lastSyncedAt: string | null; +}; + +/** Library folders on the fake server: a new folder indexes on its next list. */ +async function fakeLibrary(page: Page, initial: Source[] = []) { + let sources = initial; + const calls: { proc: string; input: unknown }[] = []; + await page.route("**/rpc/media/library/**", (route) => { + const proc = new URL(route.request().url()).pathname.split("/").at(-1) ?? ""; + const input = (route.request().postDataJSON() as { json?: unknown } | null)?.json; + calls.push({ proc, input }); + if (proc === "list") { + const listed = sources; + sources = sources.map((source) => + source.status === "syncing" + ? { ...source, status: "idle", assetCount: 1284, lastSyncedAt: new Date().toISOString() } + : source, + ); + return route.fulfill({ json: { json: listed } }); + } + if (proc === "add") { + const { path } = input as { path: string }; + if (!path.startsWith("/")) { + return route.fulfill({ + status: 400, + json: { + json: { + defined: false, + code: "BAD_REQUEST", + status: 400, + message: "Use an absolute folder path.", + }, + }, + }); + } + const source: Source = { + id: `source-${sources.length + 1}`, + path, + status: "syncing", + error: null, + assetCount: 0, + lastSyncedAt: null, + }; + sources = [...sources, source]; + return route.fulfill({ json: { json: source } }); + } + if (proc === "remove") { + const { sourceId } = input as { sourceId: string }; + sources = sources.filter((source) => source.id !== sourceId); + return route.fulfill({ json: { json: { ok: true } } }); + } + return route.fallback(); + }); + return calls; +} + async function asDeploymentOwner(page: Page, isDeploymentOwner: boolean) { await page.route("**/rpc/bootstrap", async (route) => { const response = await route.fetch(); @@ -50,6 +112,7 @@ async function asDeploymentOwner(page: Page, isDeploymentOwner: boolean) { test("the deployment owner points media at their own servers", async ({ page }, testInfo) => { await asDeploymentOwner(page, true); + await fakeLibrary(page); let current: ReturnType = { comfyui: null, transcription: null }; const saved: Draft[] = []; await page.route("**/rpc/media/settings/get", (route) => @@ -113,6 +176,55 @@ test("the deployment owner points media at their own servers", async ({ page }, await expect(comfyKey).toHaveAttribute("placeholder", "Optional"); }); +test("the deployment owner adds a photo folder to the library", async ({ page }, testInfo) => { + await asDeploymentOwner(page, true); + const calls = await fakeLibrary(page, [ + { + id: "source-0", + path: "/srv/media/Videos", + status: "failed", + error: "Folder not found.", + assetCount: 0, + lastSyncedAt: null, + }, + ]); + await page.route("**/rpc/media/settings/get", (route) => + route.fulfill({ json: { json: { comfyui: null, transcription: null } } }), + ); + + await signup(page, `media-library-${Date.now()}@rakazo.test`, "password12", "Library Owner"); + await completeOnboarding(page); + const settings = await openUserSettings(page); + await settings.getByTestId("settings-nav-media").click(); + const library = settings.getByTestId("media-library"); + await expect(library.getByText("Folder not found.")).toBeVisible(); + + const folder = library.getByLabel("Folder", { exact: true }); + await folder.fill("photos"); + await library.getByRole("button", { name: "Add", exact: true }).click(); + await expect(library.getByRole("alert")).toHaveText("Use an absolute folder path."); + + await folder.fill("/srv/media/Photos"); + await library.getByRole("button", { name: "Add", exact: true }).click(); + await expect(folder).toHaveValue(""); + await expect(library.getByText("/srv/media/Photos")).toBeVisible(); + // The new folder indexes in the background; the list catches up on its own. + await expect(library.getByText("1,284 files")).toBeVisible({ timeout: 10_000 }); + await captureScreenshot(page, testInfo, "media-library"); + + await library + .getByRole("listitem") + .filter({ hasText: "/srv/media/Videos" }) + .getByRole("button", { name: "Remove folder" }) + .click(); + await expect(library.getByText("/srv/media/Videos")).toHaveCount(0); + expect(calls.filter((call) => call.proc !== "list")).toEqual([ + { proc: "add", input: { path: "photos" } }, + { proc: "add", input: { path: "/srv/media/Photos" } }, + { proc: "remove", input: { sourceId: "source-0" } }, + ]); +}); + test("members who do not own the deployment have no media settings", async ({ page }) => { await asDeploymentOwner(page, false); await signup(page, `media-member-${Date.now()}@rakazo.test`, "password12", "Media Member"); diff --git a/apps/web/src/components/MediaLibraryFolders.tsx b/apps/web/src/components/MediaLibraryFolders.tsx new file mode 100644 index 0000000..7878daf --- /dev/null +++ b/apps/web/src/components/MediaLibraryFolders.tsx @@ -0,0 +1,162 @@ +import { Plural, Trans, useLingui } from "@lingui/react/macro"; +import type { MediaLibrarySource } from "@rakazo/contracts"; +import { Button, Input } from "@rakazo/ui-web"; +import { RefreshCw, X } from "lucide-react"; +import { useEffect, useId, useState } from "react"; +import { rpc } from "../lib/rpc"; +import { Shimmer } from "./ai/primitives"; + +const POLL_MS = 2000; + +/** Folders on the server whose photos and videos the owner's bots can search and use. */ +export function MediaLibraryFolders() { + const { t } = useLingui(); + const id = useId(); + const [sources, setSources] = useState(null); + const [folder, setFolder] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + rpc.media.library + .list() + .then((next) => { + if (!cancelled) setSources(next); + }) + .catch((err: unknown) => { + if (!cancelled) setError(err instanceof Error ? err.message : t`Couldn't load`); + }); + return () => { + cancelled = true; + }; + }, [t]); + + const syncing = sources?.some((source) => source.status === "syncing") ?? false; + useEffect(() => { + if (!syncing) return; + const timer = setInterval(() => { + rpc.media.library + .list() + .then(setSources) + .catch(() => undefined); + }, POLL_MS); + return () => clearInterval(timer); + }, [syncing]); + + async function act(action: () => Promise) { + if (busy) return; + setBusy(true); + setError(null); + try { + await action(); + } catch (err) { + setError(err instanceof Error ? err.message : t`Couldn't save`); + } finally { + setBusy(false); + } + } + + function replace(next: MediaLibrarySource) { + setSources((current) => [...(current ?? []).filter((source) => source.id !== next.id), next]); + } + + const disabled = busy || sources === null; + + return ( +
+

+ Library +

+ {sources?.length ? ( +
    + {sources.map((source) => ( +
  • +
    +

    {source.path}

    + {source.status === "syncing" ? ( +

    + + Indexing… + +

    + ) : source.status === "failed" ? ( +

    {source.error}

    + ) : ( +

    + +

    + )} +
    + + +
  • + ))} +
+ ) : null} +
{ + event.preventDefault(); + const path = folder.trim(); + if (!path) return; + void act(async () => { + replace(await rpc.media.library.add({ path })); + setFolder(""); + }); + }} + > + setFolder(event.target.value)} + /> + +
+ {error ? ( +

+ {error} +

+ ) : null} +
+ ); +} diff --git a/apps/web/src/components/MediaSettings.tsx b/apps/web/src/components/MediaSettings.tsx index 2ff24d2..0e667e1 100644 --- a/apps/web/src/components/MediaSettings.tsx +++ b/apps/web/src/components/MediaSettings.tsx @@ -4,6 +4,7 @@ import { Button, Field, FieldLabel, Input, Textarea } from "@rakazo/ui-web"; import { useEffect, useId, useRef, useState } from "react"; import { rpc } from "../lib/rpc"; import { SuccessPop } from "./ai/primitives"; +import { MediaLibraryFolders } from "./MediaLibraryFolders"; type KeyDraft = { value: string; remove: boolean }; @@ -67,7 +68,9 @@ function ApiKeyField({ ); } -/** The deployment owner's media servers: ComfyUI for generation, a transcription server. */ +/** + * The deployment owner's media: library folders, ComfyUI for generation, a transcription server. + */ export function MediaSettings() { const { t } = useLingui(); const ids = useId(); @@ -152,6 +155,7 @@ export function MediaSettings() { return (
+

ComfyUI

diff --git a/apps/web/src/locales/de/messages.po b/apps/web/src/locales/de/messages.po index 3a4c2c7..b497e5b 100644 --- a/apps/web/src/locales/de/messages.po +++ b/apps/web/src/locales/de/messages.po @@ -27,6 +27,11 @@ msgstr "(optional)" msgid "{0, plural, one {# day} other {# days}}" msgstr "{0, plural, one {# Tag} other {# Tage}}" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "{0, plural, one {# Datei} other {# Dateien}}" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "Aktives Modell" msgid "Activity" msgstr "Aktivität" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "Stimme konnte nicht aktualisiert werden" msgid "Could not upload {name}. Try again." msgstr "{name} konnte nicht hochgeladen werden. Versuche es erneut." +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "Konnte nicht geladen werden" msgid "Couldn't load messaging settings" msgstr "Messaging-Einstellungen konnten nicht geladen werden" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "Schwung" msgid "Flower" msgstr "Blume" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "Ordner" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "Schriften lokal geladen" @@ -2813,6 +2825,14 @@ msgstr "In Arbeit" msgid "Inbox & email" msgstr "Posteingang & E-Mail" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "Neu einlesen" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "Wird eingelesen…" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "Standard übernehmen" @@ -2919,6 +2939,10 @@ msgstr "Später" msgid "Leave" msgstr "Verlassen" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "Mediathek" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "Privatkanal — ich bin da." @@ -4145,6 +4169,10 @@ msgstr "Entfernen" msgid "Remove {0}" msgstr "{0} entfernen" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "Ordner entfernen" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "Git-Ereignis entfernen" diff --git a/apps/web/src/locales/en/messages.po b/apps/web/src/locales/en/messages.po index 85a0267..b1d5bef 100644 --- a/apps/web/src/locales/en/messages.po +++ b/apps/web/src/locales/en/messages.po @@ -27,6 +27,11 @@ msgstr "(optional)" msgid "{0, plural, one {# day} other {# days}}" msgstr "{0, plural, one {# day} other {# days}}" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "{0, plural, one {# file} other {# files}}" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "Active model" msgid "Activity" msgstr "Activity" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "Could not update voice" msgid "Could not upload {name}. Try again." msgstr "Could not upload {name}. Try again." +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "Couldn't load" msgid "Couldn't load messaging settings" msgstr "Couldn't load messaging settings" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "Flick" msgid "Flower" msgstr "Flower" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "Folder" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "Fonts loaded locally" @@ -2813,6 +2825,14 @@ msgstr "In progress" msgid "Inbox & email" msgstr "Inbox & email" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "Index again" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "Indexing…" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "Inherit default" @@ -2919,6 +2939,10 @@ msgstr "Later" msgid "Leave" msgstr "Leave" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "Library" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "Life channel — I'm here." @@ -4145,6 +4169,10 @@ msgstr "Remove" msgid "Remove {0}" msgstr "Remove {0}" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "Remove folder" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "Remove Git event" diff --git a/apps/web/src/locales/es/messages.po b/apps/web/src/locales/es/messages.po index 214143b..69cc77e 100644 --- a/apps/web/src/locales/es/messages.po +++ b/apps/web/src/locales/es/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "Modelo activo" msgid "Activity" msgstr "Actividad" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "No se pudo cargar la configuración de mensajería" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "Heredar predeterminado" @@ -2919,6 +2939,10 @@ msgstr "" msgid "Leave" msgstr "Salir" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "Quitar" msgid "Remove {0}" msgstr "Quitar {0}" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "" diff --git a/apps/web/src/locales/hi/messages.po b/apps/web/src/locales/hi/messages.po index 51e1b09..e5e0917 100644 --- a/apps/web/src/locales/hi/messages.po +++ b/apps/web/src/locales/hi/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "सक्रिय मॉडल" msgid "Activity" msgstr "गतिविधि" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "डिफ़ॉल्ट का पालन करें" @@ -2919,6 +2939,10 @@ msgstr "" msgid "Leave" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "हटाएं" msgid "Remove {0}" msgstr "{0} हटाएं" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "" diff --git a/apps/web/src/locales/ko/messages.po b/apps/web/src/locales/ko/messages.po index 478118c..5d80cb6 100644 --- a/apps/web/src/locales/ko/messages.po +++ b/apps/web/src/locales/ko/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "현재 모델" msgid "Activity" msgstr "활동" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "메시징 설정을 불러올 수 없습니다" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "기본 설정 사용" @@ -2919,6 +2939,10 @@ msgstr "" msgid "Leave" msgstr "나가기" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "삭제" msgid "Remove {0}" msgstr "{0} 삭제" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "" diff --git a/apps/web/src/locales/pt-BR/messages.po b/apps/web/src/locales/pt-BR/messages.po index 0b3193e..48cf91b 100644 --- a/apps/web/src/locales/pt-BR/messages.po +++ b/apps/web/src/locales/pt-BR/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "Modelo ativo" msgid "Activity" msgstr "Atividade" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "Herdar (padrão)" @@ -2919,6 +2939,10 @@ msgstr "" msgid "Leave" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "Remover" msgid "Remove {0}" msgstr "Remover {0}" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "" diff --git a/apps/web/src/locales/ru/messages.po b/apps/web/src/locales/ru/messages.po index 026f751..426b602 100644 --- a/apps/web/src/locales/ru/messages.po +++ b/apps/web/src/locales/ru/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "Активная модель" msgid "Activity" msgstr "Активность" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "Не удалось загрузить настройки обмена сообщениями." +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "Наследовать значение по умолчанию" @@ -2919,6 +2939,10 @@ msgstr "Позже" msgid "Leave" msgstr "Выйти" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "Удалить" msgid "Remove {0}" msgstr "Удалить {0}" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "Удалить событие Git" diff --git a/apps/web/src/locales/tr/messages.po b/apps/web/src/locales/tr/messages.po index 9b782c4..bbc9266 100644 --- a/apps/web/src/locales/tr/messages.po +++ b/apps/web/src/locales/tr/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "Etkin model" msgid "Activity" msgstr "Etkinlik" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "Varsayılanı devral" @@ -2919,6 +2939,10 @@ msgstr "" msgid "Leave" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "Kaldır" msgid "Remove {0}" msgstr "{0} kaldır" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "" diff --git a/apps/web/src/locales/zh-CN/messages.po b/apps/web/src/locales/zh-CN/messages.po index 4932ef1..4d1f650 100644 --- a/apps/web/src/locales/zh-CN/messages.po +++ b/apps/web/src/locales/zh-CN/messages.po @@ -27,6 +27,11 @@ msgstr "" msgid "{0, plural, one {# day} other {# days}}" msgstr "" +#. placeholder {0}: source.assetCount +#: src/components/MediaLibraryFolders.tsx +msgid "{0, plural, one {# file} other {# files}}" +msgstr "" + #. placeholder {0}: group.entries.length #: src/pages/ModelSettingsOverlay.tsx msgid "{0, plural, one {# model} other {# models}}" @@ -368,6 +373,7 @@ msgstr "当前模型" msgid "Activity" msgstr "动态" +#: src/components/MediaLibraryFolders.tsx #: src/components/onboarding/OnboardingScanStep.tsx #: src/pages/PluginsOverlay.tsx #: src/pages/ScratchpadSection.tsx @@ -1825,6 +1831,7 @@ msgstr "" msgid "Could not upload {name}. Try again." msgstr "" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -1835,6 +1842,7 @@ msgstr "" msgid "Couldn't load messaging settings" msgstr "无法加载消息设置" +#: src/components/MediaLibraryFolders.tsx #: src/components/MediaSettings.tsx #: src/components/NotificationSettingsPanel.tsx #: src/components/UserProfileSettings.tsx @@ -2515,6 +2523,10 @@ msgstr "" msgid "Flower" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Folder" +msgstr "" + #: src/pages/PrivacySettingsOverlay.tsx msgid "Fonts loaded locally" msgstr "" @@ -2813,6 +2825,14 @@ msgstr "" msgid "Inbox & email" msgstr "" +#: src/components/MediaLibraryFolders.tsx +msgid "Index again" +msgstr "" + +#: src/components/MediaLibraryFolders.tsx +msgid "Indexing…" +msgstr "" + #: src/pages/shell/bot-panel.tsx msgid "Inherit default" msgstr "沿用默认" @@ -2919,6 +2939,10 @@ msgstr "" msgid "Leave" msgstr "退出" +#: src/components/MediaLibraryFolders.tsx +msgid "Library" +msgstr "" + #: src/lib/voice-persona-copy.ts msgid "Life channel — I'm here." msgstr "" @@ -4145,6 +4169,10 @@ msgstr "移除" msgid "Remove {0}" msgstr "移除 {0}" +#: src/components/MediaLibraryFolders.tsx +msgid "Remove folder" +msgstr "" + #: src/pages/RoutineEditor.tsx msgid "Remove Git event" msgstr "" diff --git a/docs/self-host.md b/docs/self-host.md index 8349f41..a0e44d0 100644 --- a/docs/self-host.md +++ b/docs/self-host.md @@ -80,6 +80,26 @@ Docker computer topology: Set any of them to `0`, `none` or `unlimited` to remove that ceiling. A malformed value fails the supervisor at startup naming the variable, rather than surfacing later as a failed bot. +### Media library + +The deployment owner can add folders of photos and videos under Settings, Media. The worker +indexes them (names, dates, sizes; never moving or changing a file), re-indexes every six hours, +and the owner's bots can then search the library and copy a file into a conversation. Paths are +read on the server, so under Compose mount each folder read-only into both `api` and `worker` at +the same path, for example in an overlay: + +```yaml +services: + api: + volumes: ["/srv/media/Photos:/media/Photos:ro"] + worker: + volumes: ["/srv/media/Photos:/media/Photos:ro"] +``` + +and add `/media/Photos` in the settings. Indexed are the photo and video types a conversation can +hold; capture dates come from video metadata when ffmpeg is installed, otherwise from the file's +modification time. + ## Docker Compose (single machine) 1. Copy `.env.example` to `.env` and set `POSTGRES_PASSWORD` (`openssl rand -hex 16`), plus `BETTER_AUTH_SECRET`, `ENCRYPTION_KEY`, and `SCREEN_PROXY_SECRET` to independent long random strings (32+ characters; 64 hex for `ENCRYPTION_KEY`). Docker sandboxes also need a dedicated `SANDBOX_SUPERVISOR_TOKEN`. Keep existing `ENCRYPTION_KEY` values so stored credentials stay decryptable. diff --git a/packages/adapter-kit/src/background-jobs.test.ts b/packages/adapter-kit/src/background-jobs.test.ts index 1846253..e7eee9f 100644 --- a/packages/adapter-kit/src/background-jobs.test.ts +++ b/packages/adapter-kit/src/background-jobs.test.ts @@ -4,6 +4,9 @@ import { historyCompactJob, historyCompactJobKey, MEDIA_JOB_QUEUE, + MEDIA_LIBRARY_JOB_QUEUE, + mediaLibrarySyncJob, + mediaLibrarySyncJobKey, mediaRunJob, mediaRunJobKey, messagingDeliverJob, @@ -26,6 +29,7 @@ function handlers(): BackgroundJobHandlers { "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), "media.run": vi.fn(async () => undefined), + "media.library.sync": vi.fn(async () => undefined), }; } @@ -134,3 +138,21 @@ describe("mediaRunJob", () => { expect(mediaRunJobKey("job-1")).not.toBe(mediaRunJobKey("job-2")); }); }); + +describe("mediaLibrarySyncJob", () => { + it("validates, dispatches, and keeps one pending sync per folder on the library queue", async () => { + const target = handlers(); + await dispatchBackgroundJob(target, "media.library.sync", { sourceId: "source-1" }); + expect(target["media.library.sync"]).toHaveBeenCalledWith({ sourceId: "source-1" }); + expect(() => parseBackgroundJob("media.library.sync", { sourceId: "" })).toThrow(); + const at = new Date("2026-10-05T12:00:00.000Z"); + expect(mediaLibrarySyncJob("source-1", at)).toEqual({ + name: "media.library.sync", + payload: { sourceId: "source-1" }, + replaceKey: mediaLibrarySyncJobKey("source-1"), + queue: MEDIA_LIBRARY_JOB_QUEUE, + availableAt: at, + }); + expect(mediaLibrarySyncJob("source-1").availableAt).toBeUndefined(); + }); +}); diff --git a/packages/adapter-kit/src/background-jobs.ts b/packages/adapter-kit/src/background-jobs.ts index 6705ad5..a2a456f 100644 --- a/packages/adapter-kit/src/background-jobs.ts +++ b/packages/adapter-kit/src/background-jobs.ts @@ -24,6 +24,7 @@ const payloadSchemas = { "cloud_agent.poll": z.object({ agentId: z.string().min(1) }), "profile.consolidate": z.object({ userId: z.string().min(1) }), "media.run": z.object({ jobId: z.string().min(1) }), + "media.library.sync": z.object({ sourceId: z.string().min(1) }), } satisfies { [Name in BackgroundJobName]: z.ZodType }; export function parseBackgroundJob(name: string, payload: unknown): BackgroundJob { @@ -178,3 +179,20 @@ export function mediaRunJob(jobId: string, availableAt?: Date): BackgroundJob { ...(availableAt ? { availableAt } : {}), }; } + +/** Library syncs walk disks; one at a time keeps them from competing for the same drive. */ +export const MEDIA_LIBRARY_JOB_QUEUE = "media-library"; + +export function mediaLibrarySyncJobKey(sourceId: string): string { + return `media.library.sync:${sourceId}`; +} + +export function mediaLibrarySyncJob(sourceId: string, availableAt?: Date): BackgroundJob { + return { + name: "media.library.sync", + payload: { sourceId }, + replaceKey: mediaLibrarySyncJobKey(sourceId), + queue: MEDIA_LIBRARY_JOB_QUEUE, + ...(availableAt ? { availableAt } : {}), + }; +} diff --git a/packages/adapter-kit/src/types.ts b/packages/adapter-kit/src/types.ts index 8ac55ec..340d772 100644 --- a/packages/adapter-kit/src/types.ts +++ b/packages/adapter-kit/src/types.ts @@ -535,6 +535,8 @@ export interface BackgroundJobPayloads { "profile.consolidate": { userId: string }; /** Run one media job; scope, input and lease live on the job row. */ "media.run": { jobId: string }; + /** Re-index one media library folder; the source row holds the path and owner. */ + "media.library.sync": { sourceId: string }; } export type BackgroundJobName = keyof BackgroundJobPayloads; diff --git a/packages/adapters/src/background-job-handlers.ts b/packages/adapters/src/background-job-handlers.ts index 1c45073..16fd28a 100644 --- a/packages/adapters/src/background-job-handlers.ts +++ b/packages/adapters/src/background-job-handlers.ts @@ -16,6 +16,7 @@ import { scheduleComputerSleep, sleepComputerIfIdle } from "./computer-idle.js"; import { performComputerUpdate } from "./computer-update.js"; import type { createRunExecutor } from "./executor.js"; import { compactHistory } from "./history-compaction.js"; +import { syncMediaLibrarySource } from "./media-library.js"; import type { MediaRuntimeHost } from "./media-runtime.js"; import type { MemoryProviderResolver } from "./memory-provider-factory.js"; import { deliverMessagingOutbound, mirrorMessagingOutbound } from "./messaging-delivery.js"; @@ -117,6 +118,9 @@ export function createBackgroundJobHandlers(deps: { "media.run": async (payload) => { await deps.media?.runJob(payload.jobId); }, + "media.library.sync": async (payload) => { + await syncMediaLibrarySource(deps, payload.sourceId); + }, "profile.consolidate": async (payload) => { await consolidateProfile( { diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index fb87c63..ddf8695 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -573,6 +573,31 @@ export const builtinAgentTools: ConnectorTool[] = [ "Transcribe speech in an audio or video artifact into SRT and VTT subtitle files, shown on a media card. Set continue_after to get the transcript back, e.g. to burn the .srt in with edit_video.", inputSchema: mediaParameters(TranscribeInputSchema), }, + { + name: "search_media", + description: + "Search the user's photo and video library, newest first. query matches words in folder and file names (trips, events, dates people put in names). Returns asset ids; import_media brings one into the conversation.", + inputSchema: { + type: "object", + properties: { + query: { type: "string" }, + kind: { type: "string", enum: ["photo", "video"] }, + taken_after: { type: "string", description: "ISO 8601 date-time." }, + taken_before: { type: "string", description: "ISO 8601 date-time." }, + limit: { type: "integer", minimum: 1, maximum: 50, description: "Default 20." }, + }, + }, + }, + { + name: "import_media", + description: + "Copy a photo or video from the user's library into this conversation as an artifact, so it can be shown or edited with the media tools. The original file is not changed.", + inputSchema: { + type: "object", + properties: { asset_id: { type: "string" } }, + required: ["asset_id"], + }, + }, { name: "add_mcp_server", description: diff --git a/packages/adapters/src/executor-tools.test.ts b/packages/adapters/src/executor-tools.test.ts index f1035d0..3b2cb4f 100644 --- a/packages/adapters/src/executor-tools.test.ts +++ b/packages/adapters/src/executor-tools.test.ts @@ -13,15 +13,23 @@ const unrestricted = { kind: "specialist", restrictBuiltinTools: false, allowedB function select( bot: { kind: string; restrictBuiltinTools: boolean; allowedBuiltinTools: string[] }, - options: { graphicalToolsAllowed?: boolean; agentConnectionsEnabled?: boolean } = {}, + options: { + graphicalToolsAllowed?: boolean; + agentConnectionsEnabled?: boolean; + mediaLibraryEnabled?: boolean; + foreignAgentRun?: boolean; + messagingChannelRun?: boolean; + } = {}, ) { return selectBuiltinToolsForRun({ graphicalToolsAllowed: options.graphicalToolsAllowed ?? true, groupId: null, trigger: "user", semanticMemoryEnabled: false, - messagingChannelRun: false, + messagingChannelRun: options.messagingChannelRun ?? false, agentConnectionsEnabled: options.agentConnectionsEnabled, + mediaLibraryEnabled: options.mediaLibraryEnabled, + foreignAgentRun: options.foreignAgentRun, bot, }).map((tool) => tool.name); } @@ -39,6 +47,18 @@ describe("builtin tools per bot", () => { ); }); + it("offers the media library tools only to the user's own runs once a folder exists", () => { + const library = ["search_media", "import_media"]; + expect(select(unrestricted)).not.toEqual(expect.arrayContaining(["search_media"])); + expect(select(unrestricted, { mediaLibraryEnabled: true })).toEqual( + expect.arrayContaining(library), + ); + for (const run of [{ foreignAgentRun: true }, { messagingChannelRun: true }]) { + const names = select(unrestricted, { mediaLibraryEnabled: true, ...run }); + expect(names.filter((name) => library.includes(name))).toEqual([]); + } + }); + it("applies the allowlist to agent connection tools too", () => { const bot = { kind: "specialist", diff --git a/packages/adapters/src/executor-tools.ts b/packages/adapters/src/executor-tools.ts index 34d5577..07ea74d 100644 --- a/packages/adapters/src/executor-tools.ts +++ b/packages/adapters/src/executor-tools.ts @@ -34,6 +34,9 @@ export const MAIN_ONLY_BUILTIN_TOOLS: ReadonlySet = new Set([ ...PERSONAL_AGENT_TOOLS, ]); +/** Tools over the user's own photo and video library. */ +export const MEDIA_LIBRARY_TOOLS: ReadonlySet = new Set(["search_media", "import_media"]); + /** Tools that write what the user's bots know about the user, so only the user may drive them. */ const USER_MODEL_TOOLS: ReadonlySet = new Set([ "update_user_profile", @@ -110,6 +113,8 @@ export function selectBuiltinToolsForRun(options: { cloudAgentEnabled?: boolean; /** Media tools appear only for operations the space can run. */ mediaCapabilities?: MediaCapabilities; + /** The run's user has a media library folder. */ + mediaLibraryEnabled?: boolean; messagingChannelRun: boolean; /** Cross-owner agent connections only exist for chat-linked bots. */ agentConnectionsEnabled?: boolean; @@ -138,6 +143,11 @@ export function selectBuiltinToolsForRun(options: { ).filter( (tool) => !(options.foreignAgentRun && USER_MODEL_TOOLS.has(tool.name)) && + // The library is the user's private media: not for other owners' agents or shared chats. + (!MEDIA_LIBRARY_TOOLS.has(tool.name) || + (options.mediaLibraryEnabled === true && + !options.foreignAgentRun && + !options.messagingChannelRun)) && (!options.messagingChannelRun || (![ "remember", diff --git a/packages/adapters/src/executor.ts b/packages/adapters/src/executor.ts index c6805a9..f1faf42 100644 --- a/packages/adapters/src/executor.ts +++ b/packages/adapters/src/executor.ts @@ -20,6 +20,7 @@ import type { WebProvider, } from "@rakazo/adapter-kit"; import { + ArtifactTooLargeError, historyCompactJob, routineJobKey, routineWakeupJob, @@ -34,6 +35,8 @@ import { BotSecretName, BotSecretSubmission, isAttachmentImageMimeType, + MediaLibraryImportInputSchema, + MediaLibrarySearchInputSchema, type MessageBlock, MessageBlock as MessageBlockSchema, OPENAI_COMPATIBLE_PROVIDER_ID, @@ -61,6 +64,7 @@ import { isMessagingChannelRun, isOneShotRoutineCrons, isTerminal, + messageBlockForArtifact, messagingChannelId, messagingChannelPrivacyBlock, messagingDmSurfaceNote, @@ -304,6 +308,12 @@ import { needsOAuthProbe, parseMcpServerToolArgs, } from "./mcp-server-tool.js"; +import { + importMediaLibraryAsset, + MediaLibraryError, + searchMediaLibrary, + userHasMediaLibrary, +} from "./media-library.js"; import { executeMediaTool, isMediaToolName } from "./media-tools.js"; import { loadAgentMemoryContext } from "./memory-context.js"; import type { MemoryProviderResolver } from "./memory-provider-factory.js"; @@ -1559,6 +1569,9 @@ export function createRunExecutor(deps: ExecutorDeps) { semanticMemoryEnabled, cloudAgentEnabled: cloudAgentsEnabled(cloudAgent, run.spaceId), mediaCapabilities: await deps.media?.capabilities(run.spaceId).catch(() => undefined), + mediaLibraryEnabled: await userHasMediaLibrary(deps.prisma, run.userId).catch( + () => false, + ), messagingChannelRun, agentConnectionsEnabled: hasMessagingIdentity, foreignAgentRun, @@ -2533,6 +2546,38 @@ export function createRunExecutor(deps: ExecutorDeps) { }); } } + if (name === "search_media") { + const parsed = MediaLibrarySearchInputSchema.safeParse(args); + if (!parsed.success) return finish({ error: parsed.error.issues[0]?.message }); + return finish(await searchMediaLibrary(deps.prisma, run.userId, parsed.data)); + } + if (name === "import_media") { + const parsed = MediaLibraryImportInputSchema.safeParse(args); + if (!parsed.success) return finish({ error: "asset_id is required" }); + if (!deps.artifacts) return finish({ error: "artifact storage unavailable" }); + try { + const row = await importMediaLibraryAsset( + { prisma: deps.prisma, artifacts: deps.artifacts }, + { + spaceId: run.spaceId, + userId: run.userId, + botId: bot.id, + groupId: thread.groupId, + runId: run.id, + signal: context.signal, + }, + parsed.data.asset_id, + ); + await publishMessage(deps, run, "bot", [messageBlockForArtifact(row)]); + return finish({ ok: true, artifactId: row.id, name: row.name }); + } catch (error) { + if (error instanceof ArtifactTooLargeError) { + return finish({ error: `file exceeds the ${formatMiB(error.maxBytes)} limit` }); + } + if (error instanceof MediaLibraryError) return finish({ error: error.message }); + throw error; + } + } if (name === "attach_file") { const filePath = String(args.path ?? ""); if (!deps.artifacts) { diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 62c10ce..3450600 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -91,6 +91,7 @@ export { runFfmpeg, } from "./media/ffmpeg.js"; export { createVideoOperations } from "./media/video-operations.js"; +export * from "./media-library.js"; export * from "./media-operations.js"; export * from "./media-runtime.js"; export * from "./media-settings.js"; diff --git a/packages/adapters/src/media-library.test.ts b/packages/adapters/src/media-library.test.ts new file mode 100644 index 0000000..b2c363e --- /dev/null +++ b/packages/adapters/src/media-library.test.ts @@ -0,0 +1,63 @@ +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + checkMediaLibraryFolder, + mediaLibraryFileType, + walkMediaLibrary, +} from "./media-library.js"; + +const dirs: string[] = []; +function tempDir() { + const dir = mkdtempSync(path.join(tmpdir(), "rakazo-library-unit-")); + dirs.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +async function walk(root: string) { + const out: string[] = []; + for await (const file of walkMediaLibrary(root)) out.push(file); + return out; +} + +describe("media library", () => { + it("indexes photos and videos only", () => { + expect(mediaLibraryFileType("IMG_0001.JPG")).toEqual({ kind: "photo", mimeType: "image/jpeg" }); + expect(mediaLibraryFileType("clip.mp4")).toEqual({ kind: "video", mimeType: "video/mp4" }); + expect(mediaLibraryFileType("song.mp3")).toBeNull(); + expect(mediaLibraryFileType("notes.txt")).toBeNull(); + }); + + it("walks below the folder without hidden entries or symlinks", async () => { + const root = tempDir(); + const outside = tempDir(); + mkdirSync(path.join(root, "2024", "Summer"), { recursive: true }); + mkdirSync(path.join(root, ".cache")); + writeFileSync(path.join(root, "2024", "Summer", "b.jpg"), "x"); + writeFileSync(path.join(root, "2024", "a.png"), "x"); + writeFileSync(path.join(root, "2024", "readme.txt"), "x"); + writeFileSync(path.join(root, ".cache", "c.jpg"), "x"); + writeFileSync(path.join(root, ".hidden.jpg"), "x"); + writeFileSync(path.join(outside, "d.jpg"), "x"); + symlinkSync(path.join(outside, "d.jpg"), path.join(root, "link.jpg")); + symlinkSync(outside, path.join(root, "linked-folder")); + + expect(await walk(root)).toEqual(["2024/Summer/b.jpg", "2024/a.png"]); + }); + + it("accepts only an absolute path to an existing folder", async () => { + const root = tempDir(); + writeFileSync(path.join(root, "file.jpg"), "x"); + expect(await checkMediaLibraryFolder(root)).toBeNull(); + expect(await checkMediaLibraryFolder("photos")).toBe("Use an absolute folder path."); + expect(await checkMediaLibraryFolder(path.join(root, "file.jpg"))).toBe( + "This is not a folder.", + ); + expect(await checkMediaLibraryFolder(path.join(root, "missing"))).toBe("Folder not found."); + }); +}); diff --git a/packages/adapters/src/media-library.ts b/packages/adapters/src/media-library.ts new file mode 100644 index 0000000..3030fe5 --- /dev/null +++ b/packages/adapters/src/media-library.ts @@ -0,0 +1,343 @@ +import type { Dirent } from "node:fs"; +import { lstat, readdir, realpath, stat } from "node:fs/promises"; +import path from "node:path"; +import type { ArtifactStore, JobPublisher } from "@rakazo/adapter-kit"; +import { mediaLibrarySyncJob } from "@rakazo/adapter-kit"; +import type { + MediaLibraryKind, + MediaLibrarySearchInput, + MediaLibrarySource, + MediaLibrarySourceStatus, +} from "@rakazo/contracts"; +import { attachmentKindOf } from "@rakazo/contracts"; +import { inferAttachmentMimeType } from "@rakazo/core"; +import type { PrismaClient } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; +import { storeArtifactFromFile } from "./artifact-files.js"; +import { detectFfmpeg, probeMedia } from "./media/ffmpeg.js"; + +/* + * Media library folders: the deployment owner points evOo at folders on the server, a sync job + * indexes the photos and videos below them, and the user's bots search the index and import a + * file into the conversation when they need its bytes. Files are never moved or changed. + */ + +/** A folder is re-indexed this long after its last sync, so new files show up on their own. */ +export const MEDIA_LIBRARY_RESYNC_MS = 6 * 60 * 60 * 1000; +const DELETE_BATCH = 500; +const DEFAULT_SEARCH_LIMIT = 20; + +export class MediaLibraryError extends Error { + constructor(message: string) { + super(message); + this.name = "MediaLibraryError"; + } +} + +/** Photo or video for files a conversation can hold; null for everything else. */ +export function mediaLibraryFileType( + name: string, +): { kind: MediaLibraryKind; mimeType: string } | null { + const mimeType = inferAttachmentMimeType(name); + if (!mimeType) return null; + const kind = attachmentKindOf(mimeType); + if (kind === "image") return { kind: "photo", mimeType }; + if (kind === "video") return { kind: "video", mimeType }; + return null; +} + +function visible(entry: Dirent): boolean { + return !entry.name.startsWith(".") && !entry.isSymbolicLink(); +} + +/** + * Relative POSIX paths of the media files below `root`. Hidden entries and symlinks are skipped + * and never followed, so the walk cannot leave the folder; unreadable subfolders are skipped. + */ +export async function* walkMediaLibrary( + root: string, + signal?: AbortSignal, + relativeDir = "", +): AsyncGenerator { + signal?.throwIfAborted(); + let entries: Dirent[]; + try { + entries = await readdir(path.join(root, relativeDir), { withFileTypes: true }); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (relativeDir && (code === "ENOENT" || code === "EACCES" || code === "ENOTDIR")) return; + throw error; + } + for (const entry of entries.filter(visible).sort((a, b) => (a.name < b.name ? -1 : 1))) { + const relative = relativeDir ? `${relativeDir}/${entry.name}` : entry.name; + if (entry.isDirectory()) yield* walkMediaLibrary(root, signal, relative); + else if (entry.isFile() && mediaLibraryFileType(entry.name)) yield relative; + } +} + +export function mediaLibrarySourceView(row: { + id: string; + path: string; + status: string; + error: string | null; + assetCount: number; + lastSyncedAt: Date | null; +}): MediaLibrarySource { + return { + id: row.id, + path: row.path, + status: row.status as MediaLibrarySourceStatus, + error: row.error, + assetCount: row.assetCount, + lastSyncedAt: row.lastSyncedAt?.toISOString() ?? null, + }; +} + +/** An absolute path to an existing folder, or a message saying what is wrong with it. */ +export async function checkMediaLibraryFolder(folder: string): Promise { + if (!path.isAbsolute(folder)) return "Use an absolute folder path."; + try { + if (!(await stat(folder)).isDirectory()) return "This is not a folder."; + } catch { + return "Folder not found."; + } + return null; +} + +type IndexedFile = { + relativePath: string; + name: string; + kind: MediaLibraryKind; + mimeType: string; + size: bigint; + modifiedAt: Date; + takenAt: Date; + width: number | null; + height: number | null; + durationMs: number | null; +}; + +async function indexFile( + root: string, + relativePath: string, + canProbe: boolean, + signal?: AbortSignal, +): Promise { + const name = path.posix.basename(relativePath); + const type = mediaLibraryFileType(name); + if (!type) return null; + const full = path.join(root, ...relativePath.split("/")); + const info = await lstat(full).catch(() => null); + if (!info?.isFile()) return null; + const probe = canProbe + ? await probeMedia(full, { signal, mimeType: type.mimeType }).catch(() => null) + : null; + return { + relativePath, + name, + ...type, + size: BigInt(info.size), + modifiedAt: info.mtime, + takenAt: probe?.createdAt ?? info.mtime, + width: probe?.video?.width ?? null, + height: probe?.video?.height ?? null, + durationMs: type.kind === "video" && probe?.durationMs ? probe.durationMs : null, + }; +} + +/** + * Re-indexes one folder: new and changed files are probed and stored, files that are gone are + * dropped, unchanged files cost one stat. A failure is recorded on the source instead of + * retried; the next sync is scheduled either way. + */ +export async function syncMediaLibrarySource( + deps: { prisma: PrismaClient; jobs: JobPublisher }, + sourceId: string, + options: { signal?: AbortSignal; now?: () => Date } = {}, +): Promise { + const now = options.now ?? (() => new Date()); + const source = await deps.prisma.mediaLibrarySource.findUnique({ where: { id: sourceId } }); + if (!source) return; + await deps.prisma.mediaLibrarySource.update({ + where: { id: sourceId }, + data: { status: "syncing", error: null }, + }); + let result: { status: MediaLibrarySourceStatus; error: string | null; synced: boolean }; + try { + const problem = await checkMediaLibraryFolder(source.path); + if (problem) throw new MediaLibraryError(problem); + await indexFolder(deps.prisma, source, options.signal); + result = { status: "idle", error: null, synced: true }; + } catch (error) { + const message = + error instanceof MediaLibraryError ? error.message : "Could not read this folder."; + if (!(error instanceof MediaLibraryError)) { + getLogger().error(`media library sync failed for source ${sourceId}`, error); + } + result = { status: "failed", error: message, synced: false }; + } + const assetCount = await deps.prisma.mediaLibraryAsset.count({ where: { sourceId } }); + const updated = await deps.prisma.mediaLibrarySource.updateMany({ + where: { id: sourceId }, + data: { + status: result.status, + error: result.error, + assetCount, + ...(result.synced ? { lastSyncedAt: now() } : {}), + }, + }); + // Removed while syncing: nothing left to schedule. + if (updated.count === 0) return; + await deps.jobs.enqueue( + mediaLibrarySyncJob(sourceId, new Date(now().getTime() + MEDIA_LIBRARY_RESYNC_MS)), + ); +} + +async function indexFolder( + prisma: PrismaClient, + source: { id: string; userId: string; path: string }, + signal?: AbortSignal, +) { + const canProbe = (await detectFfmpeg()) !== null; + const known = new Map( + ( + await prisma.mediaLibraryAsset.findMany({ + where: { sourceId: source.id }, + select: { id: true, relativePath: true, size: true, modifiedAt: true }, + }) + ).map((row) => [row.relativePath, row]), + ); + const seen = new Set(); + for await (const relativePath of walkMediaLibrary(source.path, signal)) { + seen.add(relativePath); + const row = known.get(relativePath); + if (row) { + const info = await lstat(path.join(source.path, ...relativePath.split("/"))).catch( + () => null, + ); + if ( + info && + BigInt(info.size) === row.size && + info.mtime.getTime() === row.modifiedAt.getTime() + ) { + continue; + } + } + const file = await indexFile(source.path, relativePath, canProbe, signal); + if (!file) continue; + await prisma.mediaLibraryAsset.upsert({ + where: { sourceId_relativePath: { sourceId: source.id, relativePath } }, + create: { ...file, sourceId: source.id, userId: source.userId }, + update: file, + }); + } + const gone = [...known.values()].filter((row) => !seen.has(row.relativePath)); + for (let index = 0; index < gone.length; index += DELETE_BATCH) { + await prisma.mediaLibraryAsset.deleteMany({ + where: { id: { in: gone.slice(index, index + DELETE_BATCH).map((row) => row.id) } }, + }); + } +} + +export async function userHasMediaLibrary(prisma: PrismaClient, userId: string) { + return (await prisma.mediaLibrarySource.count({ where: { userId } })) > 0; +} + +/** The `search_media` tool: the user's library, newest capture first. */ +export async function searchMediaLibrary( + prisma: PrismaClient, + userId: string, + input: MediaLibrarySearchInput, +) { + const words = input.query?.split(/\s+/).filter(Boolean) ?? []; + const where = { + userId, + ...(input.kind ? { kind: input.kind } : {}), + ...(input.taken_after || input.taken_before + ? { + takenAt: { + ...(input.taken_after ? { gte: new Date(input.taken_after) } : {}), + ...(input.taken_before ? { lt: new Date(input.taken_before) } : {}), + }, + } + : {}), + // Folder and file names carry most of what people know about their media. + AND: words.map((word) => ({ + relativePath: { contains: word, mode: "insensitive" as const }, + })), + }; + const [total, rows] = await Promise.all([ + prisma.mediaLibraryAsset.count({ where }), + prisma.mediaLibraryAsset.findMany({ + where, + orderBy: [{ takenAt: "desc" }, { id: "asc" }], + take: input.limit ?? DEFAULT_SEARCH_LIMIT, + }), + ]); + return { + total, + items: rows.map((row) => ({ + asset_id: row.id, + path: row.relativePath, + kind: row.kind, + taken_at: row.takenAt.toISOString(), + size_mb: Math.round((Number(row.size) / (1024 * 1024)) * 10) / 10, + ...(row.width && row.height ? { width: row.width, height: row.height } : {}), + ...(row.durationMs ? { duration_s: Math.round(row.durationMs / 100) / 10 } : {}), + })), + }; +} + +/** + * The asset's file, or a MediaLibraryError. The resolved path must still lie inside the folder: + * a folder swapped for a symlink after indexing does not lead anywhere else. + */ +export async function resolveMediaLibraryFile( + prisma: PrismaClient, + userId: string, + assetId: string, +) { + const asset = await prisma.mediaLibraryAsset.findFirst({ + where: { id: assetId, userId }, + include: { source: { select: { path: true } } }, + }); + if (!asset) throw new MediaLibraryError("No media with this id in the library."); + const root = await realpath(asset.source.path).catch(() => null); + const file = await realpath(path.join(asset.source.path, ...asset.relativePath.split("/"))).catch( + () => null, + ); + if (!root || !file?.startsWith(`${root}${path.sep}`)) { + throw new MediaLibraryError("This file is no longer in the library folder."); + } + return { asset, file }; +} + +/** The `import_media` tool: copies a library file into the conversation as an artifact. */ +export async function importMediaLibraryAsset( + deps: { prisma: PrismaClient; artifacts: ArtifactStore }, + scope: { + spaceId: string; + userId: string; + botId: string | null; + groupId: string | null; + runId: string | null; + signal?: AbortSignal; + }, + assetId: string, +) { + const { asset, file } = await resolveMediaLibraryFile(deps.prisma, scope.userId, assetId); + return storeArtifactFromFile(deps, { + spaceId: scope.spaceId, + userId: scope.userId, + botId: scope.botId, + groupId: scope.groupId, + runId: scope.runId, + name: asset.name, + mimeType: asset.mimeType, + path: file, + signal: scope.signal, + ...(asset.width ? { width: asset.width } : {}), + ...(asset.height ? { height: asset.height } : {}), + ...(asset.durationMs ? { durationMs: asset.durationMs } : {}), + }); +} diff --git a/packages/adapters/src/media/ffmpeg.ts b/packages/adapters/src/media/ffmpeg.ts index dba7930..c184eb7 100644 --- a/packages/adapters/src/media/ffmpeg.ts +++ b/packages/adapters/src/media/ffmpeg.ts @@ -30,6 +30,8 @@ export interface MediaProbe { }; audio?: { codec: string; sampleRate: number; channels: number }; hasAudio: boolean; + /** When the recording was made, from the container's `creation_time` tag. */ + createdAt?: Date; } export interface FfmpegProgress { @@ -218,7 +220,7 @@ interface ProbeStream { /** Parses `ffprobe -print_format json -show_format -show_streams` output. */ export function parseProbeOutput(json: string): MediaProbe { const data = JSON.parse(json) as { - format?: { duration?: string }; + format?: { duration?: string; tags?: { creation_time?: string } }; streams?: ProbeStream[]; }; const streams = data.streams ?? []; @@ -245,9 +247,16 @@ export function parseProbeOutput(json: string): MediaProbe { } : {}), hasAudio: Boolean(audio), + ...creationTime(data.format?.tags?.creation_time), }; } +/** Cameras without a clock write 1970 or 1904; those are no capture time. */ +function creationTime(value: string | undefined): { createdAt?: Date } { + const time = value ? Date.parse(value) : Number.NaN; + return Number.isFinite(time) && time > Date.UTC(1990, 0, 1) ? { createdAt: new Date(time) } : {}; +} + function probeVideo(stream: ProbeStream, width: number, height: number) { const rotation = normalizeRotation( stream.side_data_list?.find((entry) => typeof entry.rotation === "number")?.rotation ?? diff --git a/packages/adapters/src/pi-runtime.ts b/packages/adapters/src/pi-runtime.ts index 95f295c..170d6eb 100644 --- a/packages/adapters/src/pi-runtime.ts +++ b/packages/adapters/src/pi-runtime.ts @@ -472,6 +472,8 @@ export function describeToolActivity(toolName: string, args: unknown): string { if (toolName === "write_file") return `Writing ${detail(record.path)}`; if (toolName === "list_files") return `Listing ${detail(record.path ?? ".")}`; if (toolName === "attach_file") return `Attaching ${detail(record.path)}`; + if (toolName === "search_media") return "Searching your photos and videos"; + if (toolName === "import_media") return "Bringing in a photo or video"; if (toolName === "open_path") return `Opening ${detail(record.path)}`; if (toolName === "render_plot") return "Rendering a chart"; if (toolName === "add_mcp_server") return `Connecting MCP server: ${detail(record.name)}`; diff --git a/packages/adapters/src/wakeup-graphile-host.test.ts b/packages/adapters/src/wakeup-graphile-host.test.ts index 7b955b9..a45ce67 100644 --- a/packages/adapters/src/wakeup-graphile-host.test.ts +++ b/packages/adapters/src/wakeup-graphile-host.test.ts @@ -25,6 +25,7 @@ function handlers(): BackgroundJobHandlers { "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), "media.run": vi.fn(async () => undefined), + "media.library.sync": vi.fn(async () => undefined), }; } diff --git a/packages/adapters/src/wakeup.postgres.test.ts b/packages/adapters/src/wakeup.postgres.test.ts index ad9d241..67965ca 100644 --- a/packages/adapters/src/wakeup.postgres.test.ts +++ b/packages/adapters/src/wakeup.postgres.test.ts @@ -30,6 +30,7 @@ function handlers(overrides: Partial = {}): BackgroundJob "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), "media.run": vi.fn(async () => undefined), + "media.library.sync": vi.fn(async () => undefined), ...overrides, }; } diff --git a/packages/adapters/src/wakeup.test.ts b/packages/adapters/src/wakeup.test.ts index 691f17a..14c57d9 100644 --- a/packages/adapters/src/wakeup.test.ts +++ b/packages/adapters/src/wakeup.test.ts @@ -15,6 +15,7 @@ function handlers(): BackgroundJobHandlers { "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), "media.run": vi.fn(async () => undefined), + "media.library.sync": vi.fn(async () => undefined), }; } diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 9f1ea52..5f265fe 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -17,6 +17,7 @@ export * from "./mcp.js"; export * from "./media.js"; export * from "./media-generation.js"; export * from "./media-jobs.js"; +export * from "./media-library.js"; export * from "./media-settings.js"; export * from "./media-transcription.js"; export * from "./media-video.js"; diff --git a/packages/contracts/src/media-library.ts b/packages/contracts/src/media-library.ts new file mode 100644 index 0000000..b03e745 --- /dev/null +++ b/packages/contracts/src/media-library.ts @@ -0,0 +1,39 @@ +import { z } from "zod"; + +/** Kinds a library folder indexes; everything else in the folder is ignored. */ +export const MediaLibraryKindSchema = z.enum(["photo", "video"]); +export type MediaLibraryKind = z.infer; + +export const MediaLibrarySourceStatusSchema = z.enum(["idle", "syncing", "failed"]); +export type MediaLibrarySourceStatus = z.infer; + +export const MediaLibrarySourceSchema = z.object({ + id: z.string(), + path: z.string(), + status: MediaLibrarySourceStatusSchema, + error: z.string().nullable(), + assetCount: z.number().int().nonnegative(), + lastSyncedAt: z.string().nullable(), +}); +export type MediaLibrarySource = z.infer; + +export const MediaLibrarySourceAddInputSchema = z.object({ + path: z.string().trim().min(1).max(4096), +}); +export type MediaLibrarySourceAddInput = z.infer; + +/** Arguments of the `search_media` agent tool. */ +export const MediaLibrarySearchInputSchema = z.object({ + query: z.string().trim().max(200).optional(), + kind: MediaLibraryKindSchema.optional(), + taken_after: z.string().datetime({ offset: true }).optional(), + taken_before: z.string().datetime({ offset: true }).optional(), + limit: z.number().int().min(1).max(50).optional(), +}); +export type MediaLibrarySearchInput = z.infer; + +/** Arguments of the `import_media` agent tool. */ +export const MediaLibraryImportInputSchema = z.object({ + asset_id: z.string().min(1).max(64), +}); +export type MediaLibraryImportInput = z.infer; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index db0cb0e..8b1ad34 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -108,6 +108,7 @@ import { MediaJobStartInputSchema, MediaJobStartOutputSchema, } from "./media-jobs.js"; +import { MediaLibrarySourceAddInputSchema, MediaLibrarySourceSchema } from "./media-library.js"; import { MediaSettingsUpdateSchema, MediaSettingsViewSchema } from "./media-settings.js"; import { ModelTierMappingSchema, SetModelTierInput } from "./model-tiers.js"; import { @@ -799,6 +800,13 @@ export const appContract = { get: oc.input(z.object({ jobId: Id })).output(MediaJobSnapshotSchema), cancel: oc.input(z.object({ jobId: Id })).output(MediaJobSnapshotSchema), }, + /** Server folders the user's bots can search and use; deployment owner only. */ + library: { + list: oc.output(z.array(MediaLibrarySourceSchema)), + add: oc.input(MediaLibrarySourceAddInputSchema).output(MediaLibrarySourceSchema), + remove: oc.input(z.object({ sourceId: Id })).output(z.object({ ok: z.literal(true) })), + sync: oc.input(z.object({ sourceId: Id })).output(MediaLibrarySourceSchema), + }, }, connections: { catalog: oc diff --git a/packages/db/prisma/migrations/20261005090000_media_library/migration.sql b/packages/db/prisma/migrations/20261005090000_media_library/migration.sql new file mode 100644 index 0000000..620e1b3 --- /dev/null +++ b/packages/db/prisma/migrations/20261005090000_media_library/migration.sql @@ -0,0 +1,52 @@ +-- Folders on the server whose photos and videos a user's bots can search and use, and their index. +-- CreateTable +CREATE TABLE "media_library_sources" ( + "id" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "path" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'idle', + "error" TEXT, + "assetCount" INTEGER NOT NULL DEFAULT 0, + "lastSyncedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_library_sources_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_library_assets" ( + "id" TEXT NOT NULL, + "sourceId" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "relativePath" TEXT NOT NULL, + "name" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "mimeType" TEXT NOT NULL, + "size" BIGINT NOT NULL, + "modifiedAt" TIMESTAMP(3) NOT NULL, + "takenAt" TIMESTAMP(3) NOT NULL, + "width" INTEGER, + "height" INTEGER, + "durationMs" INTEGER, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_library_assets_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "media_library_sources_userId_path_key" ON "media_library_sources"("userId", "path"); + +-- CreateIndex +CREATE INDEX "media_library_assets_userId_takenAt_idx" ON "media_library_assets"("userId", "takenAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "media_library_assets_sourceId_relativePath_key" ON "media_library_assets"("sourceId", "relativePath"); + +-- AddForeignKey +ALTER TABLE "media_library_sources" ADD CONSTRAINT "media_library_sources_userId_fkey" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_library_assets" ADD CONSTRAINT "media_library_assets_sourceId_fkey" FOREIGN KEY ("sourceId") REFERENCES "media_library_sources"("id") ON DELETE CASCADE ON UPDATE CASCADE; + diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 2a70125..2ba907b 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -54,6 +54,7 @@ model User { checkIns CheckIn[] dailyBriefs DailyBrief[] pushSubscriptions PushSubscription[] + mediaLibrarySources MediaLibrarySource[] notificationSettings UserNotificationSettings? @@unique([email]) @@ -1189,6 +1190,53 @@ model MediaJob { @@map("media_jobs") } +/// A folder on the server whose photos and videos the user's bots can search and use. +model MediaLibrarySource { + id String @id @default(cuid()) + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + /// Absolute folder path on the server the API and worker run on. + path String + /// idle, syncing or failed. + status String @default("idle") + error String? + assetCount Int @default(0) + lastSyncedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + assets MediaLibraryAsset[] + + @@unique([userId, path]) + @@map("media_library_sources") +} + +/// One photo or video in a library folder. Files stay where they are; this is the index. +model MediaLibraryAsset { + id String @id @default(cuid()) + sourceId String + source MediaLibrarySource @relation(fields: [sourceId], references: [id], onDelete: Cascade) + userId String + /// POSIX path below the source folder. + relativePath String + name String + /// photo or video. + kind String + mimeType String + size BigInt + modifiedAt DateTime + /// Capture time from the file's metadata, else its modification time. + takenAt DateTime + width Int? + height Int? + durationMs Int? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([sourceId, relativePath]) + @@index([userId, takenAt]) + @@map("media_library_assets") +} + model UsageRecord { id String @id @default(cuid()) spaceId String diff --git a/packages/testkit/src/cli/harness.ts b/packages/testkit/src/cli/harness.ts index cc051fe..13ab1dd 100644 --- a/packages/testkit/src/cli/harness.ts +++ b/packages/testkit/src/cli/harness.ts @@ -109,6 +109,7 @@ async function main() { "packages/testkit/src/bot-secrets.test.ts", "packages/testkit/src/delegation.postgres.test.ts", "packages/testkit/src/media.postgres.test.ts", + "packages/testkit/src/media-library.postgres.test.ts", "packages/testkit/src/user-profile.postgres.test.ts", "packages/testkit/src/onboarding-intake.postgres.test.ts", "packages/testkit/src/onboarding-discovery.postgres.test.ts", diff --git a/packages/testkit/src/media-library.postgres.test.ts b/packages/testkit/src/media-library.postgres.test.ts new file mode 100644 index 0000000..bf99963 --- /dev/null +++ b/packages/testkit/src/media-library.postgres.test.ts @@ -0,0 +1,204 @@ +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + ComposioEmulator, + importMediaLibraryAsset, + LocalArtifactStore, + MediaLibraryError, + searchMediaLibrary, +} from "@rakazo/adapters"; +import type { MediaLibrarySource } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { createApp } from "../../../apps/api/src/app.ts"; +import { sessionCookieHeader } from "./index.js"; + +type App = { request: (input: string, init?: RequestInit) => Response | Promise }; +type AppHandles = Awaited>; + +process.env.WAKEUP_DRIVER = "memory"; +process.env.SANDBOX_PROVIDER = "fake"; +process.env.AGENT_RUNTIME = "scripted"; + +const hasDb = process.env.VERIFY_DATABASE === "1" && Boolean(process.env.DATABASE_URL); +const describeWithDatabase = hasDb ? describe.sequential : describe.skip; + +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]); + +describeWithDatabase("media library (PostgreSQL)", () => { + let handles: AppHandles; + let app: App; + const stamp = `${Date.now()}-${Math.random().toString(36).slice(2)}`; + const dataDir = mkdtempSync(path.join(tmpdir(), "rakazo-library-data-")); + const library = mkdtempSync(path.join(tmpdir(), "rakazo-library-")); + const outside = mkdtempSync(path.join(tmpdir(), "rakazo-library-outside-")); + + beforeAll(async () => { + const { createApp } = await import("../../../apps/api/src/app.ts"); + handles = await createApp({ + databaseUrl: process.env.DATABASE_URL!, + dataDir, + sandboxProvider: "fake", + agentRuntime: "scripted", + wakeupDriver: "memory", + signupsEnabled: "true", + composio: new ComposioEmulator(), + }); + app = handles.app; + + mkdirSync(path.join(library, "Italy 2024")); + mkdirSync(path.join(library, ".thumbnails")); + writeFileSync(path.join(library, "Italy 2024", "beach.png"), PNG); + writeFileSync(path.join(library, "Italy 2024", "notes.txt"), "not media"); + writeFileSync(path.join(library, "birthday.png"), PNG); + writeFileSync(path.join(library, ".thumbnails", "hidden.png"), PNG); + writeFileSync(path.join(outside, "secret.png"), PNG); + symlinkSync(path.join(outside, "secret.png"), path.join(library, "linked.png")); + utimesSync( + path.join(library, "Italy 2024", "beach.png"), + new Date("2024-07-01"), + new Date("2024-07-01"), + ); + utimesSync(path.join(library, "birthday.png"), new Date("2025-03-01"), new Date("2025-03-01")); + }); + + afterAll(async () => { + await handles?.stop(); + for (const dir of [dataDir, library, outside]) rmSync(dir, { recursive: true, force: true }); + }); + + async function user(label: string) { + const cookie = await signup(app, `${label}-${stamp}@rakazo.test`, label); + const me = await rpc<{ userId: string; spaceId: string }>(app, cookie, "me"); + return { cookie, me }; + } + + async function waitForSync(cookie: string, sourceId: string) { + const start = Date.now(); + while (Date.now() - start < 20_000) { + const sources = await rpc(app, cookie, "media/library/list"); + const source = sources.find((entry) => entry.id === sourceId); + if (source && source.status !== "syncing") return source; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`library source ${sourceId} never finished syncing`); + } + + it("indexes, searches and imports only the owner's own library files", async () => { + const owner = await user("library-owner"); + const other = await user("library-other"); + await handles.prisma.deploymentSettings.upsert({ + where: { id: "default" }, + create: { id: "default", ownerUserId: owner.me.userId }, + update: { ownerUserId: owner.me.userId }, + }); + + expect((await raw(app, other.cookie, "media/library/list")).status).toBe(403); + expect((await raw(app, other.cookie, "media/library/add", { path: library })).status).toBe(403); + expect( + (await raw(app, owner.cookie, "media/library/add", { path: "relative/folder" })).status, + ).toBe(400); + expect( + (await raw(app, owner.cookie, "media/library/add", { path: path.join(library, "nope") })) + .status, + ).toBe(400); + + const added = await rpc(app, owner.cookie, "media/library/add", { + path: library, + }); + expect(added).toMatchObject({ path: library, status: "syncing" }); + // Adding the same folder again returns it instead of a duplicate. + expect( + (await rpc(app, owner.cookie, "media/library/add", { path: library })).id, + ).toBe(added.id); + const synced = await waitForSync(owner.cookie, added.id); + expect(synced).toMatchObject({ status: "idle", error: null, assetCount: 2 }); + expect(synced.lastSyncedAt).not.toBeNull(); + + const all = await searchMediaLibrary(handles.prisma, owner.me.userId, {}); + expect(all.total).toBe(2); + expect(all.items.map((item) => item.path)).toEqual(["birthday.png", "Italy 2024/beach.png"]); + const italy = await searchMediaLibrary(handles.prisma, owner.me.userId, { + query: "italy BEACH", + }); + expect(italy.items.map((item) => item.path)).toEqual(["Italy 2024/beach.png"]); + const before = await searchMediaLibrary(handles.prisma, owner.me.userId, { + taken_before: "2025-01-01T00:00:00Z", + }); + expect(before.items.map((item) => item.path)).toEqual(["Italy 2024/beach.png"]); + expect((await searchMediaLibrary(handles.prisma, other.me.userId, {})).total).toBe(0); + + const artifacts = new LocalArtifactStore(dataDir); + const assetId = italy.items[0]!.asset_id; + const imported = await importMediaLibraryAsset( + { prisma: handles.prisma, artifacts }, + { + spaceId: owner.me.spaceId, + userId: owner.me.userId, + botId: null, + groupId: null, + runId: null, + }, + assetId, + ); + expect(imported).toMatchObject({ name: "beach.png", mimeType: "image/png", size: PNG.length }); + await expect( + importMediaLibraryAsset( + { prisma: handles.prisma, artifacts }, + { + spaceId: other.me.spaceId, + userId: other.me.userId, + botId: null, + groupId: null, + runId: null, + }, + assetId, + ), + ).rejects.toBeInstanceOf(MediaLibraryError); + + // Another user can neither resync nor remove the owner's folder. + expect( + (await raw(app, other.cookie, "media/library/sync", { sourceId: added.id })).status, + ).toBe(403); + + // A deleted file leaves the index on the next sync. + rmSync(path.join(library, "birthday.png")); + await rpc(app, owner.cookie, "media/library/sync", { sourceId: added.id }); + expect(await waitForSync(owner.cookie, added.id)).toMatchObject({ assetCount: 1 }); + + await rpc(app, owner.cookie, "media/library/remove", { sourceId: added.id }); + expect(await rpc(app, owner.cookie, "media/library/list")).toEqual([]); + expect(await handles.prisma.mediaLibraryAsset.count({ where: { sourceId: added.id } })).toBe(0); + expect( + (await raw(app, owner.cookie, "media/library/sync", { sourceId: added.id })).status, + ).toBe(404); + }); +}); + +async function signup(app: App, email: string, name: string) { + const res = await app.request("/api/auth/sign-up/email", { + method: "POST", + headers: { "content-type": "application/json", origin: "http://127.0.0.1:5173" }, + body: JSON.stringify({ email, password: "password12", name }), + }); + if (res.status >= 400) throw new Error(`signup failed ${res.status}: ${await res.text()}`); + return sessionCookieHeader(res); +} + +function raw(app: App, cookie: string, proc: string, body: unknown = {}) { + return app.request(`/rpc/${proc}`, { + method: "POST", + headers: { "content-type": "application/json", cookie, origin: "http://127.0.0.1:5173" }, + body: JSON.stringify({ json: body }), + }); +} + +async function rpc(app: App, cookie: string, proc: string, body: unknown = {}): Promise { + const res = await raw(app, cookie, proc, body); + const text = await res.text(); + const parsed = JSON.parse(text) as { json?: T; error?: { message?: string } }; + if (res.status >= 400 || parsed.error) { + throw new Error(`${proc} ${res.status}: ${parsed.error?.message ?? text}`); + } + return parsed.json as T; +}