From 47988602e25cbfdc1678c263312e25e7e42d5c80 Mon Sep 17 00:00:00 2001 From: cynarAI <155274210+cynarAI@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:23:59 +0200 Subject: [PATCH] feat: let bots answer with cards the user taps (#137) Claude-Session: https://claude.ai/code/session_01UmSCnoCgXqBp2fU3vwz4o1 Co-authored-by: Claude --- apps/api/src/router.ts | 8 + apps/api/src/thread-cards.postgres.test.ts | 150 +++++++++++++ apps/api/src/thread-cards.ts | 79 +++++++ apps/mobile/app/thread.tsx | 42 ++++ apps/mobile/components/thread-card.tsx | 235 +++++++++++++++++++++ apps/mobile/lib/api.ts | 6 + apps/web/e2e/thread-card.spec.ts | 54 +++++ apps/web/src/components/ThreadCard.tsx | 182 ++++++++++++++++ apps/web/src/pages/Shell.tsx | 19 ++ apps/web/src/pages/shell/transcript.tsx | 21 +- packages/adapters/src/builtin-tools.ts | 53 +++++ packages/adapters/src/executor-tools.ts | 4 +- packages/adapters/src/executor.ts | 21 ++ packages/adapters/src/scripted-runtime.ts | 20 ++ packages/contracts/src/cards.ts | 112 ++++++++++ packages/contracts/src/events.ts | 6 +- packages/contracts/src/index.ts | 1 + packages/contracts/src/rpc.ts | 12 ++ packages/core/src/attachments.ts | 2 + packages/core/src/cards.test.ts | 106 ++++++++++ packages/core/src/cards.ts | 210 ++++++++++++++++++ packages/core/src/index.ts | 1 + packages/core/src/speech-text.ts | 5 +- 23 files changed, 1341 insertions(+), 8 deletions(-) create mode 100644 apps/api/src/thread-cards.postgres.test.ts create mode 100644 apps/api/src/thread-cards.ts create mode 100644 apps/mobile/components/thread-card.tsx create mode 100644 apps/web/e2e/thread-card.spec.ts create mode 100644 apps/web/src/components/ThreadCard.tsx create mode 100644 packages/contracts/src/cards.ts create mode 100644 packages/core/src/cards.test.ts create mode 100644 packages/core/src/cards.ts diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index 35d5fa6..0e77ae1 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -234,6 +234,7 @@ import { assertTeachingTakeoverAllowed, createTaughtSkillsService, } from "./taught-skills.js"; +import { actOnThreadCard } from "./thread-cards.js"; import { isPeerRun, loadAllMessages, @@ -1869,6 +1870,13 @@ export function createRouter(deps: RouterDeps) { } return { ok: true as const }; }), + act: authed.threads.act.handler(async ({ context, input }) => { + if (input.act.type !== "toggle" && (await modelSetup(deps, context.actor)).needsModel) { + throw new ORPCError("BAD_REQUEST", { message: "Connect a model to start a run." }); + } + const target = await resolveThreadTarget(deps.prisma, context.actor, input); + return actOnThreadCard(deps, context.actor, target, input); + }), stop: authed.threads.stop.handler(async ({ context, input }) => { const target = await resolveThreadTarget(deps.prisma, context.actor, input); await stopThreadRuns(deps, context.actor, target); diff --git a/apps/api/src/thread-cards.postgres.test.ts b/apps/api/src/thread-cards.postgres.test.ts new file mode 100644 index 0000000..0d18c34 --- /dev/null +++ b/apps/api/src/thread-cards.postgres.test.ts @@ -0,0 +1,150 @@ +import type { JobPublisher } from "@rakazo/adapter-kit"; +import type { Actor, CardBlock, MessageBlock } from "@rakazo/contracts"; +import { buildCardFromTool } from "@rakazo/core"; +import type { PrismaClient, ThreadEvents } from "@rakazo/db"; +import { createDb, provisionMessagingIdentity } from "@rakazo/db"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { actOnThreadCard } from "./thread-cards.js"; +import type { ThreadTarget } from "./thread-target.js"; +import { resolveThreadTarget } from "./thread-target.js"; + +const databaseUrl = process.env.DATABASE_URL; +const describePostgres = + process.env.VERIFY_DATABASE && databaseUrl ? describe.sequential : describe.skip; + +describePostgres("actOnThreadCard (PostgreSQL)", () => { + let prisma: PrismaClient; + let close: () => Promise; + let identity: { userId: string; spaceId: string; botId: string; threadId: string }; + let actor: Actor; + let target: ThreadTarget; + const notify = vi.fn(async () => undefined); + const enqueue = vi.fn(async () => undefined); + let deps: { prisma: PrismaClient; events: ThreadEvents; jobs: JobPublisher }; + let seq = 0; + let nonce = 0; + + beforeAll(async () => { + const db = createDb(databaseUrl!); + prisma = db.prisma; + close = async () => { + await db.prisma.$disconnect(); + await db.pool.end(); + }; + identity = await provisionMessagingIdentity( + prisma, + { provider: "sendblue", address: `+1556${Date.now().toString().slice(-7)}` }, + { signupsEnabled: undefined, signupAllowlist: undefined }, + ); + actor = { + userId: identity.userId, + spaceId: identity.spaceId, + email: "card@example.com", + isDeploymentOwner: false, + }; + deps = { + prisma, + events: { notify } as unknown as ThreadEvents, + jobs: { + enqueue, + cancel: vi.fn(async () => undefined), + close: vi.fn(async () => undefined), + } as unknown as JobPublisher, + }; + target = await resolveThreadTarget(prisma, actor, { botId: identity.botId }); + seq = + ( + await prisma.message.aggregate({ + where: { threadId: identity.threadId }, + _max: { seq: true }, + }) + )._max.seq ?? 0; + }); + + afterAll(async () => { + if (identity) { + await prisma.messagingIdentity.deleteMany({ where: { userId: identity.userId } }); + await prisma.organization.deleteMany({ where: { id: identity.spaceId } }); + await prisma.user.deleteMany({ where: { id: identity.userId } }); + } + await close?.(); + }); + + async function postCard(args: Parameters[0]) { + const built = buildCardFromTool(args, `card${++seq}`); + if (!("ok" in built)) throw new Error(built.error); + const message = await prisma.message.create({ + data: { + threadId: identity.threadId, + seq, + role: "bot", + botId: identity.botId, + blocks: [built.block], + }, + }); + return { messageId: message.id, cardId: built.block.id! }; + } + + async function card(messageId: string) { + const row = await prisma.message.findUniqueOrThrow({ where: { id: messageId } }); + return (row.blocks as MessageBlock[])[0] as CardBlock; + } + + const act = (input: Omit[3], "clientNonce">, again?: string) => + actOnThreadCard(deps, actor, target, { ...input, clientNonce: again ?? `nonce-${++nonce}` }); + + it("ticks a checklist item without starting a run", async () => { + const posted = await postCard({ title: "Shopping", checklist: ["Milk", "Eggs"] }); + await act({ + ...posted, + act: { type: "toggle", sectionId: "checklist", itemId: "i2", checked: true }, + }); + const updated = await card(posted.messageId); + expect(updated.sections?.[1]).toMatchObject({ + items: [{ id: "i1" }, { id: "i2", checked: true }], + }); + expect(await prisma.message.count({ where: { replyToMessageId: posted.messageId } })).toBe(0); + expect( + await prisma.event.count({ + where: { + threadId: identity.threadId, + type: "thread.message.updated", + payload: { path: ["messageId"], equals: posted.messageId }, + }, + }), + ).toBe(1); + }); + + it("answers a choice once and sends it to the bot as the user's reply", async () => { + const posted = await postCard({ title: "Dinner", options: ["Pasta", "Curry"] }); + const pick = { + ...posted, + act: { type: "select" as const, sectionId: "select", optionId: "o2" }, + }; + await act(pick, "pick-curry"); + // A retried tap is a no-op rather than an error. + await act(pick, "pick-curry"); + expect((await card(posted.messageId)).answer).toEqual({ text: "Curry" }); + const replies = await prisma.message.findMany({ + where: { replyToMessageId: posted.messageId }, + select: { role: true, blocks: true, runId: true }, + }); + expect(replies).toHaveLength(1); + expect(replies[0]).toMatchObject({ role: "user", blocks: [{ kind: "text", text: "Curry" }] }); + expect(replies[0]?.runId).toBeTruthy(); + await expect( + act({ ...posted, act: { type: "select", sectionId: "select", optionId: "o1" } }), + ).rejects.toThrow("already answered"); + }); + + it("keeps other threads' cards out of reach", async () => { + const posted = await postCard({ title: "Elsewhere", checklist: ["One"] }); + await expect( + act({ + ...posted, + cardId: "not-this-card", + act: { type: "toggle", sectionId: "checklist", itemId: "i1", checked: true }, + }), + ).rejects.toThrow(); + }); +}); diff --git a/apps/api/src/thread-cards.ts b/apps/api/src/thread-cards.ts new file mode 100644 index 0000000..b6025b8 --- /dev/null +++ b/apps/api/src/thread-cards.ts @@ -0,0 +1,79 @@ +import { ORPCError } from "@orpc/server"; +import type { JobPublisher } from "@rakazo/adapter-kit"; +import type { Actor, CardAct, CardBlock, MessageBlock } from "@rakazo/contracts"; +import { applyCardAct } from "@rakazo/core"; +import type { PrismaClient, ThreadEvents } from "@rakazo/db"; +import { appendEventInTransaction, IsolationError } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; +import type { ThreadTarget } from "./thread-target.js"; +import { groupDefaultBotId, sendThreadMessage } from "./thread-target.js"; + +type CardDeps = { prisma: PrismaClient; events: ThreadEvents; jobs: JobPublisher }; + +function isCard(block: MessageBlock, cardId: string): block is CardBlock { + return block.kind === "card" && block.id === cardId; +} + +/** + * Apply one tap on a bot's card. The card changes in place for everyone watching the thread; a + * choice or an action then goes to the card's bot as the user's reply, through the normal send. + */ +export async function actOnThreadCard( + deps: CardDeps, + actor: Actor, + target: ThreadTarget, + input: { messageId: string; cardId: string; act: CardAct; clientNonce: string }, +): Promise<{ ok: true }> { + // A retried answer already landed as the user's reply; the card was answered with it. + const replayed = await deps.prisma.message.findUnique({ + where: { threadId_clientNonce: { threadId: target.threadId, clientNonce: input.clientNonce } }, + select: { id: true }, + }); + if (replayed) return { ok: true }; + + const applied = await deps.prisma.$transaction(async (tx) => { + await tx.$queryRaw`SELECT id FROM threads WHERE id = ${target.threadId} FOR UPDATE`; + const message = await tx.message.findFirst({ + where: { id: input.messageId, threadId: target.threadId, role: "bot" }, + select: { id: true, blocks: true, botId: true }, + }); + if (!message) throw new IsolationError(); + const blocks = (Array.isArray(message.blocks) ? message.blocks : []) as MessageBlock[]; + const index = blocks.findIndex((block) => isCard(block, input.cardId)); + const card = blocks[index]; + if (card?.kind !== "card") throw new IsolationError(); + const result = applyCardAct(card, input.act); + if (!result.ok) throw new ORPCError("CONFLICT", { message: result.error }); + if (!result.changed) return { reply: result.reply, botId: message.botId, seq: null }; + const next = blocks.map((block, position) => (position === index ? result.block : block)); + await tx.message.update({ where: { id: message.id }, data: { blocks: next } }); + const botId = + message.botId ?? (target.kind === "bot" ? target.botId : groupDefaultBotId(target)); + if (!botId) throw new IsolationError(); + const event = await appendEventInTransaction(tx, { + spaceId: actor.spaceId, + threadId: target.threadId, + botId, + type: "thread.message.updated", + payload: { messageId: message.id, role: "bot", blocks: next }, + }); + return { reply: result.reply, botId: message.botId, seq: event.seq }; + }); + if (applied.seq !== null) { + await deps.events.notify(target.threadId, applied.seq).catch((error) => { + getLogger().error("thread card realtime notification", error); + }); + } + if (applied.reply) { + await sendThreadMessage(deps, actor, target, { + text: applied.reply, + replyToMessageId: input.messageId, + clientNonce: input.clientNonce, + // In a group the answer goes to the bot that asked, not to whoever leads. + ...(target.kind === "group" && applied.botId + ? { mentions: [{ kind: "bot" as const, id: applied.botId }] } + : {}), + }); + } + return { ok: true }; +} diff --git a/apps/mobile/app/thread.tsx b/apps/mobile/app/thread.tsx index 708609b..89172ea 100644 --- a/apps/mobile/app/thread.tsx +++ b/apps/mobile/app/thread.tsx @@ -1,6 +1,8 @@ import { ChatMarkdown } from "@rakazo/chat-ui/native"; import type { AgentSkillCatalogEntry, + CardAct, + CardBlock, Connection, ConnectionCatalogItem, MessageBlock, @@ -69,6 +71,7 @@ import { } from "../components/markdown-artifact-preview"; import { NativeSymbol } from "../components/native-symbol"; import { RoutineEventLine } from "../components/routine-event-line"; +import { ThreadCard } from "../components/thread-card"; import type { MobileBot, MobileGroup, @@ -1227,6 +1230,23 @@ function Thread() { [botId, groupId], ); + const actOnCard = useCallback( + async (message: MobileMessage, cardId: string, act: CardAct) => { + const targetBotId = botId; + const targetGroupId = groupId; + if (!targetBotId && !targetGroupId) return; + await rpc("threads/act", { + ...(targetGroupId ? { groupId: targetGroupId } : { botId: targetBotId! }), + messageId: message.id, + cardId, + act, + clientNonce: newClientNonce(), + }); + if (isCurrentTarget(targetBotId, targetGroupId)) await refresh(); + }, + [botId, groupId], + ); + const openBot = useCallback( (id: string, botName: string) => router.push({ pathname: "/thread", params: { botId: id, name: botName } }), @@ -1476,6 +1496,7 @@ function Thread() { } canAnswer={message.id === answerableAskMessageId} onAnswer={answerMessage} + onCardAct={actOnCard} onOpenBot={openBot} onPreviewMarkdown={setMarkdownPreview} actionProps={actionProps} @@ -2350,6 +2371,7 @@ const MessageBubble = memo(function MessageBubble({ replyPreview, canAnswer, onAnswer, + onCardAct, onOpenBot, onPreviewMarkdown, actionProps, @@ -2363,6 +2385,7 @@ const MessageBubble = memo(function MessageBubble({ replyPreview?: MobileMessage; canAnswer: boolean; onAnswer: (message: MobileMessage, answer: string) => Promise; + onCardAct: (message: MobileMessage, cardId: string, act: CardAct) => Promise; onOpenBot: (botId: string, name: string) => void; onPreviewMarkdown: (target: MarkdownArtifactPreviewTarget) => void; actionProps: MessageActionProps; @@ -2402,6 +2425,25 @@ const MessageBubble = memo(function MessageBubble({ ); } + const cards = message.blocks.filter( + (block): block is CardBlock => block.kind === "card" && Boolean(block.sections), + ); + if (cards.length) { + return ( + + {cards.map((card, index) => { + const cardId = card.id; + return ( + onCardAct(message, cardId, act) : undefined} + /> + ); + })} + + ); + } if (isRoutineEventMessage(message.blocks)) { return ( diff --git a/apps/mobile/components/thread-card.tsx b/apps/mobile/components/thread-card.tsx new file mode 100644 index 0000000..d26b8e0 --- /dev/null +++ b/apps/mobile/components/thread-card.tsx @@ -0,0 +1,235 @@ +import type { CardAct, CardBlock, CardSection } from "@rakazo/contracts"; +import { useState } from "react"; +import { Alert, Pressable, StyleSheet, Text, View } from "react-native"; +import { mobileTokens } from "../lib/appearance"; +import { useI18n } from "../lib/i18n"; +import { NativeSymbol } from "./native-symbol"; + +type Section = Extract; + +/** A bot's interactive card. Taps go to the server, which owns the card's state. */ +export function ThreadCard({ + block, + onAct, +}: { + block: CardBlock; + onAct?: (act: CardAct) => Promise; +}) { + const tokens = mobileTokens(); + const { t } = useI18n(); + const [pending, setPending] = useState(null); + const answered = Boolean(block.answer); + const busy = !onAct || pending !== null; + + async function act(next: CardAct) { + if (!onAct || pending) return; + setPending(next); + try { + await onAct(next); + } catch (error) { + Alert.alert( + t("Could not submit answer"), + error instanceof Error ? error.message : t("Please try again."), + ); + } finally { + setPending(null); + } + } + + const checkedNow = (list: Section<"checklist">, itemId: string, stored: boolean) => + pending?.type === "toggle" && pending.sectionId === list.id && pending.itemId === itemId + ? pending.checked + : stored; + const selectedNow = (select: Section<"select">, optionId: string) => + pending?.type === "select" && pending.sectionId === select.id && pending.optionId === optionId + ? (pending.selected ?? true) + : (select.selected?.includes(optionId) ?? false); + const select = block.sections?.find((part): part is Section<"select"> => part.type === "select"); + const nothingChosen = Boolean(select?.multiple && !select.selected?.length); + + return ( + + {block.sections?.map((part, index) => { + if (part.type === "header") { + return ( + + {part.title} + {part.subtitle ? ( + + {part.subtitle} + + ) : null} + {part.meta?.length ? ( + + {part.meta.join(" · ")} + + ) : null} + + ); + } + if (part.type === "facts") { + return ( + + {part.rows.map((row) => ( + + {row.label} + + {row.value} + + + ))} + + ); + } + if (part.type === "checklist") { + return ( + + {part.items.map((item) => { + const checked = checkedNow(part, item.id, Boolean(item.checked)); + return ( + + void act({ + type: "toggle", + sectionId: part.id, + itemId: item.id, + checked: !checked, + }) + } + style={styles.checkRow} + > + + + {item.label} + + + ); + })} + + ); + } + if (part.type === "select") { + return ( + + {part.options.map((option) => { + const chosen = selectedNow(part, option.id); + return ( + + void act({ + type: "select", + sectionId: part.id, + optionId: option.id, + ...(part.multiple ? { selected: !chosen } : {}), + }) + } + style={[ + styles.option, + part.multiple ? null : { alignSelf: "stretch" }, + { + borderColor: chosen ? tokens.primary : tokens.border, + backgroundColor: chosen ? tokens.primary : "transparent", + opacity: answered && !chosen ? 0.5 : 1, + }, + ]} + > + + {option.label} + + + ); + })} + + ); + } + if (answered) return null; + return ( + + {part.actions.map((action, position) => { + const primary = position === 0; + const disabled = busy || nothingChosen; + return ( + void act({ type: "action", actionId: action.id })} + style={[ + styles.option, + { + borderColor: primary ? tokens.primary : tokens.border, + backgroundColor: primary ? tokens.primary : "transparent", + opacity: disabled ? 0.5 : 1, + }, + ]} + > + + {action.label} + + + ); + })} + + ); + })} + + ); +} + +const styles = StyleSheet.create({ + card: { + width: "90%", + gap: 14, + borderRadius: 18, + borderWidth: 1, + paddingHorizontal: 16, + paddingVertical: 14, + }, + title: { fontSize: 16, fontWeight: "600" }, + body: { fontSize: 15.5, lineHeight: 22 }, + small: { fontSize: 13.5 }, + factRow: { flexDirection: "row", gap: 12 }, + factValue: { flex: 1 }, + checkRow: { flexDirection: "row", alignItems: "center", gap: 10, paddingVertical: 6 }, + chips: { flexDirection: "row", flexWrap: "wrap", gap: 6 }, + option: { borderRadius: 12, borderWidth: 1, paddingHorizontal: 14, paddingVertical: 10 }, +}); diff --git a/apps/mobile/lib/api.ts b/apps/mobile/lib/api.ts index 79c23a1..d7a1de3 100644 --- a/apps/mobile/lib/api.ts +++ b/apps/mobile/lib/api.ts @@ -902,6 +902,12 @@ export function blockText(message: MobileMessage) { } if (block.kind === "chart") return `[chart: ${block.name ?? "chart"}]`; if (block.kind === "event") return block.name; + if (block.kind === "card") { + const header = block.sections?.find((part) => part.type === "header"); + return header?.type === "header" + ? header.title + : (block.lines ?? []).map((line) => `${line.k}: ${line.v}`).join("\n"); + } if (block.kind === "image") return `[image: ${block.name ?? "attachment"}]`; if (block.kind === "file") { return `[file: ${block.name ?? "attachment"}${block.size ? ` (${block.size} bytes)` : ""}]`; diff --git a/apps/web/e2e/thread-card.spec.ts b/apps/web/e2e/thread-card.spec.ts new file mode 100644 index 0000000..57d68b7 --- /dev/null +++ b/apps/web/e2e/thread-card.spec.ts @@ -0,0 +1,54 @@ +import { expect, type Page, test } from "@playwright/test"; +import { captureScreenshot, completeOnboarding, openUserSettings, signup } from "./helpers"; + +async function useAppearance(page: Page, appearance: "light" | "dark") { + const settings = await openUserSettings(page); + await settings.getByTestId(`ui-appearance-${appearance}`).click(); + await expect(page.locator("html")).toHaveAttribute("data-theme", appearance); + await settings.getByRole("button", { name: "Close user settings" }).click(); + await expect(settings).toBeHidden(); +} + +test("a bot's card is ticked off and answered with a tap", async ({ page }, testInfo) => { + const stamp = Date.now(); + await signup(page, `thread-card-${stamp}@rakazo.test`, "password12", "Thread Card"); + await completeOnboarding(page); + + await page.getByPlaceholder(/Message/).fill("What should we cook tonight?"); + await page.keyboard.press("Enter"); + + const card = page.getByTestId("thread-card"); + await expect(card).toBeVisible({ timeout: 60_000 }); + await expect(card).toContainText("Dinner tonight"); + await expect(card).toContainText("30 min · 2 servings"); + + await card.getByRole("checkbox", { name: "Tomatoes" }).click(); + await expect(card.getByRole("checkbox", { name: "Tomatoes" })).toBeChecked(); + // The tick is stored, not just drawn. + await page.reload({ waitUntil: "domcontentloaded" }); + await expect( + page.getByTestId("thread-card").getByRole("checkbox", { name: "Tomatoes" }), + ).toBeChecked({ + timeout: 30_000, + }); + + await useAppearance(page, "light"); + await card.scrollIntoViewIfNeeded(); + await captureScreenshot(page, testInfo, "thread-card-light"); + await useAppearance(page, "dark"); + await card.scrollIntoViewIfNeeded(); + await captureScreenshot(page, testInfo, "thread-card-dark"); + + await card.getByRole("button", { name: "Curry" }).click(); + await expect(card).toHaveAttribute("data-answered", "true", { timeout: 15_000 }); + await expect(card.getByRole("button", { name: "Pasta" })).toBeDisabled(); + // The choice reaches the bot as the user's reply. + await expect( + page.getByTestId("message-user-bubble").filter({ hasText: /^\s*Curry\s*$/ }), + ).toHaveCount(1, { + timeout: 30_000, + }); + // A checklist stays usable after the choice. + await card.getByRole("checkbox", { name: "Basil" }).click(); + await expect(card.getByRole("checkbox", { name: "Basil" })).toBeChecked(); +}); diff --git a/apps/web/src/components/ThreadCard.tsx b/apps/web/src/components/ThreadCard.tsx new file mode 100644 index 0000000..bf75d4b --- /dev/null +++ b/apps/web/src/components/ThreadCard.tsx @@ -0,0 +1,182 @@ +import type { CardAct, CardBlock, CardSection } from "@rakazo/contracts"; +import { Button, Checkbox, cn } from "@rakazo/ui-web"; +import { useId, useState } from "react"; + +type Section = Extract; + +/** + * A bot's interactive card. The server owns the state: every tap goes through `onAct` and the + * card re-renders from the updated message, so only the tap in flight is held locally. + */ +export function ThreadCard({ + block, + onAct, + className, +}: { + block: CardBlock; + onAct?: (act: CardAct) => Promise; + className?: string; +}) { + const idPrefix = useId(); + const [pending, setPending] = useState(null); + const [error, setError] = useState(null); + const answered = Boolean(block.answer); + + async function act(next: CardAct) { + if (!onAct || pending) return; + setPending(next); + setError(null); + try { + await onAct(next); + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + } finally { + setPending(null); + } + } + + const checkedNow = (list: Section<"checklist">, itemId: string, stored: boolean) => + pending?.type === "toggle" && pending.sectionId === list.id && pending.itemId === itemId + ? pending.checked + : stored; + + const selectedNow = (select: Section<"select">, optionId: string) => { + const stored = select.selected?.includes(optionId) ?? false; + return pending?.type === "select" && + pending.sectionId === select.id && + pending.optionId === optionId + ? (pending.selected ?? true) + : stored; + }; + + const select = block.sections?.find((part): part is Section<"select"> => part.type === "select"); + const nothingChosen = Boolean(select?.multiple && !select.selected?.length); + + return ( +
+ {block.sections?.map((part, index) => { + if (part.type === "header") { + return ( +
+
{part.title}
+ {part.subtitle ? ( +
{part.subtitle}
+ ) : null} + {part.meta?.length ? ( +
{part.meta.join(" · ")}
+ ) : null} +
+ ); + } + if (part.type === "facts") { + return ( +
+ {part.rows.map((row) => ( +
+
{row.label}
+
{row.value}
+
+ ))} +
+ ); + } + if (part.type === "checklist") { + return ( +
    + {part.items.map((item) => { + const checked = checkedNow(part, item.id, Boolean(item.checked)); + return ( +
  • + +
    • + ); + })} +
+ ); + } + if (part.type === "select") { + return ( +
+ {part.options.map((option) => { + const chosen = selectedNow(part, option.id); + return ( + + ); + })} +
+ ); + } + if (answered) return null; + return ( +
+ {part.actions.map((action, position) => ( + + ))} +
+ ); + })} + {error ?

{error}

: null} +
+ ); +} diff --git a/apps/web/src/pages/Shell.tsx b/apps/web/src/pages/Shell.tsx index 4e61dbe..adb3e32 100644 --- a/apps/web/src/pages/Shell.tsx +++ b/apps/web/src/pages/Shell.tsx @@ -4,6 +4,7 @@ import { Trans, useLingui } from "@lingui/react/macro"; import type { Bot, BotSection, + CardAct, ComputerMode, ComputerReleaseReason, ComputerStatus, @@ -1961,6 +1962,23 @@ export function ShellPage() { await refreshThreadRef.current(botId); } }, []); + const actOnCard = useCallback(async (message: ThreadMessage, cardId: string, act: CardAct) => { + const botId = activeBotId.current; + const groupId = activeGroupId.current; + if (!botId && !groupId) return; + await rpc.threads.act({ + ...(groupId ? { groupId } : { botId: botId! }), + messageId: message.id, + cardId, + act, + clientNonce: newClientNonce(), + }); + if (groupId && activeGroupId.current === groupId) { + await refreshGroupThreadRef.current(groupId); + } else if (botId && activeBotId.current === botId) { + await refreshThreadRef.current(botId); + } + }, []); const reactToMessage = useCallback( async (message: ThreadMessage, reaction: MessageReaction) => { const botId = activeBotId.current; @@ -3312,6 +3330,7 @@ export function ShellPage() { onOpenBot={openBot} onOpenGroup={openGroup} onAnswer={answerMessage} + onCardAct={actOnCard} onReply={(message) => { setReplyTarget(message); setReplyQuote(null); diff --git a/apps/web/src/pages/shell/transcript.tsx b/apps/web/src/pages/shell/transcript.tsx index fa16dcd..57fe3a6 100644 --- a/apps/web/src/pages/shell/transcript.tsx +++ b/apps/web/src/pages/shell/transcript.tsx @@ -1,7 +1,7 @@ import { i18n } from "@lingui/core"; import { Trans, useLingui } from "@lingui/react/macro"; import { ChatMarkdown } from "@rakazo/chat-ui/web"; -import type { MessageReaction, ThreadMessage } from "@rakazo/contracts"; +import type { CardAct, MessageReaction, ThreadMessage } from "@rakazo/contracts"; import { canReactToThreadMessage, MESSAGE_REACTIONS } from "@rakazo/contracts"; import type { WorkspaceArtifact } from "@rakazo/core"; import { @@ -37,6 +37,7 @@ import { CloudAgentCard } from "../../components/CloudAgentCard"; import { MessageHoverMetadata } from "../../components/MessageHoverMetadata"; import { MediaCard } from "../../components/media/MediaCard"; import { MediaPlayer } from "../../components/media/MediaPlayer"; +import { ThreadCard } from "../../components/ThreadCard"; import { SkillDraftCard } from "../../components/teach/SkillDraftCard"; import type { ArtifactTarget } from "../../lib/artifact-open"; import { copyableMessageText } from "../../lib/message-text"; @@ -69,6 +70,7 @@ export const Transcript = memo(function Transcript({ onOpenBot, onOpenGroup, onAnswer, + onCardAct, onReply, onQuote, onReact, @@ -99,6 +101,7 @@ export const Transcript = memo(function Transcript({ onOpenBot: (botId: string) => void; onOpenGroup: (groupId: string) => void; onAnswer: (message: ThreadMessage, text: string) => Promise; + onCardAct?: (message: ThreadMessage, cardId: string, act: CardAct) => Promise; onReply: (message: ThreadMessage) => void; onQuote: (message: ThreadMessage, quote: string) => void; onReact: (message: ThreadMessage, reaction: MessageReaction) => Promise; @@ -446,6 +449,7 @@ export const Transcript = memo(function Transcript({ onOpenGroup={onOpenGroup} onOpenPeerMessages={onOpenPeerMessages} onAnswer={onAnswer} + onCardAct={onCardAct} encounterMode={encounterMode} speakerName={ peerReceipt @@ -728,6 +732,7 @@ const MessageView = memo(function MessageView({ canAnswer, message, onAnswer, + onCardAct, onOpenBot, onOpenGroup, onOpenPeerMessages, @@ -750,6 +755,7 @@ const MessageView = memo(function MessageView({ canAnswer: boolean; message: ThreadMessage; onAnswer: (message: ThreadMessage, text: string) => Promise; + onCardAct?: (message: ThreadMessage, cardId: string, act: CardAct) => Promise; onOpenBot: (botId: string) => void; onOpenGroup: (groupId: string) => void; onOpenPeerMessages: (peer: { peerBotId: string; peerBotName: string }) => void; @@ -1243,11 +1249,22 @@ const MessageView = memo(function MessageView({ ); } + if (block.kind === "card" && block.sections) { + const cardId = block.id; + return ( +
+ onCardAct(message, cardId, act) : undefined} + /> +
+ ); + } if (block.kind === "card") { return (
- {block.lines.map((line) => ( + {(block.lines ?? []).map((line) => (
✓ {line.k} diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index 6bd00d5..6d1c9e8 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -244,6 +244,59 @@ export const builtinAgentTools: ConnectorTool[] = [ required: ["question", "options"], }, }, + { + name: "show_card", + description: + "Show the user a card instead of a list in prose: a title, short facts, a checklist they can tick off, options they tap, or up to three buttons. Use it for everyday things worth keeping in view, such as a shopping list, a plan for the day or a choice between options. A tap on an option or button arrives as the user's next message. Keep text short; the card is the answer, so do not repeat it in your reply.", + inputSchema: { + type: "object", + properties: { + title: { type: "string", maxLength: 120 }, + subtitle: { type: "string", maxLength: 120 }, + meta: { + type: "array", + description: 'Up to three short facts beside the title, such as "30 min".', + items: { type: "string", maxLength: 80 }, + maxItems: 3, + }, + facts: { + type: "array", + items: { + type: "object", + properties: { + label: { type: "string", maxLength: 80 }, + value: { type: "string", maxLength: 200 }, + }, + required: ["label", "value"], + }, + maxItems: 8, + }, + checklist: { + type: "array", + items: { type: "string", minLength: 1, maxLength: 80 }, + maxItems: 30, + }, + options: { + type: "array", + items: { type: "string", minLength: 1, maxLength: 80 }, + minItems: 2, + maxItems: 6, + uniqueItems: true, + }, + multiple: { + type: "boolean", + description: "Let the user pick several options and send them with an action.", + }, + actions: { + type: "array", + items: { type: "string", minLength: 1, maxLength: 80 }, + maxItems: 3, + uniqueItems: true, + }, + }, + required: ["title"], + }, + }, { name: "message_user", description: diff --git a/packages/adapters/src/executor-tools.ts b/packages/adapters/src/executor-tools.ts index 09c99cf..fabe476 100644 --- a/packages/adapters/src/executor-tools.ts +++ b/packages/adapters/src/executor-tools.ts @@ -135,7 +135,9 @@ export function selectBuiltinToolsForRun(options: { (tool) => !(options.foreignAgentRun && USER_MODEL_TOOLS.has(tool.name)) && (!options.messagingChannelRun || - (!["remember", "save_memory", "recall_memory", "forget_memory"].includes(tool.name) && + (!["remember", "save_memory", "recall_memory", "forget_memory", "show_card"].includes( + tool.name, + ) && !USER_MODEL_TOOLS.has(tool.name) && !tool.name.startsWith("scratchpad_"))), ); diff --git a/packages/adapters/src/executor.ts b/packages/adapters/src/executor.ts index 1074430..ba957a1 100644 --- a/packages/adapters/src/executor.ts +++ b/packages/adapters/src/executor.ts @@ -38,6 +38,7 @@ import { MessageBlock as MessageBlockSchema, OPENAI_COMPATIBLE_PROVIDER_ID, } from "@rakazo/contracts"; +import type { ShowCardArgs } from "@rakazo/core"; import { type ActionApprovalRule, appendTextSegment, @@ -45,6 +46,7 @@ import { applyJudgeDecision, assertTransition, botMessageAllowsSilence, + buildCardFromTool, computerScreenSize, connectorKindFromToolName, containsSecret, @@ -3488,6 +3490,25 @@ export function createRunExecutor(deps: ExecutorDeps) { autoSpeak: updated.autoSpeak, }); } + if (name === "show_card") { + const built = buildCardFromTool( + JSON.parse(redactSecrets(JSON.stringify(args), runSecrets)) as ShowCardArgs, + randomUUID(), + ); + if (!("ok" in built)) return finish(built); + await flushProgress(); + await publishMidTurnNarration(); + await publishMessage( + deps, + run, + "bot", + [built.block], + undefined, + `card:${run.id}:${executionId}`, + ); + publishedMidTurnUserMessage = true; + return finish({ ok: true, cardId: built.block.id }); + } if (name === "message_user") { const rawMessage = redactSecrets(String(args.message ?? ""), runSecrets); const text = clampUserProgressMessage(rawMessage); diff --git a/packages/adapters/src/scripted-runtime.ts b/packages/adapters/src/scripted-runtime.ts index b0197ee..3c33339 100644 --- a/packages/adapters/src/scripted-runtime.ts +++ b/packages/adapters/src/scripted-runtime.ts @@ -275,6 +275,26 @@ export function inferScript( }, ]; } + // Everyday answers come as a card the user taps, not a list in prose. + if (lower.includes("what should we cook tonight")) { + return [ + { + assistant: "", + toolCalls: [ + { + name: "show_card", + args: { + title: "Dinner tonight", + meta: ["30 min", "2 servings"], + checklist: ["Spaghetti", "Tomatoes", "Basil"], + options: ["Pasta", "Curry", "Salad"], + }, + }, + ], + complete: true, + }, + ]; + } if (lower.includes("delegate to the bot named")) { const name = namedBot(prompt) ?? "Researcher"; const brief = diff --git a/packages/contracts/src/cards.ts b/packages/contracts/src/cards.ts new file mode 100644 index 0000000..2b12537 --- /dev/null +++ b/packages/contracts/src/cards.ts @@ -0,0 +1,112 @@ +import * as z from "zod"; + +/** + * Interactive cards: a fixed catalog of sections a bot fills with content. The model never + * chooses layout or markup; the backend validates every card and applies every tap. + */ + +export const CARD_LIMITS = { + facts: 8, + checklistItems: 30, + selectOptions: 6, + actions: 3, + label: 80, + title: 120, + meta: 3, +} as const; + +const Label = z.string().trim().min(1).max(CARD_LIMITS.label); +const SectionId = z.string().min(1).max(40); + +export const CardHeaderSchema = z.object({ + type: z.literal("header"), + title: z.string().trim().min(1).max(CARD_LIMITS.title), + subtitle: z.string().trim().max(CARD_LIMITS.title).optional(), + /** Short facts beside the title, such as "30 min" or "4 servings". */ + meta: z.array(Label).max(CARD_LIMITS.meta).optional(), +}); + +export const CardFactsSchema = z.object({ + type: z.literal("facts"), + rows: z + .array(z.object({ label: Label, value: z.string().trim().min(1).max(200) })) + .min(1) + .max(CARD_LIMITS.facts), +}); + +export const CardChecklistSchema = z.object({ + type: z.literal("checklist"), + id: SectionId, + items: z + .array(z.object({ id: SectionId, label: Label, checked: z.boolean().optional() })) + .min(1) + .max(CARD_LIMITS.checklistItems), +}); + +export const CardSelectSchema = z.object({ + type: z.literal("select"), + id: SectionId, + /** Several options before sending; otherwise one tap answers. */ + multiple: z.boolean().optional(), + options: z + .array(z.object({ id: SectionId, label: Label })) + .min(2) + .max(CARD_LIMITS.selectOptions), + selected: z.array(SectionId).optional(), +}); + +/** "reply" sends the label, with any selection, to the bot as the user's answer. */ +export const CardActionSchema = z.object({ + id: SectionId, + label: Label, + kind: z.literal("reply"), +}); + +export const CardActionsSchema = z.object({ + type: z.literal("actions"), + actions: z.array(CardActionSchema).min(1).max(CARD_LIMITS.actions), +}); + +export const CardSectionSchema = z.discriminatedUnion("type", [ + CardHeaderSchema, + CardFactsSchema, + CardChecklistSchema, + CardSelectSchema, + CardActionsSchema, +]); +export type CardSection = z.infer; + +/** Set once the user answered the card; the card then stays as it was answered. */ +export const CardAnswerSchema = z.object({ text: z.string(), actionId: SectionId.optional() }); + +/** + * One card per block. `lines` is the older read-only shape, kept so stored messages still parse; + * new cards carry `id` and `sections`. + */ +export const CardBlockSchema = z.object({ + kind: z.literal("card"), + id: SectionId.optional(), + lines: z.array(z.object({ k: z.string(), v: z.string() })).optional(), + sections: z.array(CardSectionSchema).max(5).optional(), + answer: CardAnswerSchema.optional(), +}); +export type CardBlock = z.infer; + +/** What a tap on a card asks the backend to do. Toggles carry the new state, so a retry is a no-op. */ +export const CardActSchema = z.discriminatedUnion("type", [ + z.object({ + type: z.literal("toggle"), + sectionId: SectionId, + itemId: SectionId, + checked: z.boolean(), + }), + z.object({ + type: z.literal("select"), + sectionId: SectionId, + optionId: SectionId, + /** Multiple choice only: whether the option is now chosen. */ + selected: z.boolean().optional(), + }), + z.object({ type: z.literal("action"), actionId: SectionId }), +]); +export type CardAct = z.infer; diff --git a/packages/contracts/src/events.ts b/packages/contracts/src/events.ts index f82b0ba..517ef60 100644 --- a/packages/contracts/src/events.ts +++ b/packages/contracts/src/events.ts @@ -1,5 +1,6 @@ import * as z from "zod"; import { BotSecretDestination } from "./bot-secrets.js"; +import { CardBlockSchema } from "./cards.js"; import { Id } from "./ids.js"; import { McpTransportSchema } from "./mcp.js"; import { AudioBlockSchema, MediaJobBlockSchema, VideoBlockSchema } from "./media.js"; @@ -193,10 +194,7 @@ export type RoutineEventBlock = z.infer; export const MessageBlock = z.discriminatedUnion("kind", [ z.object({ kind: z.literal("text"), text: z.string() }), - z.object({ - kind: z.literal("card"), - lines: z.array(z.object({ k: z.string(), v: z.string() })), - }), + CardBlockSchema, AskBlockSchema, ChoiceBlockSchema, IntakeNoteBlockSchema, diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 0124725..7b3b427 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -1,6 +1,7 @@ export * from "./ai-consent.js"; export * from "./attachments.js"; export * from "./bot-secrets.js"; +export * from "./cards.js"; export * from "./clock-time.js"; export * from "./coach.js"; export * from "./computer-pointer.js"; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index cf14c28..fff2504 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -2,6 +2,7 @@ import { eventIterator, oc } from "@orpc/contract"; import * as z from "zod"; import { AiConsentQuerySchema, AiConsentStatusSchema } from "./ai-consent.js"; import { ATTACHMENT_MAX_BASE64_LENGTH, ATTACHMENT_MAX_COUNT } from "./attachments.js"; +import { CardActSchema } from "./cards.js"; import { CheckInSchema, COACH_TITLE_MAX_LENGTH, @@ -452,6 +453,17 @@ export const appContract = { }), ) .output(z.object({ ok: z.literal(true) })), + /** A tap on an interactive card: the backend applies it and may answer the bot. */ + act: oc + .input( + threadTarget.safeExtend({ + messageId: Id, + cardId: z.string().min(1).max(40), + act: CardActSchema, + clientNonce: z.string().min(1).max(200), + }), + ) + .output(z.object({ ok: z.literal(true) })), markRead: oc.input(threadTarget).output(z.object({ ok: z.literal(true) })), markUnread: oc.input(threadTarget).output(z.object({ ok: z.literal(true) })), }, diff --git a/packages/core/src/attachments.ts b/packages/core/src/attachments.ts index 997b4ba..78fcde3 100644 --- a/packages/core/src/attachments.ts +++ b/packages/core/src/attachments.ts @@ -6,6 +6,7 @@ import { isAllowedAttachmentMimeType, isAttachmentImageMimeType, } from "@rakazo/contracts"; +import { cardHistoryText } from "./cards.js"; import { routineEventHistoryText } from "./routine-events.js"; export class AttachmentValidationError extends Error { @@ -139,6 +140,7 @@ export function blocksToAgentHistoryText(blocks: MessageBlock[]): string { return `[task for ${block.assigneeBotName}: ${block.title} (${block.status})] ${block.text}`; } if (block.kind === "event") return routineEventHistoryText(block); + if (block.kind === "card") return cardHistoryText(block); if (block.kind === "handoff") { return `[handoff ${block.fromBotId} -> ${block.toBotId}] ${block.text}`; } diff --git a/packages/core/src/cards.test.ts b/packages/core/src/cards.test.ts new file mode 100644 index 0000000..6271cb6 --- /dev/null +++ b/packages/core/src/cards.test.ts @@ -0,0 +1,106 @@ +import type { CardBlock } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { applyCardAct, buildCardFromTool, cardHistoryText } from "./cards.js"; + +function card(args: Parameters[0]): CardBlock { + const built = buildCardFromTool(args, "c1"); + if (!("ok" in built)) throw new Error(built.error); + return built.block; +} + +describe("buildCardFromTool", () => { + it("builds sections in a fixed order with stable ids", () => { + const block = card({ + actions: ["Send"], + options: ["Pasta", "Curry"], + checklist: ["Milk", "Eggs"], + facts: [{ label: "Time", value: "30 min" }], + title: "Dinner", + meta: ["4 servings"], + }); + expect(block.sections?.map((part) => part.type)).toEqual([ + "header", + "facts", + "checklist", + "select", + "actions", + ]); + expect(block.sections?.[2]).toMatchObject({ items: [{ id: "i1" }, { id: "i2" }] }); + expect(block.sections?.[3]).toMatchObject({ options: [{ id: "o1" }, { id: "o2" }] }); + }); + + it("rejects cards that say nothing or cannot be answered", () => { + expect(buildCardFromTool({ title: "Only a title" }, "c1")).toHaveProperty("error"); + expect(buildCardFromTool({ title: "", checklist: ["a"] }, "c1")).toHaveProperty("error"); + expect(buildCardFromTool({ title: "One", options: ["a"] }, "c1")).toHaveProperty("error"); + expect( + buildCardFromTool({ title: "Many", options: ["a", "b"], multiple: true }, "c1"), + ).toHaveProperty("error"); + expect(buildCardFromTool({ title: "Long", checklist: ["x".repeat(81)] }, "c1")).toHaveProperty( + "error", + ); + }); +}); + +describe("applyCardAct", () => { + it("sets checklist state idempotently, even after the card was answered", () => { + const block = { ...card({ title: "List", checklist: ["Milk"] }), answer: { text: "Done" } }; + const first = applyCardAct(block, { + type: "toggle", + sectionId: "checklist", + itemId: "i1", + checked: true, + }); + expect(first).toMatchObject({ ok: true, changed: true }); + if (!first.ok) return; + const again = applyCardAct(first.block, { + type: "toggle", + sectionId: "checklist", + itemId: "i1", + checked: true, + }); + expect(again).toMatchObject({ ok: true, changed: false }); + expect(cardHistoryText(first.block)).toContain("checklist 1/1 done"); + }); + + it("answers a single choice with the option's label, once", () => { + const block = card({ title: "Dinner", options: ["Pasta", "Curry"] }); + const picked = applyCardAct(block, { type: "select", sectionId: "select", optionId: "o2" }); + expect(picked).toMatchObject({ + ok: true, + reply: "Curry", + block: { answer: { text: "Curry" } }, + }); + if (!picked.ok) return; + expect( + applyCardAct(picked.block, { type: "select", sectionId: "select", optionId: "o1" }), + ).toMatchObject({ ok: false }); + }); + + it("collects a multiple choice in card order and sends it with an action", () => { + let block = card({ + title: "Toppings", + options: ["A", "B", "C"], + multiple: true, + actions: ["Send"], + }); + expect(applyCardAct(block, { type: "action", actionId: "a1" })).toMatchObject({ ok: false }); + for (const optionId of ["o3", "o1"]) { + const next = applyCardAct(block, { + type: "select", + sectionId: "select", + optionId, + selected: true, + }); + if (!next.ok) throw new Error(next.error); + block = next.block; + } + const sent = applyCardAct(block, { type: "action", actionId: "a1" }); + expect(sent).toMatchObject({ ok: true, reply: "Send: A, C" }); + expect(cardHistoryText(sent.ok ? sent.block : block)).toContain("answered: Send: A, C"); + }); + + it("reads legacy cards as their lines", () => { + expect(cardHistoryText({ kind: "card", lines: [{ k: "Mail", v: "sent" }] })).toBe("Mail: sent"); + }); +}); diff --git a/packages/core/src/cards.ts b/packages/core/src/cards.ts new file mode 100644 index 0000000..0271e6c --- /dev/null +++ b/packages/core/src/cards.ts @@ -0,0 +1,210 @@ +import type { CardAct, CardBlock, CardSection } from "@rakazo/contracts"; +import { CardBlockSchema } from "@rakazo/contracts"; + +/** What `show_card` accepts: plain content in a fixed order, never layout. */ +export interface ShowCardArgs { + title?: unknown; + subtitle?: unknown; + meta?: unknown; + facts?: unknown; + checklist?: unknown; + options?: unknown; + multiple?: unknown; + actions?: unknown; +} + +const strings = (value: unknown): string[] => + Array.isArray(value) ? value.map((item) => String(item ?? "").trim()).filter(Boolean) : []; + +const text = (value: unknown) => (typeof value === "string" ? value.trim() : ""); + +/** Build a validated card from tool arguments; the error is phrased for the model. */ +export function buildCardFromTool( + args: ShowCardArgs, + id: string, +): { ok: true; block: CardBlock } | { error: string } { + const title = text(args.title); + if (!title) return { error: "title is required." }; + const facts = Array.isArray(args.facts) + ? args.facts + .map((row) => ({ + label: text((row as { label?: unknown })?.label), + value: text((row as { value?: unknown })?.value), + })) + .filter((row) => row.label && row.value) + : []; + const checklist = strings(args.checklist); + const options = strings(args.options); + const actions = strings(args.actions); + const multiple = args.multiple === true; + if (!facts.length && !checklist.length && !options.length && !actions.length) { + return { + error: "Add facts, checklist, options or actions; a card with only a title says nothing.", + }; + } + if (options.length === 1) return { error: "options needs at least two choices." }; + if (multiple && options.length && !actions.length) { + return { error: 'A multiple choice needs an action to send it, such as "Send".' }; + } + const subtitle = text(args.subtitle); + const meta = strings(args.meta); + const sections: CardSection[] = [ + { + type: "header", + title, + ...(subtitle ? { subtitle } : {}), + ...(meta.length ? { meta } : {}), + }, + ]; + if (facts.length) sections.push({ type: "facts", rows: facts }); + if (checklist.length) { + sections.push({ + type: "checklist", + id: "checklist", + items: checklist.map((label, index) => ({ id: `i${index + 1}`, label })), + }); + } + if (options.length) { + sections.push({ + type: "select", + id: "select", + ...(multiple ? { multiple: true } : {}), + options: options.map((label, index) => ({ id: `o${index + 1}`, label })), + }); + } + if (actions.length) { + sections.push({ + type: "actions", + actions: actions.map((label, index) => ({ id: `a${index + 1}`, label, kind: "reply" })), + }); + } + const parsed = CardBlockSchema.safeParse({ kind: "card", id, sections }); + if (!parsed.success) { + const issue = parsed.error.issues[0]; + return { error: `Card is not valid: ${issue?.path.join(".")} ${issue?.message}`.trim() }; + } + return { ok: true, block: parsed.data }; +} + +export type CardActResult = + | { ok: true; block: CardBlock; changed: boolean; reply?: string } + | { ok: false; error: string }; + +function section( + block: CardBlock, + type: T, + id?: string, +): Extract | undefined { + return block.sections?.find( + (candidate): candidate is Extract => + candidate.type === type && (id === undefined || ("id" in candidate && candidate.id === id)), + ); +} + +function replaceSection(block: CardBlock, next: CardSection, index: number): CardBlock { + const sections = [...(block.sections ?? [])]; + sections[index] = next; + return { ...block, sections }; +} + +/** + * Apply one tap. Checklists stay usable after the card is answered (a shopping list is still a + * shopping list); a choice or an action answers the card once. + */ +export function applyCardAct(block: CardBlock, act: CardAct): CardActResult { + const sections = block.sections ?? []; + if (act.type === "toggle") { + const index = sections.findIndex( + (candidate) => candidate.type === "checklist" && candidate.id === act.sectionId, + ); + const list = sections[index]; + if (!list || list.type !== "checklist") return { ok: false, error: "No such checklist." }; + if (!list.items.some((item) => item.id === act.itemId)) { + return { ok: false, error: "No such item." }; + } + const current = list.items.find((item) => item.id === act.itemId); + if (Boolean(current?.checked) === act.checked) return { ok: true, changed: false, block }; + const items = list.items.map((item) => + item.id === act.itemId ? { ...item, checked: act.checked } : item, + ); + return { ok: true, changed: true, block: replaceSection(block, { ...list, items }, index) }; + } + if (block.answer) return { ok: false, error: "This card was already answered." }; + if (act.type === "select") { + const index = sections.findIndex( + (candidate) => candidate.type === "select" && candidate.id === act.sectionId, + ); + const select = sections[index]; + if (!select || select.type !== "select") return { ok: false, error: "No such choice." }; + const option = select.options.find((candidate) => candidate.id === act.optionId); + if (!option) return { ok: false, error: "No such option." }; + if (select.multiple) { + const current = select.selected ?? []; + const on = act.selected ?? !current.includes(option.id); + if (current.includes(option.id) === on) return { ok: true, changed: false, block }; + // Keep the options' order so the reply reads the way the card does. + const selected = select.options + .map((candidate) => candidate.id) + .filter((id) => (id === option.id ? on : current.includes(id))); + return { + ok: true, + changed: true, + block: replaceSection(block, { ...select, selected }, index), + }; + } + const next = replaceSection(block, { ...select, selected: [option.id] }, index); + return { + ok: true, + changed: true, + reply: option.label, + block: { ...next, answer: { text: option.label } }, + }; + } + const actions = section(block, "actions"); + const action = actions?.actions.find((candidate) => candidate.id === act.actionId); + if (!action) return { ok: false, error: "No such action." }; + const select = section(block, "select"); + const chosen = select?.options + .filter((option) => select.selected?.includes(option.id)) + .map((option) => option.label); + if (select?.multiple && !chosen?.length) { + return { ok: false, error: "Choose at least one option first." }; + } + const reply = chosen?.length ? `${action.label}: ${chosen.join(", ")}` : action.label; + return { + ok: true, + changed: true, + reply, + block: { ...block, answer: { text: reply, actionId: action.id } }, + }; +} + +/** The card as a later turn reads it: content plus what the user has done with it. */ +export function cardHistoryText(block: CardBlock): string { + if (!block.sections) return (block.lines ?? []).map((line) => `${line.k}: ${line.v}`).join("\n"); + const parts: string[] = []; + for (const part of block.sections) { + if (part.type === "header") { + parts.push([part.title, part.subtitle, ...(part.meta ?? [])].filter(Boolean).join(" · ")); + } else if (part.type === "facts") { + parts.push(part.rows.map((row) => `${row.label}: ${row.value}`).join("; ")); + } else if (part.type === "checklist") { + const done = part.items.filter((item) => item.checked).length; + const open = part.items.filter((item) => !item.checked).map((item) => item.label); + parts.push( + `checklist ${done}/${part.items.length} done${open.length ? `, open: ${open.join(", ")}` : ""}`, + ); + } else if (part.type === "select") { + const chosen = part.options.filter((option) => part.selected?.includes(option.id)); + parts.push( + `options: ${part.options.map((option) => option.label).join(" | ")}${ + chosen.length ? ` (selected: ${chosen.map((option) => option.label).join(", ")})` : "" + }`, + ); + } else { + parts.push(`actions: ${part.actions.map((action) => action.label).join(" | ")}`); + } + } + if (block.answer) parts.push(`answered: ${block.answer.text}`); + return `[card: ${parts.join(" — ")}]`; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 67dcab4..5ad8ea1 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -9,6 +9,7 @@ export * from "./avatar-shape.js"; export * from "./bot-messages.js"; export * from "./bot-reply-sanitize.js"; export * from "./bot-sections.js"; +export * from "./cards.js"; export * from "./cloud-agent.js"; export * from "./coach.js"; export * from "./compose-update.js"; diff --git a/packages/core/src/speech-text.ts b/packages/core/src/speech-text.ts index 904696c..e531b91 100644 --- a/packages/core/src/speech-text.ts +++ b/packages/core/src/speech-text.ts @@ -249,7 +249,10 @@ export function speechFromBlocks(blocks: MessageBlock[]): string { return block.result || `${block.name} finished`; } if (block.kind === "card") { - return block.lines.map((line) => `${line.k}: ${line.v}`).join(". "); + const header = block.sections?.find((part) => part.type === "header"); + if (header?.type === "header") + return [header.title, header.subtitle].filter(Boolean).join(". "); + return (block.lines ?? []).map((line) => `${line.k}: ${line.v}`).join(". "); } return ""; })