feat(web,core,adapters): unify right sidebar styling, hover resize handles, voice & search improvements
This commit is contained in:
cynarAI
parent
75423579b7
commit
35b7291660
@ -1,6 +1,7 @@
|
||||
import type { Actor, MessageBlock, SearchHit } from "@rakazo/contracts";
|
||||
import { extractLinksFromText, matchesSearchQuery, snippetAroundMatch } from "@rakazo/core";
|
||||
import { Prisma, type PrismaClient } from "@rakazo/db";
|
||||
import type { PrismaClient } from "@rakazo/db";
|
||||
import { Prisma } from "@rakazo/db";
|
||||
|
||||
const SEARCH_LIMIT = 25;
|
||||
/** Cap name matches so content hits (messages/files/links/routines) keep most of the budget. */
|
||||
@ -64,19 +65,65 @@ export async function querySpaceSearch(
|
||||
bucket.push(hit);
|
||||
}
|
||||
|
||||
const bots = await prisma.bot.findMany({
|
||||
where: {
|
||||
spaceId: actor.spaceId,
|
||||
userId: actor.userId,
|
||||
archivedAt: null,
|
||||
OR: [
|
||||
{ name: { contains: query, mode: "insensitive" } },
|
||||
{ title: { contains: query, mode: "insensitive" } },
|
||||
{ description: { contains: query, mode: "insensitive" } },
|
||||
],
|
||||
},
|
||||
take: CONVERSATION_HIT_LIMIT,
|
||||
});
|
||||
const [bots, groups, artifacts, groupArtifacts, routines] = await Promise.all([
|
||||
prisma.bot.findMany({
|
||||
where: {
|
||||
spaceId: actor.spaceId,
|
||||
userId: actor.userId,
|
||||
archivedAt: null,
|
||||
OR: [
|
||||
{ name: { contains: query, mode: "insensitive" } },
|
||||
{ title: { contains: query, mode: "insensitive" } },
|
||||
{ description: { contains: query, mode: "insensitive" } },
|
||||
],
|
||||
},
|
||||
take: CONVERSATION_HIT_LIMIT,
|
||||
}),
|
||||
prisma.chatGroup.findMany({
|
||||
where: {
|
||||
spaceId: actor.spaceId,
|
||||
userId: actor.userId,
|
||||
name: { contains: query, mode: "insensitive" },
|
||||
},
|
||||
take: CONVERSATION_HIT_LIMIT,
|
||||
}),
|
||||
prisma.artifact.findMany({
|
||||
where: {
|
||||
spaceId: actor.spaceId,
|
||||
userId: actor.userId,
|
||||
groupId: null,
|
||||
botId: { not: null },
|
||||
name: { contains: query, mode: "insensitive" },
|
||||
bot: { archivedAt: null },
|
||||
},
|
||||
include: { bot: { select: { name: true } } },
|
||||
take: SEARCH_LIMIT,
|
||||
}),
|
||||
prisma.artifact.findMany({
|
||||
where: {
|
||||
spaceId: actor.spaceId,
|
||||
userId: actor.userId,
|
||||
groupId: { not: null },
|
||||
name: { contains: query, mode: "insensitive" },
|
||||
},
|
||||
include: { group: { select: { name: true } } },
|
||||
take: SEARCH_LIMIT,
|
||||
}),
|
||||
prisma.routine.findMany({
|
||||
where: {
|
||||
spaceId: actor.spaceId,
|
||||
userId: actor.userId,
|
||||
OR: [
|
||||
{ name: { contains: query, mode: "insensitive" } },
|
||||
{ prompt: { contains: query, mode: "insensitive" } },
|
||||
],
|
||||
bot: { archivedAt: null },
|
||||
},
|
||||
include: { bot: { select: { name: true } } },
|
||||
take: SEARCH_LIMIT,
|
||||
}),
|
||||
]);
|
||||
|
||||
for (const bot of bots) {
|
||||
pushInto(
|
||||
conversationHits,
|
||||
@ -91,14 +138,6 @@ export async function querySpaceSearch(
|
||||
);
|
||||
}
|
||||
|
||||
const groups = await prisma.chatGroup.findMany({
|
||||
where: {
|
||||
spaceId: actor.spaceId,
|
||||
userId: actor.userId,
|
||||
name: { contains: query, mode: "insensitive" },
|
||||
},
|
||||
take: CONVERSATION_HIT_LIMIT,
|
||||
});
|
||||
for (const group of groups) {
|
||||
pushInto(
|
||||
conversationHits,
|
||||
@ -114,27 +153,72 @@ export async function querySpaceSearch(
|
||||
}
|
||||
|
||||
const contentBudget = Math.max(0, SEARCH_LIMIT - conversationHits.length);
|
||||
const pattern = `%${query}%`;
|
||||
|
||||
const [botArtifactMessages, groupArtifactMessages, messageRows, groupMessageRows] =
|
||||
await Promise.all([
|
||||
findArtifactMessages(
|
||||
prisma,
|
||||
actor,
|
||||
artifacts.flatMap((artifact) =>
|
||||
artifact.botId && artifact.bot ? [{ id: artifact.id, targetId: artifact.botId }] : [],
|
||||
),
|
||||
"bot",
|
||||
),
|
||||
findArtifactMessages(
|
||||
prisma,
|
||||
actor,
|
||||
groupArtifacts.flatMap((artifact) =>
|
||||
artifact.groupId && artifact.group
|
||||
? [{ id: artifact.id, targetId: artifact.groupId }]
|
||||
: [],
|
||||
),
|
||||
"group",
|
||||
),
|
||||
prisma.$queryRaw<
|
||||
Array<{
|
||||
id: string;
|
||||
threadId: string;
|
||||
seq: number;
|
||||
blocks: Prisma.JsonValue;
|
||||
botId: string;
|
||||
botName: string;
|
||||
}>
|
||||
>`
|
||||
SELECT m.id, m."threadId", m.seq, m.blocks, b.id AS "botId", b.name AS "botName"
|
||||
FROM messages m
|
||||
INNER JOIN threads t ON t.id = m."threadId"
|
||||
INNER JOIN bots b ON b.id = t."botId"
|
||||
WHERE t."spaceId" = ${actor.spaceId}
|
||||
AND t."userId" = ${actor.userId}
|
||||
AND b."archivedAt" IS NULL
|
||||
AND m.blocks::text ILIKE ${pattern}
|
||||
ORDER BY m."createdAt" DESC
|
||||
LIMIT ${SEARCH_LIMIT}
|
||||
`,
|
||||
prisma.$queryRaw<
|
||||
Array<{
|
||||
id: string;
|
||||
threadId: string;
|
||||
seq: number;
|
||||
blocks: Prisma.JsonValue;
|
||||
groupId: string;
|
||||
groupName: string;
|
||||
}>
|
||||
>`
|
||||
SELECT m.id, m."threadId", m.seq, m.blocks, g.id AS "groupId", g.name AS "groupName"
|
||||
FROM messages m
|
||||
INNER JOIN threads t ON t.id = m."threadId"
|
||||
INNER JOIN chat_groups g ON g.id = t."groupId"
|
||||
WHERE t."spaceId" = ${actor.spaceId}
|
||||
AND t."userId" = ${actor.userId}
|
||||
AND t."groupId" IS NOT NULL
|
||||
AND m.blocks::text ILIKE ${pattern}
|
||||
ORDER BY m."createdAt" DESC
|
||||
LIMIT ${SEARCH_LIMIT}
|
||||
`,
|
||||
]);
|
||||
|
||||
const artifacts = await prisma.artifact.findMany({
|
||||
where: {
|
||||
spaceId: actor.spaceId,
|
||||
userId: actor.userId,
|
||||
groupId: null,
|
||||
botId: { not: null },
|
||||
name: { contains: query, mode: "insensitive" },
|
||||
bot: { archivedAt: null },
|
||||
},
|
||||
include: { bot: { select: { name: true } } },
|
||||
take: SEARCH_LIMIT,
|
||||
});
|
||||
const botArtifactMessages = await findArtifactMessages(
|
||||
prisma,
|
||||
actor,
|
||||
artifacts.flatMap((artifact) =>
|
||||
artifact.botId && artifact.bot ? [{ id: artifact.id, targetId: artifact.botId }] : [],
|
||||
),
|
||||
"bot",
|
||||
);
|
||||
for (const artifact of artifacts) {
|
||||
if (!artifact.botId || !artifact.bot) continue;
|
||||
const message = botArtifactMessages.get(artifact.id);
|
||||
@ -155,24 +239,6 @@ export async function querySpaceSearch(
|
||||
);
|
||||
}
|
||||
|
||||
const groupArtifacts = await prisma.artifact.findMany({
|
||||
where: {
|
||||
spaceId: actor.spaceId,
|
||||
userId: actor.userId,
|
||||
groupId: { not: null },
|
||||
name: { contains: query, mode: "insensitive" },
|
||||
},
|
||||
include: { group: { select: { name: true } } },
|
||||
take: SEARCH_LIMIT,
|
||||
});
|
||||
const groupArtifactMessages = await findArtifactMessages(
|
||||
prisma,
|
||||
actor,
|
||||
groupArtifacts.flatMap((artifact) =>
|
||||
artifact.groupId && artifact.group ? [{ id: artifact.id, targetId: artifact.groupId }] : [],
|
||||
),
|
||||
"group",
|
||||
);
|
||||
for (const artifact of groupArtifacts) {
|
||||
if (!artifact.groupId || !artifact.group) continue;
|
||||
const message = groupArtifactMessages.get(artifact.id);
|
||||
@ -193,19 +259,6 @@ export async function querySpaceSearch(
|
||||
);
|
||||
}
|
||||
|
||||
const routines = await prisma.routine.findMany({
|
||||
where: {
|
||||
spaceId: actor.spaceId,
|
||||
userId: actor.userId,
|
||||
OR: [
|
||||
{ name: { contains: query, mode: "insensitive" } },
|
||||
{ prompt: { contains: query, mode: "insensitive" } },
|
||||
],
|
||||
bot: { archivedAt: null },
|
||||
},
|
||||
include: { bot: { select: { name: true } } },
|
||||
take: SEARCH_LIMIT,
|
||||
});
|
||||
for (const routine of routines) {
|
||||
pushInto(
|
||||
contentHits,
|
||||
@ -221,29 +274,6 @@ export async function querySpaceSearch(
|
||||
);
|
||||
}
|
||||
|
||||
const pattern = `%${query}%`;
|
||||
const messageRows = await prisma.$queryRaw<
|
||||
Array<{
|
||||
id: string;
|
||||
threadId: string;
|
||||
seq: number;
|
||||
blocks: Prisma.JsonValue;
|
||||
botId: string;
|
||||
botName: string;
|
||||
}>
|
||||
>`
|
||||
SELECT m.id, m."threadId", m.seq, m.blocks, b.id AS "botId", b.name AS "botName"
|
||||
FROM messages m
|
||||
INNER JOIN threads t ON t.id = m."threadId"
|
||||
INNER JOIN bots b ON b.id = t."botId"
|
||||
WHERE t."spaceId" = ${actor.spaceId}
|
||||
AND t."userId" = ${actor.userId}
|
||||
AND b."archivedAt" IS NULL
|
||||
AND m.blocks::text ILIKE ${pattern}
|
||||
ORDER BY m."createdAt" DESC
|
||||
LIMIT ${SEARCH_LIMIT}
|
||||
`;
|
||||
|
||||
for (const row of messageRows) {
|
||||
pushMessageHits(row.blocks as MessageBlock[], {
|
||||
botId: row.botId,
|
||||
@ -255,28 +285,6 @@ export async function querySpaceSearch(
|
||||
});
|
||||
}
|
||||
|
||||
const groupMessageRows = await prisma.$queryRaw<
|
||||
Array<{
|
||||
id: string;
|
||||
threadId: string;
|
||||
seq: number;
|
||||
blocks: Prisma.JsonValue;
|
||||
groupId: string;
|
||||
groupName: string;
|
||||
}>
|
||||
>`
|
||||
SELECT m.id, m."threadId", m.seq, m.blocks, g.id AS "groupId", g.name AS "groupName"
|
||||
FROM messages m
|
||||
INNER JOIN threads t ON t.id = m."threadId"
|
||||
INNER JOIN chat_groups g ON g.id = t."groupId"
|
||||
WHERE t."spaceId" = ${actor.spaceId}
|
||||
AND t."userId" = ${actor.userId}
|
||||
AND t."groupId" IS NOT NULL
|
||||
AND m.blocks::text ILIKE ${pattern}
|
||||
ORDER BY m."createdAt" DESC
|
||||
LIMIT ${SEARCH_LIMIT}
|
||||
`;
|
||||
|
||||
for (const row of groupMessageRows) {
|
||||
pushMessageHits(row.blocks as MessageBlock[], {
|
||||
groupId: row.groupId,
|
||||
|
||||
@ -69,7 +69,7 @@ test("Oo browser shows live screen, cursor, cowork click, and user context", asy
|
||||
});
|
||||
});
|
||||
|
||||
await page.getByRole("button", { name: "Browser", exact: true }).click();
|
||||
await page.getByTitle("Agent computer").click();
|
||||
const browser = page.getByTestId("in-app-browser");
|
||||
await expect(browser.getByTestId("oo-computer-browser-screen")).toHaveAttribute(
|
||||
"src",
|
||||
|
||||
@ -74,7 +74,7 @@ export function DualSurfaceDivider({
|
||||
onDoubleClick={handleDoubleClick}
|
||||
onKeyDown={handleKeyDown}
|
||||
className={cn(
|
||||
"group relative flex w-3 md:w-4 shrink-0 cursor-col-resize select-none touch-none items-center justify-center transition-colors bg-border/0 hover:bg-accent/40 active:bg-primary/20 focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring",
|
||||
"group relative flex w-3 md:w-3 shrink-0 cursor-col-resize select-none touch-none items-center justify-center transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring",
|
||||
disabled && "pointer-events-none cursor-default opacity-50",
|
||||
className,
|
||||
)}
|
||||
@ -82,8 +82,8 @@ export function DualSurfaceDivider({
|
||||
>
|
||||
<div
|
||||
className={cn(
|
||||
"h-full w-1 rounded-full bg-border transition-colors group-hover:bg-accent group-active:bg-primary",
|
||||
isDragging && "bg-primary",
|
||||
"h-12 w-1 rounded-full bg-border opacity-0 transition-opacity group-hover:opacity-100 group-active:bg-primary group-active:opacity-100",
|
||||
isDragging && "bg-primary opacity-100",
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@ -5,8 +5,8 @@
|
||||
|
||||
type AudioContextCtor = new () => AudioContext;
|
||||
|
||||
/** Master output ~−15 LUFS for short procedural peaks. */
|
||||
const SFX_MASTER_GAIN = 0.58;
|
||||
/** Master output ~−12 LUFS for clear, warm procedural peaks. */
|
||||
const SFX_MASTER_GAIN = 0.85;
|
||||
/** Duck depth when voice lane active (TTS or mic wash). */
|
||||
const SFX_DUCK_RATIO = 0.28;
|
||||
const DUCK_RAMP_S = 0.065;
|
||||
@ -22,8 +22,10 @@ class OoAudioBus {
|
||||
private ctx: AudioContext | null = null;
|
||||
private sfxGain: GainNode | null = null;
|
||||
private voiceHold = 0;
|
||||
private unlockBound = false;
|
||||
|
||||
ensureContext(): AudioContext | null {
|
||||
this.bindGestureUnlock();
|
||||
if (this.ctx && this.ctx.state !== "closed") return this.ctx;
|
||||
const Ctor = audioContextCtor();
|
||||
if (!Ctor) return null;
|
||||
@ -43,6 +45,30 @@ class OoAudioBus {
|
||||
}
|
||||
}
|
||||
|
||||
/** Unlocks Web Audio on the user's first click, key, or tap so async SSE events can play audio. */
|
||||
bindGestureUnlock(): void {
|
||||
if (this.unlockBound || typeof window === "undefined" || typeof window.addEventListener !== "function") {
|
||||
return;
|
||||
}
|
||||
this.unlockBound = true;
|
||||
const unlock = () => {
|
||||
const ctx = this.ensureContext();
|
||||
if (ctx && ctx.state === "suspended") {
|
||||
void ctx.resume().catch(() => undefined);
|
||||
}
|
||||
if (typeof window !== "undefined" && "speechSynthesis" in window) {
|
||||
try {
|
||||
window.speechSynthesis.getVoices?.();
|
||||
} catch {
|
||||
// Ignore voice preload errors.
|
||||
}
|
||||
}
|
||||
};
|
||||
window.addEventListener("pointerdown", unlock, { capture: true, passive: true });
|
||||
window.addEventListener("keydown", unlock, { capture: true, passive: true });
|
||||
window.addEventListener("touchstart", unlock, { capture: true, passive: true });
|
||||
}
|
||||
|
||||
/** Route for oo-sound / oo-vocals / oo-voice-mark oscillators. */
|
||||
sfxDestination(ctx: AudioContext): AudioNode {
|
||||
if (ctx !== this.ctx) {
|
||||
|
||||
@ -68,61 +68,62 @@ function playCueOnContext(ctx: AudioContext, cue: OoSoundCue) {
|
||||
|
||||
switch (cue) {
|
||||
case "app-start":
|
||||
sweep(ctx, master, 523.25, 659.25, t0, 0.14, { gain: 0.038, type: "sine" });
|
||||
sweep(ctx, master, 659.25, 493.88, t0 + 0.1, 0.18, { gain: 0.032, type: "triangle" });
|
||||
sweep(ctx, master, 523.25, 659.25, t0, 0.16, { gain: 0.11, type: "sine" });
|
||||
sweep(ctx, master, 659.25, 783.99, t0 + 0.1, 0.2, { gain: 0.095, type: "triangle" });
|
||||
break;
|
||||
case "interact":
|
||||
tone(ctx, master, 1046.5, t0, 0.06, {
|
||||
gain: 0.018,
|
||||
attack: 0.003,
|
||||
release: 0.04,
|
||||
tone(ctx, master, 880, t0, 0.075, {
|
||||
gain: 0.065,
|
||||
attack: 0.004,
|
||||
release: 0.045,
|
||||
type: "triangle",
|
||||
});
|
||||
break;
|
||||
case "success-resolve":
|
||||
case "work-complete":
|
||||
// mint → gold resolve (~270 ms)
|
||||
sweep(ctx, master, 523.25, 659.25, t0, 0.12, { gain: 0.042, type: "sine" });
|
||||
sweep(ctx, master, 659.25, 783.99, t0 + 0.1, 0.22, { gain: 0.036, type: "triangle" });
|
||||
tone(ctx, master, 880, t0 + 0.2, 0.28, { gain: 0.03, type: "sine" });
|
||||
sweep(ctx, master, 523.25, 659.25, t0, 0.13, { gain: 0.12, type: "sine" });
|
||||
sweep(ctx, master, 659.25, 783.99, t0 + 0.1, 0.22, { gain: 0.105, type: "triangle" });
|
||||
tone(ctx, master, 880, t0 + 0.2, 0.28, { gain: 0.09, type: "sine" });
|
||||
break;
|
||||
case "need-you":
|
||||
sweep(ctx, master, 220, 330, t0, 0.1, { gain: 0.042, type: "triangle" });
|
||||
tone(ctx, master, 277.18, t0 + 0.09, 0.12, { gain: 0.035, release: 0.08 });
|
||||
sweep(ctx, master, 261.63, 392, t0, 0.12, { gain: 0.125, type: "triangle" });
|
||||
tone(ctx, master, 523.25, t0 + 0.1, 0.18, { gain: 0.11, release: 0.1 });
|
||||
break;
|
||||
case "listening-start":
|
||||
case "voice_listen":
|
||||
tone(ctx, master, 220, t0, 0.22, { gain: 0.045, type: "triangle" });
|
||||
tone(ctx, master, 330, t0 + 0.04, 0.28, { gain: 0.035 });
|
||||
tone(ctx, master, 293.66, t0, 0.2, { gain: 0.11, type: "triangle" });
|
||||
tone(ctx, master, 440, t0 + 0.05, 0.24, { gain: 0.095 });
|
||||
break;
|
||||
case "listening-stop":
|
||||
tone(ctx, master, 330, t0, 0.18, { gain: 0.03, release: 0.12 });
|
||||
tone(ctx, master, 392, t0, 0.16, { gain: 0.085, release: 0.1 });
|
||||
tone(ctx, master, 523.25, t0 + 0.06, 0.18, { gain: 0.075, release: 0.11 });
|
||||
break;
|
||||
case "speaking-start":
|
||||
tone(ctx, master, 440, t0, 0.2, { gain: 0.04, type: "sine" });
|
||||
sweep(ctx, master, 392, 523.25, t0, 0.14, { gain: 0.095, type: "sine" });
|
||||
break;
|
||||
case "speaking-done":
|
||||
tone(ctx, master, 523.25, t0, 0.24, { gain: 0.035 });
|
||||
tone(ctx, master, 392, t0 + 0.08, 0.3, { gain: 0.028, release: 0.14 });
|
||||
tone(ctx, master, 523.25, t0, 0.2, { gain: 0.085 });
|
||||
tone(ctx, master, 392, t0 + 0.08, 0.26, { gain: 0.075, release: 0.14 });
|
||||
break;
|
||||
case "voice_bargein":
|
||||
sweep(ctx, master, 523.25, 261.63, t0, 0.05, { gain: 0.032, type: "sine" });
|
||||
sweep(ctx, master, 523.25, 261.63, t0, 0.07, { gain: 0.09, type: "sine" });
|
||||
break;
|
||||
case "attention":
|
||||
tone(ctx, master, 277.18, t0, 0.16, { gain: 0.05, type: "triangle" });
|
||||
tone(ctx, master, 233.08, t0 + 0.12, 0.22, { gain: 0.045 });
|
||||
tone(ctx, master, 329.63, t0, 0.16, { gain: 0.13, type: "triangle" });
|
||||
tone(ctx, master, 261.63, t0 + 0.12, 0.22, { gain: 0.115 });
|
||||
break;
|
||||
case "message-sent":
|
||||
tone(ctx, master, 880, t0, 0.1, { gain: 0.022, attack: 0.004, release: 0.06 });
|
||||
sweep(ctx, master, 587.33, 880, t0, 0.11, { gain: 0.08, type: "sine" });
|
||||
break;
|
||||
case "handoff":
|
||||
tone(ctx, master, 349.23, t0, 0.2, { gain: 0.04 });
|
||||
tone(ctx, master, 466.16, t0 + 0.1, 0.24, { gain: 0.035, type: "triangle" });
|
||||
tone(ctx, master, 392, t0, 0.18, { gain: 0.11 });
|
||||
tone(ctx, master, 523.25, t0 + 0.09, 0.22, { gain: 0.1, type: "triangle" });
|
||||
break;
|
||||
case "group-spark":
|
||||
tone(ctx, master, 587.33, t0, 0.11, { gain: 0.032, attack: 0.004, type: "triangle" });
|
||||
tone(ctx, master, 739.99, t0 + 0.06, 0.14, { gain: 0.028, type: "sine" });
|
||||
tone(ctx, master, 880, t0 + 0.12, 0.1, { gain: 0.022, release: 0.05 });
|
||||
tone(ctx, master, 587.33, t0, 0.11, { gain: 0.095, attack: 0.004, type: "triangle" });
|
||||
tone(ctx, master, 739.99, t0 + 0.06, 0.14, { gain: 0.085, type: "sine" });
|
||||
tone(ctx, master, 880, t0 + 0.12, 0.12, { gain: 0.075, release: 0.06 });
|
||||
break;
|
||||
case "routine_start":
|
||||
tone(ctx, master, 392, t0, 0.08, { gain: 0.025, attack: 0.005, release: 0.03, type: "sine" });
|
||||
|
||||
@ -3,8 +3,8 @@ import { readOoSoundEnabled } from "./oo-sound-pref";
|
||||
|
||||
export type OoVocalReaction = "ouh" | "mhm" | "aaah" | "mm" | "nod-down" | "nod-up" | "hm";
|
||||
|
||||
const COOLDOWN_MIN_MS = 8_000;
|
||||
const COOLDOWN_MAX_MS = 12_000;
|
||||
const COOLDOWN_MIN_MS = 1_600;
|
||||
const COOLDOWN_MAX_MS = 2_800;
|
||||
|
||||
let lastPlayedAt = 0;
|
||||
let nextCooldownMs = COOLDOWN_MIN_MS;
|
||||
@ -28,17 +28,23 @@ function vocalFormant(
|
||||
) {
|
||||
const osc = ctx.createOscillator();
|
||||
const gain = ctx.createGain();
|
||||
const filter = ctx.createBiquadFilter();
|
||||
filter.type = "bandpass";
|
||||
filter.frequency.value = baseHz * 2.4;
|
||||
filter.Q.value = 4.5;
|
||||
osc.type = "triangle";
|
||||
const boostedPeak = Math.min(0.22, gainPeak * 3.2);
|
||||
osc.type = "sine";
|
||||
osc.frequency.setValueAtTime(baseHz, startAt);
|
||||
osc.frequency.exponentialRampToValueAtTime(baseHz * 1.12, startAt + duration * 0.75);
|
||||
gain.gain.setValueAtTime(0.0001, startAt);
|
||||
gain.gain.exponentialRampToValueAtTime(gainPeak, startAt + 0.025);
|
||||
gain.gain.exponentialRampToValueAtTime(boostedPeak, startAt + 0.02);
|
||||
gain.gain.exponentialRampToValueAtTime(0.0001, startAt + duration);
|
||||
osc.connect(filter);
|
||||
filter.connect(gain);
|
||||
if (typeof ctx.createBiquadFilter === "function") {
|
||||
const filter = ctx.createBiquadFilter();
|
||||
filter.type = "bandpass";
|
||||
filter.frequency.value = baseHz * 1.6;
|
||||
filter.Q.value = 1.8;
|
||||
osc.connect(filter);
|
||||
filter.connect(gain);
|
||||
} else {
|
||||
osc.connect(gain);
|
||||
}
|
||||
gain.connect(destination);
|
||||
osc.start(startAt);
|
||||
osc.stop(startAt + duration + 0.02);
|
||||
|
||||
@ -215,6 +215,53 @@ describe("Speaker", () => {
|
||||
expect(speaker.state.status).toBe("idle");
|
||||
expect(speaker.state.error).toBeUndefined();
|
||||
});
|
||||
|
||||
it("filters out novelty voices and selects a natural German voice for German text", async () => {
|
||||
stubSelectedSpace("space-voice");
|
||||
const speak = vi.fn();
|
||||
const cancel = vi.fn();
|
||||
const eddy = { name: "Eddy (German (Germany))", lang: "de-DE", localService: true };
|
||||
const grandma = { name: "Grandma (German (Germany))", lang: "de-DE", localService: true };
|
||||
const anna = { name: "Anna", lang: "de-DE", localService: true };
|
||||
vi.stubGlobal(
|
||||
"SpeechSynthesisUtterance",
|
||||
class {
|
||||
onend: (() => void) | null = null;
|
||||
onerror: (() => void) | null = null;
|
||||
voice: unknown = null;
|
||||
lang = "";
|
||||
rate = 1;
|
||||
pitch = 1;
|
||||
constructor(public text: string) {}
|
||||
},
|
||||
);
|
||||
vi.stubGlobal("window", {
|
||||
speechSynthesis: {
|
||||
cancel,
|
||||
getVoices: () => [eddy, grandma, anna],
|
||||
speak,
|
||||
},
|
||||
localStorage: globalThis.localStorage,
|
||||
});
|
||||
|
||||
const speaker = new Speaker();
|
||||
vi.spyOn(
|
||||
speaker as unknown as { prepare: () => Promise<string[]> },
|
||||
"prepare",
|
||||
).mockRejectedValue(new Error("offline"));
|
||||
|
||||
const pending = speaker.speak("Ich habe das Projekt für dich aktualisiert.");
|
||||
await vi.waitFor(() => expect(speak).toHaveBeenCalledOnce());
|
||||
const utterance = speak.mock.calls[0]?.[0] as {
|
||||
voice?: { name: string };
|
||||
lang?: string;
|
||||
onend?: () => void;
|
||||
};
|
||||
expect(utterance.lang).toBe("de-DE");
|
||||
expect(utterance.voice?.name).toBe("Anna");
|
||||
utterance.onend?.();
|
||||
await pending;
|
||||
});
|
||||
});
|
||||
|
||||
describe("UtteranceStreamer", () => {
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { ClauseSegmenter, readBoundedResponseBytes, speakable } from "@rakazo/core";
|
||||
import { ooAudioBus } from "./oo-audio-bus.js";
|
||||
import { rpc, selectedSpaceId, withSpaceHeaders } from "./rpc.js";
|
||||
import { readUiLocale } from "./ui-locale.js";
|
||||
|
||||
@ -77,7 +78,19 @@ export class Speaker {
|
||||
if (this.snapshot.status !== "idle" || this.snapshot.error) this.set(IDLE);
|
||||
}
|
||||
|
||||
private bufferSource: AudioBufferSourceNode | null = null;
|
||||
|
||||
private teardownAudio() {
|
||||
if (this.bufferSource) {
|
||||
try {
|
||||
this.bufferSource.onended = null;
|
||||
this.bufferSource.stop();
|
||||
this.bufferSource.disconnect();
|
||||
} catch {
|
||||
// Ignore already-stopped buffer source errors.
|
||||
}
|
||||
this.bufferSource = null;
|
||||
}
|
||||
if (this.audio) {
|
||||
this.audio.pause();
|
||||
this.audio.src = "";
|
||||
@ -226,16 +239,51 @@ export class Speaker {
|
||||
audio.onended = null;
|
||||
audio.onerror = null;
|
||||
if (this.settlePlayback === done) this.settlePlayback = null;
|
||||
if (this.audio === audio) this.teardownAudio();
|
||||
if (this.audio === audio || this.bufferSource) this.teardownAudio();
|
||||
resolve(ok);
|
||||
};
|
||||
this.settlePlayback = done;
|
||||
audio.onended = () => done(true);
|
||||
audio.onerror = () => done(false);
|
||||
audio.play().catch(() => done(false));
|
||||
audio.onerror = () => {
|
||||
void this.playViaWebAudio(blob, live).then(done);
|
||||
};
|
||||
audio.play().catch(() => {
|
||||
void this.playViaWebAudio(blob, live).then(done);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private async playViaWebAudio(blob: Blob, live: () => boolean): Promise<boolean> {
|
||||
if (!live()) return false;
|
||||
const ctx = ooAudioBus.ensureContext();
|
||||
if (
|
||||
!ctx ||
|
||||
typeof ctx.decodeAudioData !== "function" ||
|
||||
typeof ctx.createBufferSource !== "function"
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const arrayBuffer = await blob.arrayBuffer();
|
||||
if (!live()) return false;
|
||||
const audioBuffer = await ctx.decodeAudioData(arrayBuffer.slice(0));
|
||||
if (!live()) return false;
|
||||
return await new Promise<boolean>((resolve) => {
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(ctx.destination);
|
||||
this.bufferSource = source;
|
||||
source.onended = () => {
|
||||
if (this.bufferSource === source) this.bufferSource = null;
|
||||
resolve(true);
|
||||
};
|
||||
source.start(0);
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async tryBrowserFallback(
|
||||
text: string,
|
||||
opts: SpeakOptions,
|
||||
@ -256,6 +304,7 @@ export class Speaker {
|
||||
return new Promise((resolve) => {
|
||||
try {
|
||||
window.speechSynthesis.cancel();
|
||||
window.speechSynthesis.resume?.();
|
||||
const utterance = new SpeechSynthesisUtterance(text);
|
||||
const voices = window.speechSynthesis.getVoices?.() ?? [];
|
||||
const uiLocale = readUiLocale();
|
||||
@ -269,21 +318,26 @@ export class Speaker {
|
||||
const langPrefix = isGerman ? "de" : uiLocale.split("-")[0] || "en";
|
||||
utterance.lang = isGerman ? "de-DE" : langPrefix === "en" ? "en-US" : uiLocale;
|
||||
|
||||
const langVoices = voices.filter((v) =>
|
||||
v.lang.toLowerCase().replace(/_/g, "-").startsWith(langPrefix),
|
||||
const allLangVoices = voices.filter((v) =>
|
||||
(v.lang ?? "").toLowerCase().replace(/_/g, "-").startsWith(langPrefix),
|
||||
);
|
||||
const noveltyHint =
|
||||
/\b(?:eddy|flo|grandma|grandpa|reed|rocko|sandy|shelley|albert|bad news|bahh|bells|boing|bubbles|cellos|deranged|good news|hysterical|junior|organ|superstar|trinoids|whisper|zarvox|fred|kathy|ralph)\b/i;
|
||||
const naturalLangVoices = allLangVoices.filter((v) => !noveltyHint.test(v.name ?? ""));
|
||||
const langVoices = naturalLangVoices.length > 0 ? naturalLangVoices : allLangVoices;
|
||||
const wantsMale = /^dm_/i.test(voiceId);
|
||||
const wantsFemale = /^df_/i.test(voiceId);
|
||||
const maleHint = /yannick|martin|markus|stefan|viktor|hans|daniel|thomas|male/i;
|
||||
const femaleHint = /anna|petra|helena|marlene|vicki|katja|siri|eloquence|female/i;
|
||||
const wantsFemale = /^df_/i.test(voiceId) || (!wantsMale && isGerman);
|
||||
const maleHint =
|
||||
/yannick|martin|markus|stefan|viktor|hans|daniel|thomas|florian|conrad|male/i;
|
||||
const femaleHint = /anna|petra|helena|marlene|vicki|katja|seraphina|amala|siri|female/i;
|
||||
|
||||
const matchedVoice =
|
||||
(wantsMale &&
|
||||
(langVoices.find((v) => !v.localService && maleHint.test(v.name)) ||
|
||||
langVoices.find((v) => maleHint.test(v.name)))) ||
|
||||
(langVoices.find((v) => !v.localService && maleHint.test(v.name ?? "")) ||
|
||||
langVoices.find((v) => maleHint.test(v.name ?? "")))) ||
|
||||
(wantsFemale &&
|
||||
(langVoices.find((v) => !v.localService && femaleHint.test(v.name)) ||
|
||||
langVoices.find((v) => femaleHint.test(v.name)))) ||
|
||||
(langVoices.find((v) => !v.localService && femaleHint.test(v.name ?? "")) ||
|
||||
langVoices.find((v) => femaleHint.test(v.name ?? "")))) ||
|
||||
langVoices.find((v) => !v.localService) ||
|
||||
langVoices[0] ||
|
||||
voices[0];
|
||||
|
||||
@ -75,7 +75,6 @@ import {
|
||||
ArrowLeft,
|
||||
ArrowRight,
|
||||
Calendar,
|
||||
Globe,
|
||||
PanelRight,
|
||||
Pencil,
|
||||
Plus,
|
||||
@ -1228,17 +1227,41 @@ export function ShellPage() {
|
||||
|
||||
const { agentSkills, setAgentSkills, refreshAgentSkills } = useAgentSkills();
|
||||
|
||||
const resolveBotVoiceId = useCallback(
|
||||
(botId: string | undefined, currentVoiceId?: string | null): string => {
|
||||
if (currentVoiceId && /^d[fm]_/i.test(currentVoiceId)) return currentVoiceId;
|
||||
const germanVoices = [
|
||||
"df_eva",
|
||||
"dm_thorsten",
|
||||
"df_ramona",
|
||||
"dm_karlsson",
|
||||
"df_kerstin",
|
||||
] as const;
|
||||
if (!botId) return "df_eva";
|
||||
let hash = 0;
|
||||
for (let i = 0; i < botId.length; i += 1) {
|
||||
hash = (hash * 31 + botId.charCodeAt(i)) >>> 0;
|
||||
}
|
||||
return germanVoices[hash % germanVoices.length] ?? "df_eva";
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
void rpc.voice
|
||||
.status()
|
||||
.then(async (status) => {
|
||||
if (!status.configured) {
|
||||
const defaultVoiceId =
|
||||
typeof navigator !== "undefined" && /^de\b/i.test(navigator.language)
|
||||
? "df_eva"
|
||||
: "af_heart";
|
||||
await rpc.voice
|
||||
.connect({ provider: "kokoro", apiKey: "local-kokoro", voiceId: defaultVoiceId })
|
||||
.connect({ provider: "kokoro", apiKey: "local-kokoro", voiceId: "df_eva" })
|
||||
.catch(() => undefined);
|
||||
const refreshed = await rpc.voice.status().catch(() => status);
|
||||
setVoiceStatus(refreshed);
|
||||
return;
|
||||
}
|
||||
if (!status.voiceId || /^a[fm]_/i.test(status.voiceId)) {
|
||||
await rpc.voice
|
||||
.setVoice({ provider: status.provider ?? "kokoro", voiceId: "df_eva" })
|
||||
.catch(() => undefined);
|
||||
const refreshed = await rpc.voice.status().catch(() => status);
|
||||
setVoiceStatus(refreshed);
|
||||
@ -1255,6 +1278,34 @@ export function ShellPage() {
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!bots.length) return;
|
||||
const migratedKey = "rakazo:voice-de-default-v2";
|
||||
try {
|
||||
if (localStorage.getItem(migratedKey) === "1") return;
|
||||
localStorage.setItem(migratedKey, "1");
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const staleBots = bots.filter(
|
||||
(b) => b.autoSpeak === false || !b.voiceId || /^a[fm]_/i.test(b.voiceId),
|
||||
);
|
||||
if (!staleBots.length) return;
|
||||
void Promise.all(
|
||||
staleBots.map((b) =>
|
||||
rpc.bots
|
||||
.update({
|
||||
botId: b.id,
|
||||
autoSpeak: true,
|
||||
voiceId: resolveBotVoiceId(b.id, b.voiceId),
|
||||
})
|
||||
.catch(() => undefined),
|
||||
),
|
||||
).then(() => {
|
||||
void refreshBots().catch(() => undefined);
|
||||
});
|
||||
}, [bots, refreshBots, resolveBotVoiceId]);
|
||||
|
||||
useEffect(() => {
|
||||
const currentSnapshot = inGroup
|
||||
? snapshot?.groupId === groupId
|
||||
@ -1296,7 +1347,8 @@ export function ShellPage() {
|
||||
const text = spokenSummaryFromBlocks(lastBot.blocks);
|
||||
if (!text) return;
|
||||
autoSpoken.current = lastBot.id;
|
||||
void speaker.speak(text, { botId: speakerBotId, messageId: lastBot.id });
|
||||
const voiceId = resolveBotVoiceId(speakerBotId, speakerBot?.voiceId ?? active?.voiceId);
|
||||
void speaker.speak(text, { botId: speakerBotId, voiceId, messageId: lastBot.id });
|
||||
}, [
|
||||
snapshot,
|
||||
inGroup,
|
||||
@ -1306,6 +1358,7 @@ export function ShellPage() {
|
||||
coordinatorBot?.id,
|
||||
bots,
|
||||
callOpen,
|
||||
resolveBotVoiceId,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
@ -2953,9 +3006,11 @@ export function ShellPage() {
|
||||
}
|
||||
const text = spokenSummaryFromBlocks(message.blocks) || speechFromBlocks(message.blocks);
|
||||
const id = message.botId ?? activeBotId.current ?? defaultSpeakerId ?? coordinatorBot?.id;
|
||||
if (text && id) void speaker.speak(text, { botId: id, messageId: message.id });
|
||||
const msgBot = id ? bots.find((b) => b.id === id) : undefined;
|
||||
const voiceId = resolveBotVoiceId(id, msgBot?.voiceId);
|
||||
if (text && id) void speaker.speak(text, { botId: id, voiceId, messageId: message.id });
|
||||
},
|
||||
[coordinatorBot?.id, defaultSpeakerId],
|
||||
[bots, coordinatorBot?.id, defaultSpeakerId, resolveBotVoiceId],
|
||||
);
|
||||
|
||||
async function createGroup(input: { name: string; botIds: string[] }) {
|
||||
@ -3322,8 +3377,16 @@ export function ShellPage() {
|
||||
{!botsSidebarCollapsed ? (
|
||||
<div
|
||||
onPointerDown={startResizingLeft}
|
||||
className="absolute -right-2 top-0 bottom-0 w-4 cursor-col-resize z-50 hidden md:block"
|
||||
/>
|
||||
className="group absolute -right-3 top-0 bottom-0 w-3 cursor-col-resize z-50 hidden md:flex items-center justify-center"
|
||||
>
|
||||
<div
|
||||
className={`h-12 w-1 rounded-full bg-border transition-opacity ${
|
||||
isResizingLeft
|
||||
? "bg-primary opacity-100"
|
||||
: "opacity-0 group-hover:opacity-100 group-active:bg-primary group-active:opacity-100"
|
||||
}`}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
<SidebarHeader
|
||||
activityMode={activityMode}
|
||||
@ -3816,7 +3879,7 @@ export function ShellPage() {
|
||||
minWidth: 0,
|
||||
flexShrink: 1,
|
||||
transition: "width 200ms ease, opacity 150ms ease",
|
||||
overflow: showCanvasOnDesktop ? ("visible" as const) : ("hidden" as const),
|
||||
overflow: "hidden" as const,
|
||||
}
|
||||
: {
|
||||
width:
|
||||
@ -3860,7 +3923,7 @@ export function ShellPage() {
|
||||
aria-hidden={mobileSidebarOpen || undefined}
|
||||
inert={mobileSidebarOpen}
|
||||
style={streamStyle}
|
||||
className="flex min-w-0 flex-1 flex-col bg-background md:rounded-2xl md:border md:border-border md:shadow-2xl overflow-hidden transition-all duration-300 ease-[cubic-bezier(0.16,1,0.3,1)]"
|
||||
className="relative flex min-w-0 flex-1 flex-col bg-background md:rounded-2xl md:border md:border-border md:shadow-2xl overflow-hidden transition-all duration-300 ease-[cubic-bezier(0.16,1,0.3,1)]"
|
||||
>
|
||||
{activeNavPillar === "heute" ? (
|
||||
<div className="flex h-full flex-1 flex-col overflow-hidden">
|
||||
@ -4057,29 +4120,6 @@ export function ShellPage() {
|
||||
disabled={dualSurface.isMobile}
|
||||
className="hidden md:flex"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
title={t`Browser`}
|
||||
aria-label={t`Browser`}
|
||||
onClick={() => {
|
||||
const isBrowserOpen =
|
||||
panel === "browser" && dualSurface.layoutMode !== "stream";
|
||||
if (isBrowserOpen) {
|
||||
setPanel(null);
|
||||
dualSurface.setLayoutMode("stream");
|
||||
} else {
|
||||
setCanvasTab("browser");
|
||||
setPanel("browser");
|
||||
dualSurface.setLayoutMode("split");
|
||||
}
|
||||
}}
|
||||
data-active={
|
||||
panel === "browser" && dualSurface.layoutMode !== "stream" ? "" : undefined
|
||||
}
|
||||
className="app-no-drag rk-touch-target grid size-control-sm min-h-control-sm min-w-control-sm place-items-center rounded-lg hover:bg-accent data-active:bg-accent"
|
||||
>
|
||||
<Globe strokeWidth={1.7} className="text-foreground/75 size-icon-sm" />
|
||||
</button>
|
||||
{active || groupComputerBotId ? (
|
||||
<button
|
||||
type="button"
|
||||
@ -4216,51 +4256,121 @@ export function ShellPage() {
|
||||
</Button>
|
||||
</div>
|
||||
) : null}
|
||||
<Transcript
|
||||
key={activeSnapshot?.threadId}
|
||||
encounterMode={encounterMode}
|
||||
voiceEncounterActive={voiceEncounterActive}
|
||||
scrollRef={messageScroll}
|
||||
artifactTarget={transcriptArtifactTarget}
|
||||
onOpenArtifact={wideLayout ? openArtifactBeside : undefined}
|
||||
messages={transcriptMessages}
|
||||
olderCursor={activeSnapshot?.olderCursor ?? null}
|
||||
loadingOlder={loadingOlder}
|
||||
answerableAskMessageId={answerableAskMessageId}
|
||||
running={transcriptRunning}
|
||||
runStartedAt={workingRuns[0]?.createdAt}
|
||||
workingBots={workingBots}
|
||||
onLoadOlder={loadOlder}
|
||||
onOpenBot={openBot}
|
||||
onOpenGroup={openGroup}
|
||||
onAnswer={answerMessage}
|
||||
onCardAct={actOnCard}
|
||||
onAutopilot={actOnAutopilot}
|
||||
onReply={(message) => {
|
||||
setReplyTarget(message);
|
||||
setReplyQuote(null);
|
||||
}}
|
||||
onQuote={(message, quote) => {
|
||||
setReplyTarget(message);
|
||||
setReplyQuote(quote);
|
||||
}}
|
||||
onReact={reactToMessage}
|
||||
onJumpToMessage={jumpToReplyMessage}
|
||||
onOpenPeerMessages={(peer) => {
|
||||
setPeerConversation(peer);
|
||||
}}
|
||||
memberName={resolveTranscriptMemberName}
|
||||
peerBot={resolveSpeakerBot}
|
||||
onRefresh={refreshActiveThread}
|
||||
onBotChanged={refreshBots}
|
||||
onAddRoutine={addSkillRoutine}
|
||||
voiceReady={Boolean(voiceStatus?.ready)}
|
||||
speakingMessageId={speakingMessageId}
|
||||
onSpeak={speakMessage}
|
||||
onRestartTask={sendMessage}
|
||||
onSendQueuedNow={sendQueuedMessageNow}
|
||||
onRemoveQueuedMessage={removeQueuedMessage}
|
||||
/>
|
||||
<div className="relative flex min-h-0 flex-1 flex-col overflow-hidden">
|
||||
<Transcript
|
||||
key={activeSnapshot?.threadId}
|
||||
encounterMode={encounterMode}
|
||||
voiceEncounterActive={voiceEncounterActive}
|
||||
scrollRef={messageScroll}
|
||||
artifactTarget={transcriptArtifactTarget}
|
||||
onOpenArtifact={wideLayout ? openArtifactBeside : undefined}
|
||||
messages={transcriptMessages}
|
||||
olderCursor={activeSnapshot?.olderCursor ?? null}
|
||||
loadingOlder={loadingOlder}
|
||||
answerableAskMessageId={answerableAskMessageId}
|
||||
running={transcriptRunning}
|
||||
runStartedAt={workingRuns[0]?.createdAt}
|
||||
workingBots={workingBots}
|
||||
onLoadOlder={loadOlder}
|
||||
onOpenBot={openBot}
|
||||
onOpenGroup={openGroup}
|
||||
onAnswer={answerMessage}
|
||||
onCardAct={actOnCard}
|
||||
onAutopilot={actOnAutopilot}
|
||||
onReply={(message) => {
|
||||
setReplyTarget(message);
|
||||
setReplyQuote(null);
|
||||
}}
|
||||
onQuote={(message, quote) => {
|
||||
setReplyTarget(message);
|
||||
setReplyQuote(quote);
|
||||
}}
|
||||
onReact={reactToMessage}
|
||||
onJumpToMessage={jumpToReplyMessage}
|
||||
onOpenPeerMessages={(peer) => {
|
||||
setPeerConversation(peer);
|
||||
}}
|
||||
memberName={resolveTranscriptMemberName}
|
||||
peerBot={resolveSpeakerBot}
|
||||
onRefresh={refreshActiveThread}
|
||||
onBotChanged={refreshBots}
|
||||
onAddRoutine={addSkillRoutine}
|
||||
voiceReady={Boolean(voiceStatus?.ready)}
|
||||
speakingMessageId={speakingMessageId}
|
||||
onSpeak={speakMessage}
|
||||
onRestartTask={sendMessage}
|
||||
onSendQueuedNow={sendQueuedMessageNow}
|
||||
onRemoveQueuedMessage={removeQueuedMessage}
|
||||
/>
|
||||
{recordingSkill ? (
|
||||
<div className="pointer-events-none absolute inset-x-0 bottom-24 z-30 px-6 pb-2 text-center text-ui-body-sm text-destructive">
|
||||
<Trans>
|
||||
Teaching in progress. Stop teaching before sending a new message.
|
||||
</Trans>
|
||||
</div>
|
||||
) : null}
|
||||
{(active || activeGroup) && !showProjectTimeline ? (
|
||||
<Composer
|
||||
key={inGroup ? `group:${groupId}` : `bot:${active?.id}`}
|
||||
draftKey={composerDraftThreadKey({
|
||||
botId: inGroup ? undefined : active?.id,
|
||||
groupId: inGroup ? groupId : undefined,
|
||||
})}
|
||||
running={composerRunning}
|
||||
disabled={Boolean(recordingSkill)}
|
||||
pendingAttachments={activePendingAttachments}
|
||||
attachmentNotice={attachmentNotice}
|
||||
sendError={sendError}
|
||||
runError={displayedRunError}
|
||||
runErrorId={displayedRunErrorId}
|
||||
onRunErrorPresented={handleRunErrorPresented}
|
||||
onDismissError={dismissComposerError}
|
||||
sending={sending}
|
||||
fileInputRef={fileInputRef}
|
||||
onAttachmentPick={onAttachmentPick}
|
||||
onRemoveAttachment={removeAttachment}
|
||||
onSend={sendMessage}
|
||||
onStop={stopRun}
|
||||
onVoice={
|
||||
active || inGroup
|
||||
? () => {
|
||||
startVoiceEncounter();
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
voiceReady={Boolean(voiceStatus?.ready)}
|
||||
voiceColor={voiceCallBot?.color}
|
||||
replyTarget={activeReplyTarget}
|
||||
replyQuote={activeReplyQuote}
|
||||
replyTargetName={replyTargetName}
|
||||
onClearReply={clearReply}
|
||||
mentionTargets={composerMentionTargets}
|
||||
agentSkills={agentSkills}
|
||||
onSlashOpen={refreshAgentSkills}
|
||||
onConnectModel={
|
||||
bootstrapMe?.needsModel ? () => openSettings("models") : undefined
|
||||
}
|
||||
onAutopilot={startAutopilot}
|
||||
onSlashAction={(action) => {
|
||||
if (action === "chat-settings") {
|
||||
setPanel(inGroup ? "group-settings" : "settings");
|
||||
return;
|
||||
}
|
||||
if (action === "settings-general") {
|
||||
openSettings("general");
|
||||
return;
|
||||
}
|
||||
if (action === "settings-usage") {
|
||||
void rpc.usage
|
||||
.summary()
|
||||
.then(setUsage)
|
||||
.catch(() => undefined);
|
||||
openSettings("usage");
|
||||
}
|
||||
}}
|
||||
/>
|
||||
) : null}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{activeGroup?.kind === "main" && !inProject && !showProjectTimeline ? (
|
||||
@ -4271,72 +4381,6 @@ export function ShellPage() {
|
||||
onOpenInSpace={openSpaceChat}
|
||||
/>
|
||||
) : null}
|
||||
{recordingSkill ? (
|
||||
<div className="px-6 pb-2 text-center text-ui-body-sm text-destructive">
|
||||
<Trans>Teaching in progress. Stop teaching before sending a new message.</Trans>
|
||||
</div>
|
||||
) : null}
|
||||
{(active || activeGroup) && !showProjectTimeline ? (
|
||||
<Composer
|
||||
key={inGroup ? `group:${groupId}` : `bot:${active?.id}`}
|
||||
draftKey={composerDraftThreadKey({
|
||||
botId: inGroup ? undefined : active?.id,
|
||||
groupId: inGroup ? groupId : undefined,
|
||||
})}
|
||||
running={composerRunning}
|
||||
disabled={Boolean(recordingSkill)}
|
||||
pendingAttachments={activePendingAttachments}
|
||||
attachmentNotice={attachmentNotice}
|
||||
sendError={sendError}
|
||||
runError={displayedRunError}
|
||||
runErrorId={displayedRunErrorId}
|
||||
onRunErrorPresented={handleRunErrorPresented}
|
||||
onDismissError={dismissComposerError}
|
||||
sending={sending}
|
||||
fileInputRef={fileInputRef}
|
||||
onAttachmentPick={onAttachmentPick}
|
||||
onRemoveAttachment={removeAttachment}
|
||||
onSend={sendMessage}
|
||||
onStop={stopRun}
|
||||
onVoice={
|
||||
active || inGroup
|
||||
? () => {
|
||||
startVoiceEncounter();
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
voiceReady={Boolean(voiceStatus?.ready)}
|
||||
voiceColor={voiceCallBot?.color}
|
||||
replyTarget={activeReplyTarget}
|
||||
replyQuote={activeReplyQuote}
|
||||
replyTargetName={replyTargetName}
|
||||
onClearReply={clearReply}
|
||||
mentionTargets={composerMentionTargets}
|
||||
agentSkills={agentSkills}
|
||||
onSlashOpen={refreshAgentSkills}
|
||||
onConnectModel={
|
||||
bootstrapMe?.needsModel ? () => openSettings("models") : undefined
|
||||
}
|
||||
onAutopilot={startAutopilot}
|
||||
onSlashAction={(action) => {
|
||||
if (action === "chat-settings") {
|
||||
setPanel(inGroup ? "group-settings" : "settings");
|
||||
return;
|
||||
}
|
||||
if (action === "settings-general") {
|
||||
openSettings("general");
|
||||
return;
|
||||
}
|
||||
if (action === "settings-usage") {
|
||||
void rpc.usage
|
||||
.summary()
|
||||
.then(setUsage)
|
||||
.catch(() => undefined);
|
||||
openSettings("usage");
|
||||
}
|
||||
}}
|
||||
/>
|
||||
) : null}
|
||||
</>
|
||||
)}
|
||||
</main>
|
||||
@ -4366,10 +4410,10 @@ export function ShellPage() {
|
||||
data-panel={panel ?? "closed"}
|
||||
data-auto={panel !== null && panel === autoPanel ? "" : undefined}
|
||||
style={canvasStyle}
|
||||
className={`absolute inset-0 z-30 flex min-h-0 shrink-0 flex-col overflow-hidden bg-background md:relative md:inset-auto md:z-20 md:border-0 md:rounded-none md:shadow-none ${
|
||||
className={`absolute inset-0 z-30 flex min-h-0 shrink-0 flex-col overflow-hidden bg-sidebar md:relative md:inset-auto md:z-20 md:rounded-2xl md:border md:border-sidebar-border md:shadow-2xl ${
|
||||
isPanelVisible
|
||||
? "w-full max-w-none md:max-w-none opacity-100"
|
||||
: "pointer-events-none w-0 opacity-0 md:opacity-0 md:w-0"
|
||||
: "pointer-events-none w-0 opacity-0 md:opacity-0 md:w-0 md:border-0"
|
||||
} ${!(workspacePaneOpen ? isResizingWorkspacePane : isResizingRight) ? "transition-[width,opacity] duration-300 ease-[cubic-bezier(0.16,1,0.3,1)] motion-reduce:transition-none" : ""}`}
|
||||
>
|
||||
<div
|
||||
|
||||
@ -452,11 +452,11 @@ export const Composer = memo(function Composer({
|
||||
onDragOver={handleDragOver}
|
||||
onDragLeave={handleDragLeave}
|
||||
onDrop={handleDrop}
|
||||
className={`relative z-30 m-0 min-w-0 max-w-full overflow-x-hidden border-0 bg-gradient-to-t from-background via-background/80 to-transparent px-3 pb-4 pt-3 md:px-6 md:pb-6 ${
|
||||
className={`composer-bottom-fade pointer-events-none absolute inset-x-0 bottom-0 z-30 m-0 min-w-0 max-w-full overflow-x-hidden border-0 px-3 pb-4 pt-10 md:px-6 md:pb-6 md:pt-12 ${
|
||||
draggingFiles ? "rounded-[14px] ring-2 ring-inset ring-ring" : ""
|
||||
}`}
|
||||
>
|
||||
<div className="mx-auto w-full max-w-4xl">
|
||||
<div className="pointer-events-auto mx-auto w-full max-w-4xl">
|
||||
{onConnectModel ? (
|
||||
<div
|
||||
data-testid="composer-model-needed"
|
||||
|
||||
@ -629,8 +629,8 @@ export const Transcript = memo(function Transcript({
|
||||
</button>
|
||||
) : null}
|
||||
<div
|
||||
className={`mx-auto flex w-full flex-1 flex-col pb-6 ${
|
||||
encounterMode ? "max-w-2xl gap-3" : "max-w-4xl gap-6 pb-8"
|
||||
className={`mx-auto flex w-full flex-1 flex-col ${
|
||||
encounterMode ? "max-w-2xl gap-3 pb-28 md:pb-32" : "max-w-4xl gap-6 pb-28 md:pb-32"
|
||||
}`}
|
||||
>
|
||||
{coalescedMessages.map((message) => {
|
||||
@ -883,7 +883,7 @@ export const Transcript = memo(function Transcript({
|
||||
aria-hidden={!showJumpToLatest}
|
||||
tabIndex={showJumpToLatest ? 0 : -1}
|
||||
onClick={jumpToLatest}
|
||||
className={`absolute bottom-4 left-1/2 z-20 grid h-9 w-9 -translate-x-1/2 place-items-center rounded-full border border-border bg-muted/95 text-foreground/75 shadow-md backdrop-blur transition-[opacity,transform,background-color] duration-200 ease-[cubic-bezier(.22,1,.36,1)] hover:bg-border motion-reduce:transition-none ${
|
||||
className={`absolute bottom-24 left-1/2 z-20 grid h-9 w-9 -translate-x-1/2 place-items-center rounded-full border border-border bg-muted/95 text-foreground/75 shadow-md backdrop-blur transition-[opacity,transform,background-color] duration-200 ease-[cubic-bezier(.22,1,.36,1)] hover:bg-border md:bottom-28 motion-reduce:transition-none ${
|
||||
showJumpToLatest
|
||||
? "translate-y-0 opacity-100"
|
||||
: "pointer-events-none translate-y-2 opacity-0"
|
||||
@ -896,7 +896,7 @@ export const Transcript = memo(function Transcript({
|
||||
type="button"
|
||||
data-testid="needs-you-jump"
|
||||
onClick={jumpToAsk}
|
||||
className="absolute bottom-4 left-1/2 z-20 inline-flex h-9 -translate-x-1/2 items-center gap-1.5 rounded-full border border-border bg-muted/95 px-3.5 text-ui-body-sm font-medium text-foreground shadow-md backdrop-blur hover:bg-border"
|
||||
className="absolute bottom-24 left-1/2 z-20 inline-flex h-9 -translate-x-1/2 items-center gap-1.5 rounded-full border border-border bg-muted/95 px-3.5 text-ui-body-sm font-medium text-foreground shadow-md backdrop-blur hover:bg-border md:bottom-28"
|
||||
>
|
||||
<span aria-hidden="true" className="size-2 rounded-full bg-warning" />
|
||||
{t`Needs you`}
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
import type { Bot, ThreadSnapshot } from "@rakazo/contracts";
|
||||
import { useEffect, useRef } from "react";
|
||||
import { ooAudioBus } from "../../lib/oo-audio-bus";
|
||||
import { wireOoAudioVoiceDucking } from "../../lib/oo-audio-voice-wire";
|
||||
import { ooSound } from "../../lib/oo-sound";
|
||||
import { ooVocals } from "../../lib/oo-vocals";
|
||||
import { ooVoiceMark } from "../../lib/oo-voice-mark";
|
||||
import { speaker } from "../../lib/tts";
|
||||
|
||||
export type ShellSoundsInput = {
|
||||
bots: Bot[];
|
||||
@ -27,8 +29,25 @@ export function useShellSounds({
|
||||
const seenHandoffKeys = useRef(new Set<string>());
|
||||
const appSoundStarted = useRef(false);
|
||||
const prevBotStatuses = useRef(new Map<string, string>());
|
||||
const prevThreadId = useRef<string | null>(null);
|
||||
const prevBotMsgCount = useRef<number>(0);
|
||||
|
||||
useEffect(() => wireOoAudioVoiceDucking(), []);
|
||||
useEffect(() => {
|
||||
ooAudioBus.bindGestureUnlock();
|
||||
return wireOoAudioVoiceDucking();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
let prevStatus = speaker.state.status;
|
||||
return speaker.subscribe((state) => {
|
||||
if (prevStatus !== "speaking" && state.status === "speaking" && !callOpen) {
|
||||
ooSound.play("speaking-start");
|
||||
} else if (prevStatus === "speaking" && state.status === "idle" && !callOpen) {
|
||||
ooSound.play("speaking-done");
|
||||
}
|
||||
prevStatus = state.status;
|
||||
});
|
||||
}, [callOpen]);
|
||||
|
||||
useEffect(() => {
|
||||
if (appSoundStarted.current) return;
|
||||
@ -37,6 +56,23 @@ export function useShellSounds({
|
||||
window.setTimeout(() => ooVoiceMark.maybePlayColdOpen(), 280);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const threadId = activeSnapshot?.threadId ?? null;
|
||||
if (!threadId) return;
|
||||
if (prevThreadId.current && prevThreadId.current !== threadId) {
|
||||
ooSound.play("interact");
|
||||
ooVocals.reactForEvent("nod");
|
||||
prevBotMsgCount.current = (activeSnapshot?.messages ?? []).filter(
|
||||
(m) => m.role === "bot",
|
||||
).length;
|
||||
} else if (!prevThreadId.current) {
|
||||
prevBotMsgCount.current = (activeSnapshot?.messages ?? []).filter(
|
||||
(m) => m.role === "bot",
|
||||
).length;
|
||||
}
|
||||
prevThreadId.current = threadId;
|
||||
}, [activeSnapshot?.threadId, activeSnapshot?.messages]);
|
||||
|
||||
useEffect(() => {
|
||||
for (const bot of bots) {
|
||||
const prev = prevBotStatuses.current.get(bot.id);
|
||||
@ -52,13 +88,27 @@ export function useShellSounds({
|
||||
}, [bots]);
|
||||
|
||||
useEffect(() => {
|
||||
if (prevTranscriptRunning.current && !transcriptRunning && !callOpen) {
|
||||
if (!prevTranscriptRunning.current && transcriptRunning && !callOpen) {
|
||||
ooSound.play("listening-stop");
|
||||
ooVocals.reactForEvent("curious");
|
||||
} else if (prevTranscriptRunning.current && !transcriptRunning && !callOpen) {
|
||||
ooSound.play("success-resolve");
|
||||
ooVocals.reactForEvent("success");
|
||||
}
|
||||
prevTranscriptRunning.current = transcriptRunning;
|
||||
}, [transcriptRunning, callOpen]);
|
||||
|
||||
useEffect(() => {
|
||||
const botMsgs = (activeSnapshot?.messages ?? []).filter((m) => m.role === "bot").length;
|
||||
if (prevThreadId.current === (activeSnapshot?.threadId ?? null) && botMsgs > prevBotMsgCount.current) {
|
||||
if (transcriptRunning && !callOpen) {
|
||||
ooSound.play("interact");
|
||||
ooVocals.react("mhm");
|
||||
}
|
||||
}
|
||||
prevBotMsgCount.current = botMsgs;
|
||||
}, [activeSnapshot?.messages, activeSnapshot?.threadId, transcriptRunning, callOpen]);
|
||||
|
||||
useEffect(() => {
|
||||
const runId = displayedRunErrorId;
|
||||
if (runId && runId !== prevRunErrorId.current) {
|
||||
|
||||
@ -34,15 +34,25 @@ body,
|
||||
}
|
||||
|
||||
.apple-island {
|
||||
backdrop-filter: blur(36px) saturate(210%);
|
||||
-webkit-backdrop-filter: blur(36px) saturate(210%);
|
||||
background: transparent;
|
||||
backdrop-filter: blur(16px) saturate(180%);
|
||||
-webkit-backdrop-filter: blur(16px) saturate(180%);
|
||||
background: color-mix(in srgb, var(--background) 22%, transparent);
|
||||
border: 1px solid color-mix(in srgb, var(--foreground) 13%, transparent);
|
||||
box-shadow:
|
||||
0 18px 44px -8px rgba(0, 0, 0, 0.75),
|
||||
inset 0 1px 0 rgba(255, 255, 255, 0.16);
|
||||
}
|
||||
|
||||
.composer-bottom-fade {
|
||||
background: linear-gradient(
|
||||
to top,
|
||||
var(--background) 0%,
|
||||
color-mix(in srgb, var(--background) 78%, transparent) 32%,
|
||||
color-mix(in srgb, var(--background) 32%, transparent) 68%,
|
||||
transparent 100%
|
||||
);
|
||||
}
|
||||
|
||||
.apple-glass-card {
|
||||
backdrop-filter: blur(20px);
|
||||
-webkit-backdrop-filter: blur(20px);
|
||||
|
||||
@ -45,4 +45,23 @@ describe("git shell policy", () => {
|
||||
it("parses refspec destinations", () => {
|
||||
expect(parseGitPushRef("git push origin HEAD:feature/x")).toBe("feature/x");
|
||||
});
|
||||
|
||||
it("evaluates multiline gh pr create commands with long markdown bodies in linear time", () => {
|
||||
const command = [
|
||||
"cd /home/rakazo/project && gh pr create --repo owner/repo \\",
|
||||
" --base main \\",
|
||||
" --head feature/domain-migration \\",
|
||||
' --title "chore: migrate domain" \\',
|
||||
` --body "${"## Section\n\n- bullet item with words and details\n".repeat(40)}" 2>&1`,
|
||||
].join("\n");
|
||||
const started = performance.now();
|
||||
expect(
|
||||
evaluateGitShellPolicy({
|
||||
command,
|
||||
defaultBranch: "main",
|
||||
gitPushApprovedThisRun: false,
|
||||
}),
|
||||
).toMatchObject({ action: "require_approval" });
|
||||
expect(performance.now() - started).toBeLessThan(50);
|
||||
});
|
||||
});
|
||||
|
||||
@ -9,7 +9,10 @@ import type {
|
||||
VoiceVerifyResult,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import { formatVoiceLabel } from "@rakazo/core";
|
||||
import { listPiperVoices, synthesizePiper } from "./piper-voice.js";
|
||||
import { listPiperVoices, PIPER_MODELS, piperBaseUrl, synthesizePiper } from "./piper-voice.js";
|
||||
|
||||
const GERMAN_TEXT_RE =
|
||||
/[äöüß]|(?:\b(?:und|der|die|das|ist|nicht|für|mit|den|dem|des|ein|eine|ich|wir|hier|bitte|oder|hallo|projekt|habe|sind|wird)\b)/i;
|
||||
import {
|
||||
readVoiceAudio,
|
||||
readVoiceJson,
|
||||
@ -112,7 +115,16 @@ export class KokoroVoiceProvider implements VoiceProvider {
|
||||
|
||||
async synthesize(request: VoiceSynthesizeRequest, context: AdapterContext): Promise<SpeechClip> {
|
||||
const signal = voiceDeadline(request.signal ?? context.signal, 120_000);
|
||||
const piper = await synthesizePiper(request.voiceId, request.text, signal);
|
||||
const effectiveVoiceId =
|
||||
request.voiceId &&
|
||||
!PIPER_MODELS[request.voiceId] &&
|
||||
piperBaseUrl() &&
|
||||
GERMAN_TEXT_RE.test(request.text)
|
||||
? /^[ab]m_/i.test(request.voiceId)
|
||||
? "dm_thorsten"
|
||||
: "df_eva"
|
||||
: request.voiceId;
|
||||
const piper = await synthesizePiper(effectiveVoiceId, request.text, signal);
|
||||
if (piper) return piper;
|
||||
let res: Response;
|
||||
try {
|
||||
|
||||
@ -11,10 +11,15 @@ export const PIPER_MODELS: Record<string, string> = {
|
||||
dm_karlsson: "de_DE-karlsson-low",
|
||||
};
|
||||
|
||||
/** Piper runs only where configured, so a server without it skips the request entirely. */
|
||||
/** Piper runs on 127.0.0.1:5150 by default outside unit tests, or at PIPER_BASE_URL. */
|
||||
export function piperBaseUrl(): string | undefined {
|
||||
const raw = process.env.PIPER_BASE_URL?.trim();
|
||||
return raw ? raw.replace(/\/+$/, "") : undefined;
|
||||
const raw = process.env.PIPER_BASE_URL;
|
||||
if (raw !== undefined) {
|
||||
const trimmed = raw.trim();
|
||||
return trimmed ? trimmed.replace(/\/+$/, "") : undefined;
|
||||
}
|
||||
if (process.env.VITEST || process.env.NODE_ENV === "test") return undefined;
|
||||
return "http://127.0.0.1:5150";
|
||||
}
|
||||
|
||||
export function piperUnreachable(): string {
|
||||
@ -61,5 +66,7 @@ export async function synthesizePiper(
|
||||
throw new Error(piperUnreachable());
|
||||
}
|
||||
await requireOk(res, "Local voice", "speaking");
|
||||
return { bytes: await readVoiceAudio(res, signal), mimeType: "audio/wav" };
|
||||
const contentType = res.headers.get("content-type");
|
||||
const mimeType = contentType?.startsWith("audio/mpeg") ? "audio/mpeg" : "audio/wav";
|
||||
return { bytes: await readVoiceAudio(res, signal), mimeType };
|
||||
}
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ActionApprovalRule } from "./action-approval.js";
|
||||
import {
|
||||
type ActionApprovalRule,
|
||||
applyJudgeDecision,
|
||||
classifyApprovalRiskTier,
|
||||
connectorKindFromToolName,
|
||||
connectorToolRequiresApproval,
|
||||
groupApprovalRequests,
|
||||
isApprovalAskBlock,
|
||||
isSecretAskBlock,
|
||||
planActionGate,
|
||||
@ -120,6 +122,52 @@ describe("isApprovalAskBlock", () => {
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("batches pending approvals by connector and risk tier while deduplicating identical effect keys", () => {
|
||||
expect(classifyApprovalRiskTier("create_space", false)).toBe("explicit_security");
|
||||
expect(classifyApprovalRiskTier("github_create_issue", true)).toBe("consequential_write");
|
||||
expect(classifyApprovalRiskTier("github_get_issue", true)).toBe("standard");
|
||||
|
||||
const batches = groupApprovalRequests([
|
||||
{
|
||||
id: "req-1",
|
||||
toolName: "github_create_issue",
|
||||
viaConnector: true,
|
||||
effectKey: "issue:bug-42",
|
||||
},
|
||||
{
|
||||
id: "req-2",
|
||||
toolName: "github_create_issue",
|
||||
viaConnector: true,
|
||||
effectKey: "issue:bug-42",
|
||||
},
|
||||
{
|
||||
id: "req-3",
|
||||
toolName: "github_add_comment",
|
||||
viaConnector: true,
|
||||
effectKey: "comment:bug-42",
|
||||
},
|
||||
{
|
||||
id: "req-4",
|
||||
toolName: "create_space",
|
||||
viaConnector: false,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(batches).toHaveLength(2);
|
||||
expect(batches[0]).toMatchObject({
|
||||
groupKey: "github:consequential_write",
|
||||
connectorKind: "github",
|
||||
riskTier: "consequential_write",
|
||||
deduplicatedCount: 1,
|
||||
});
|
||||
expect(batches[0]?.items.map((item) => item.id)).toEqual(["req-1", "req-3"]);
|
||||
expect(batches[1]).toMatchObject({
|
||||
groupKey: "create:explicit_security",
|
||||
riskTier: "explicit_security",
|
||||
deduplicatedCount: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("isSecretAskBlock", () => {
|
||||
|
||||
@ -248,3 +248,80 @@ export function isApprovalAskBlock(block: {
|
||||
[...actionIds].every((id) => id === "allow" || id === "always" || id === "deny")
|
||||
);
|
||||
}
|
||||
|
||||
export type ApprovalRiskTier = "explicit_security" | "consequential_write" | "standard";
|
||||
|
||||
export type PendingApprovalCandidate = {
|
||||
id: string;
|
||||
toolName: string;
|
||||
viaConnector: boolean;
|
||||
connectorKind?: string;
|
||||
effectKey?: string;
|
||||
};
|
||||
|
||||
export type ApprovalBatchGroup = {
|
||||
groupKey: string;
|
||||
connectorKind: string;
|
||||
riskTier: ApprovalRiskTier;
|
||||
items: PendingApprovalCandidate[];
|
||||
deduplicatedCount: number;
|
||||
};
|
||||
|
||||
export function classifyApprovalRiskTier(
|
||||
toolName: string,
|
||||
viaConnector: boolean,
|
||||
): ApprovalRiskTier {
|
||||
if (toolRequiresExplicitApproval(toolName)) return "explicit_security";
|
||||
if (toolRequiresApproval(toolName, viaConnector)) return "consequential_write";
|
||||
return "standard";
|
||||
}
|
||||
|
||||
/**
|
||||
* Groups and deduplicates pending tool approval requests by connector and risk tier so the user
|
||||
* can review related actions in a single coherent batch rather than repetitive interruptions.
|
||||
*/
|
||||
export function groupApprovalRequests(
|
||||
candidates: readonly PendingApprovalCandidate[],
|
||||
connectorKinds: string[] = [],
|
||||
): ApprovalBatchGroup[] {
|
||||
const groups = new Map<string, ApprovalBatchGroup>();
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const connectorKind = (
|
||||
candidate.connectorKind ?? connectorKindFromToolName(candidate.toolName, connectorKinds)
|
||||
).toLowerCase();
|
||||
const riskTier = classifyApprovalRiskTier(candidate.toolName, candidate.viaConnector);
|
||||
const groupKey = `${connectorKind}:${riskTier}`;
|
||||
|
||||
let group = groups.get(groupKey);
|
||||
if (!group) {
|
||||
group = {
|
||||
groupKey,
|
||||
connectorKind,
|
||||
riskTier,
|
||||
items: [],
|
||||
deduplicatedCount: 0,
|
||||
};
|
||||
groups.set(groupKey, group);
|
||||
}
|
||||
|
||||
const dedupeKey = candidate.effectKey?.trim()
|
||||
? `${candidate.toolName.toLowerCase()}::${candidate.effectKey.trim()}`
|
||||
: null;
|
||||
const isDuplicate =
|
||||
dedupeKey !== null &&
|
||||
group.items.some(
|
||||
(existing) =>
|
||||
existing.toolName.toLowerCase() === candidate.toolName.toLowerCase() &&
|
||||
existing.effectKey?.trim() === candidate.effectKey?.trim(),
|
||||
);
|
||||
|
||||
if (isDuplicate) {
|
||||
group.deduplicatedCount += 1;
|
||||
} else {
|
||||
group.items.push(candidate);
|
||||
}
|
||||
}
|
||||
|
||||
return [...groups.values()];
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@ import {
|
||||
AttachmentValidationError,
|
||||
attachmentsForBot,
|
||||
blocksToAgentHistoryText,
|
||||
compactAgentHistoryMessages,
|
||||
decodeAttachmentBase64,
|
||||
inferAttachmentMimeType,
|
||||
messageBlockForArtifact,
|
||||
@ -102,6 +103,46 @@ describe("attachment helpers", () => {
|
||||
expect(attachmentsForBot(attachments, "bot-two")).toEqual([attachments[1]]);
|
||||
expect(attachmentsForBot(attachments, undefined)).toEqual([]);
|
||||
});
|
||||
|
||||
it("compacts older history messages while keeping recent messages and artifact IDs intact", () => {
|
||||
const longText = "A".repeat(400);
|
||||
const history = compactAgentHistoryMessages(
|
||||
[
|
||||
{
|
||||
id: "m-old",
|
||||
role: "bot",
|
||||
blocks: [
|
||||
{ kind: "text", text: longText },
|
||||
{
|
||||
kind: "task",
|
||||
taskId: "t-1",
|
||||
assigneeBotId: "b-2",
|
||||
assigneeBotName: "Researcher",
|
||||
title: "Deep scan",
|
||||
text: "Long task brief that is already finished",
|
||||
status: "done",
|
||||
hop: 1,
|
||||
},
|
||||
{ kind: "image", artifactId: "art-keep", mimeType: "image/png", name: "diagram.png" },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "m-recent",
|
||||
role: "user",
|
||||
blocks: [{ kind: "text", text: longText }],
|
||||
},
|
||||
],
|
||||
{ keepRecentFull: 1, maxOlderTextChars: 80 },
|
||||
);
|
||||
|
||||
expect(history[0]?.compacted).toBe(true);
|
||||
expect(history[0]?.text).toContain("[…compacted…]");
|
||||
expect(history[0]?.text).toContain("[task for Researcher: Deep scan (done)]");
|
||||
expect(history[0]?.text).not.toContain("Long task brief that is already finished");
|
||||
expect(history[0]?.text).toContain("(artifact art-keep)");
|
||||
expect(history[1]?.compacted).toBe(false);
|
||||
expect(history[1]?.text).toBe(longText);
|
||||
});
|
||||
});
|
||||
|
||||
describe("peer message history", () => {
|
||||
|
||||
@ -262,3 +262,61 @@ export function userTurnBlocksForRun(
|
||||
(sourceMessageId ? message.id === sourceMessageId : message.runId === runId),
|
||||
)?.blocks;
|
||||
}
|
||||
|
||||
export type AgentHistoryMessageInput = {
|
||||
id: string;
|
||||
role: string;
|
||||
blocks: MessageBlock[];
|
||||
};
|
||||
|
||||
const DEFAULT_KEEP_RECENT_FULL = 6;
|
||||
const DEFAULT_MAX_OLDER_TEXT_CHARS = 280;
|
||||
|
||||
function compactOlderBlock(block: MessageBlock, maxTextChars: number): MessageBlock {
|
||||
if (block.kind === "task" && (block.status === "done" || block.status === "cancelled")) {
|
||||
return { ...block, text: "" };
|
||||
}
|
||||
if (block.kind === "text" && block.text.length > maxTextChars) {
|
||||
return {
|
||||
...block,
|
||||
text: `${block.text.slice(0, maxTextChars).trimEnd()} […compacted…]`,
|
||||
};
|
||||
}
|
||||
return block;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compacts older conversation turns in long-lived threads while keeping the most recent turns,
|
||||
* artifact references, and bot attributions intact.
|
||||
*/
|
||||
export function compactAgentHistoryMessages(
|
||||
messages: readonly AgentHistoryMessageInput[],
|
||||
options?: { keepRecentFull?: number; maxOlderTextChars?: number },
|
||||
): Array<{ id: string; role: string; text: string; compacted: boolean }> {
|
||||
const keepRecentFull = Math.max(0, options?.keepRecentFull ?? DEFAULT_KEEP_RECENT_FULL);
|
||||
const maxOlderTextChars = Math.max(
|
||||
40,
|
||||
options?.maxOlderTextChars ?? DEFAULT_MAX_OLDER_TEXT_CHARS,
|
||||
);
|
||||
const cutoff = Math.max(0, messages.length - keepRecentFull);
|
||||
|
||||
return messages.map((msg, index) => {
|
||||
if (index >= cutoff) {
|
||||
return {
|
||||
id: msg.id,
|
||||
role: msg.role,
|
||||
text: blocksToAgentHistoryText(msg.blocks),
|
||||
compacted: false,
|
||||
};
|
||||
}
|
||||
const compactedBlocks = msg.blocks.map((block) => compactOlderBlock(block, maxOlderTextChars));
|
||||
const fullText = blocksToAgentHistoryText(msg.blocks);
|
||||
const compactText = blocksToAgentHistoryText(compactedBlocks).trimEnd();
|
||||
return {
|
||||
id: msg.id,
|
||||
role: msg.role,
|
||||
text: compactText,
|
||||
compacted: compactText.length < fullText.length,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@ -39,6 +39,40 @@ describe("autopilot", () => {
|
||||
).toBe("failing");
|
||||
});
|
||||
|
||||
it("treats identical consecutive checkpoint texts as stagnation and pauses on loop", () => {
|
||||
let state = advanceAutopilotGuard(
|
||||
{ idleCount: 0, failCount: 0, lastCheckpointText: "Checked inbox" },
|
||||
{ checkpointed: true, failed: false, checkpointText: "checked inbox" },
|
||||
);
|
||||
expect(state.idleCount).toBe(1);
|
||||
expect(state.pause).toBeNull();
|
||||
state = advanceAutopilotGuard(state, {
|
||||
checkpointed: true,
|
||||
failed: false,
|
||||
checkpointText: " Checked inbox ",
|
||||
});
|
||||
expect(state.idleCount).toBe(2);
|
||||
state = advanceAutopilotGuard(state, {
|
||||
checkpointed: true,
|
||||
failed: false,
|
||||
checkpointText: "Checked inbox",
|
||||
});
|
||||
expect(state.pause).toBe("idle");
|
||||
});
|
||||
|
||||
it("includes a bounded recent checkpoint trail in continuation prompts when provided", () => {
|
||||
const prompt = autopilotContinuationPrompt({
|
||||
goal: "Audit modules",
|
||||
focus: null,
|
||||
until: null,
|
||||
lastCheckpoint: "Module B audited",
|
||||
recentCheckpoints: ["Module A audited", "Module B audited"],
|
||||
nextStep: "Module C",
|
||||
stepCount: 2,
|
||||
});
|
||||
expect(prompt).toContain("Recent checkpoints: 1. Module A audited | 2. Module B audited");
|
||||
});
|
||||
|
||||
it("tells each continuation the order and where it stands", () => {
|
||||
const prompt = autopilotContinuationPrompt({
|
||||
goal: "Tidy up the tickets",
|
||||
|
||||
@ -11,21 +11,35 @@ export interface AutopilotState {
|
||||
focus: string | null;
|
||||
until: Date | null;
|
||||
lastCheckpoint: string | null;
|
||||
recentCheckpoints?: readonly string[];
|
||||
nextStep: string | null;
|
||||
stepCount: number;
|
||||
}
|
||||
|
||||
const MAX_RECENT_CHECKPOINT_TRAIL = 5;
|
||||
|
||||
function normalizeCheckpointText(text: string | null | undefined): string {
|
||||
return (text ?? "").replace(/\s+/g, " ").trim().toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* The prompt of each continuation run: the order in the user's words and where the work stands,
|
||||
* so a fresh run picks up without the whole history.
|
||||
*/
|
||||
export function autopilotContinuationPrompt(state: AutopilotState, now = new Date()): string {
|
||||
const trail = (state.recentCheckpoints ?? [])
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean)
|
||||
.slice(-MAX_RECENT_CHECKPOINT_TRAIL);
|
||||
return [
|
||||
"Autopilot: continue the user's order on your own, one sensible step at a time.",
|
||||
`Order: ${state.goal}`,
|
||||
state.focus ? `Focus: ${state.focus}` : null,
|
||||
state.until ? `Until: ${state.until.toISOString()} (now ${now.toISOString()})` : "Until: done",
|
||||
`Steps so far: ${state.stepCount}`,
|
||||
trail.length > 1
|
||||
? `Recent checkpoints: ${trail.map((item, idx) => `${idx + 1}. ${item}`).join(" | ")}`
|
||||
: null,
|
||||
state.lastCheckpoint ? `Last checkpoint: ${state.lastCheckpoint}` : null,
|
||||
state.nextStep ? `Planned next: ${state.nextStep}` : null,
|
||||
"Do the next step, then call autopilot_checkpoint with what changed. When the order is done, call autopilot_finish with the result. Actions that need the user's approval are parked as suggestions; continue with other work.",
|
||||
@ -36,10 +50,20 @@ export function autopilotContinuationPrompt(state: AutopilotState, now = new Dat
|
||||
|
||||
/** Whether the run that just ended moved the autopilot on, and if it should pause now. */
|
||||
export function advanceAutopilotGuard(
|
||||
counts: { idleCount: number; failCount: number },
|
||||
run: { checkpointed: boolean; failed: boolean },
|
||||
): { idleCount: number; failCount: number; pause: "idle" | "failing" | null } {
|
||||
const idleCount = run.checkpointed ? 0 : counts.idleCount + 1;
|
||||
counts: { idleCount: number; failCount: number; lastCheckpointText?: string | null },
|
||||
run: { checkpointed: boolean; failed: boolean; checkpointText?: string | null },
|
||||
): {
|
||||
idleCount: number;
|
||||
failCount: number;
|
||||
pause: "idle" | "failing" | null;
|
||||
lastCheckpointText?: string | null;
|
||||
} {
|
||||
const prevNorm = normalizeCheckpointText(counts.lastCheckpointText);
|
||||
const nextNorm = normalizeCheckpointText(run.checkpointText);
|
||||
const repeatedSameCheckpoint =
|
||||
run.checkpointed && Boolean(prevNorm && nextNorm && prevNorm === nextNorm);
|
||||
const effectiveProgress = run.checkpointed && !repeatedSameCheckpoint;
|
||||
const idleCount = effectiveProgress ? 0 : counts.idleCount + 1;
|
||||
const failCount = run.failed ? counts.failCount + 1 : 0;
|
||||
const pause =
|
||||
failCount >= AUTOPILOT_FAIL_LIMIT
|
||||
@ -47,7 +71,18 @@ export function advanceAutopilotGuard(
|
||||
: idleCount >= AUTOPILOT_IDLE_LIMIT
|
||||
? "idle"
|
||||
: null;
|
||||
return { idleCount, failCount, pause };
|
||||
const result: {
|
||||
idleCount: number;
|
||||
failCount: number;
|
||||
pause: "idle" | "failing" | null;
|
||||
lastCheckpointText?: string | null;
|
||||
} = { idleCount, failCount, pause };
|
||||
if (run.checkpointText !== undefined || counts.lastCheckpointText !== undefined) {
|
||||
result.lastCheckpointText = run.checkpointed
|
||||
? (run.checkpointText ?? counts.lastCheckpointText ?? null)
|
||||
: (counts.lastCheckpointText ?? null);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** The autopilot's time is up. */
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { browserUserContextLine, maskBrowserUserDetail } from "./browser-cowork.js";
|
||||
import {
|
||||
browserUserContextLine,
|
||||
coalesceBrowserUserActions,
|
||||
maskBrowserUserDetail,
|
||||
} from "./browser-cowork.js";
|
||||
|
||||
describe("browser cowork context", () => {
|
||||
it("masks password-like labels and fields", () => {
|
||||
@ -13,4 +17,24 @@ describe("browser cowork context", () => {
|
||||
);
|
||||
expect(browserUserContextLine("type", "hello", true)).toBe("User typed in a password field");
|
||||
});
|
||||
|
||||
it("coalesces repeated scrolls, typing bursts, and navigate-to-page_change echoes", () => {
|
||||
const lines = coalesceBrowserUserActions([
|
||||
{ kind: "navigate", detail: "https://example.com/login" },
|
||||
{ kind: "page_change", detail: "https://example.com/login" },
|
||||
{ kind: "scroll", detail: "down" },
|
||||
{ kind: "scroll", detail: "down" },
|
||||
{ kind: "scroll", detail: "down" },
|
||||
{ kind: "type", detail: "al" },
|
||||
{ kind: "type", detail: "ice" },
|
||||
{ kind: "type", detail: "pw1", passwordField: true },
|
||||
{ kind: "type", detail: "pw2", passwordField: true },
|
||||
]);
|
||||
expect(lines).toEqual([
|
||||
"User navigated to https://example.com/login",
|
||||
"User scrolled down (3x)",
|
||||
"User typed alice",
|
||||
"User typed in a password field",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@ -33,3 +33,75 @@ export function browserUserContextLine(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export type BrowserUserActionEvent = {
|
||||
kind: BrowserUserActionKind;
|
||||
detail: string;
|
||||
passwordField?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Coalesces noisy browser cowork event streams (repeated scrolls, consecutive typing bursts,
|
||||
* or a navigate immediately echoed by a matching page_change) into concise episode lines.
|
||||
*/
|
||||
export function coalesceBrowserUserActions(events: readonly BrowserUserActionEvent[]): string[] {
|
||||
const lines: string[] = [];
|
||||
let index = 0;
|
||||
|
||||
while (index < events.length) {
|
||||
const current = events[index]!;
|
||||
|
||||
if (current.kind === "scroll") {
|
||||
const dir = current.detail.trim() || "down";
|
||||
let count = 1;
|
||||
while (
|
||||
index + count < events.length &&
|
||||
events[index + count]?.kind === "scroll" &&
|
||||
(events[index + count]?.detail.trim() || "down") === dir
|
||||
) {
|
||||
count += 1;
|
||||
}
|
||||
const base = browserUserContextLine("scroll", dir, false);
|
||||
lines.push(count > 1 ? `${base} (${count}x)` : base);
|
||||
index += count;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current.kind === "type") {
|
||||
const isPassword = Boolean(current.passwordField);
|
||||
const chunks = [current.detail];
|
||||
let count = 1;
|
||||
while (
|
||||
index + count < events.length &&
|
||||
events[index + count]?.kind === "type" &&
|
||||
Boolean(events[index + count]?.passwordField) === isPassword
|
||||
) {
|
||||
chunks.push(events[index + count]!.detail);
|
||||
count += 1;
|
||||
}
|
||||
lines.push(browserUserContextLine("type", chunks.join(""), isPassword));
|
||||
index += count;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
current.kind === "navigate" &&
|
||||
index + 1 < events.length &&
|
||||
events[index + 1]?.kind === "page_change" &&
|
||||
events[index + 1]?.detail.trim() === current.detail.trim()
|
||||
) {
|
||||
lines.push(
|
||||
browserUserContextLine("navigate", current.detail, Boolean(current.passwordField)),
|
||||
);
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
lines.push(
|
||||
browserUserContextLine(current.kind, current.detail, Boolean(current.passwordField)),
|
||||
);
|
||||
index += 1;
|
||||
}
|
||||
|
||||
return lines;
|
||||
}
|
||||
|
||||
@ -11,7 +11,7 @@ export type GitShellPolicyDecision =
|
||||
|
||||
const GIT_PUSH_RE = /\bgit\b[^\n;&|]*\bpush\b/i;
|
||||
const GIT_FORCE_PUSH_RE = /(?:--force-with-lease|--force(?:\s|$)|(?:^|\s)-f(?:\s|$))/i;
|
||||
const GH_PR_RE = /\bgh\b(?:(?:\s+[^;&|]+)*\s+pr\b|\s+pr\b)(?:\s+create\b|\s|$|[;&|])/i;
|
||||
const GH_PR_RE = /\bgh\b[^\n;&|]*\bpr\b/i;
|
||||
|
||||
export function evaluateGitShellPolicy(input: GitShellPolicyInput): GitShellPolicyDecision {
|
||||
const command = input.command.trim();
|
||||
|
||||
@ -14,14 +14,17 @@ describe("search helpers", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("matches case-insensitive substrings", () => {
|
||||
it("matches case-insensitive substrings and multi-token queries", () => {
|
||||
expect(matchesSearchQuery("Venue", "Research venues")).toBe(true);
|
||||
expect(matchesSearchQuery("research berlin", "Research top venues in Berlin")).toBe(true);
|
||||
expect(matchesSearchQuery("research tokyo", "Research top venues in Berlin")).toBe(false);
|
||||
expect(matchesSearchQuery("missing", "Research venues")).toBe(false);
|
||||
});
|
||||
|
||||
it("builds a short snippet around the query", () => {
|
||||
it("builds a short snippet around the query or first matching token", () => {
|
||||
const text = "alpha beta gamma delta epsilon zeta eta theta";
|
||||
expect(snippetAroundMatch(text, "gamma", 20)).toContain("gamma");
|
||||
expect(snippetAroundMatch(text, "zeta alpha", 25)).toContain("zeta");
|
||||
});
|
||||
|
||||
it("uses groupId for group search message jumps", () => {
|
||||
|
||||
@ -5,22 +5,42 @@ export function extractLinksFromText(text: string): string[] {
|
||||
return [...new Set(matches.map((url) => url.replace(/[.,;:!?)]+$/, "")))];
|
||||
}
|
||||
|
||||
function tokenizeSearchWords(query: string): string[] {
|
||||
const tokens: string[] = [];
|
||||
for (const match of query.toLowerCase().matchAll(/[\p{L}\p{N}_-]+/gu)) {
|
||||
if (match[0]) tokens.push(match[0]);
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
export function matchesSearchQuery(
|
||||
query: string,
|
||||
...fields: Array<string | undefined | null>
|
||||
): boolean {
|
||||
const q = query.trim().toLowerCase();
|
||||
if (!q) return false;
|
||||
return fields.some((field) => field?.toLowerCase().includes(q));
|
||||
const lowerFields = fields
|
||||
.filter((field): field is string => typeof field === "string" && field.length > 0)
|
||||
.map((field) => field.toLowerCase());
|
||||
if (lowerFields.some((field) => field.includes(q))) return true;
|
||||
const tokens = tokenizeSearchWords(q);
|
||||
if (tokens.length <= 1) return false;
|
||||
const combined = lowerFields.join(" ");
|
||||
return tokens.every((token) => combined.includes(token));
|
||||
}
|
||||
|
||||
export function snippetAroundMatch(text: string, query: string, maxLen = 120): string {
|
||||
const q = query.trim().toLowerCase();
|
||||
if (!q) return text.slice(0, maxLen);
|
||||
const lower = text.toLowerCase();
|
||||
const index = lower.indexOf(q);
|
||||
let index = lower.indexOf(q);
|
||||
if (index < 0) {
|
||||
const firstToken = tokenizeSearchWords(q).find((token) => lower.includes(token));
|
||||
if (firstToken) index = lower.indexOf(firstToken);
|
||||
}
|
||||
if (index < 0) return text.slice(0, maxLen);
|
||||
const start = Math.max(0, index - 40);
|
||||
const leadIn = Math.min(40, Math.floor(maxLen / 3));
|
||||
const start = Math.max(0, index - leadIn);
|
||||
const end = Math.min(text.length, start + maxLen);
|
||||
const prefix = start > 0 ? "…" : "";
|
||||
const suffix = end < text.length ? "…" : "";
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user