feat(web,core,adapters): unify right sidebar styling, hover resize handles, voice & search improvements
This commit is contained in:
parent
75423579b7
commit
35b7291660
@ -1,6 +1,7 @@
|
|||||||
import type { Actor, MessageBlock, SearchHit } from "@rakazo/contracts";
|
import type { Actor, MessageBlock, SearchHit } from "@rakazo/contracts";
|
||||||
import { extractLinksFromText, matchesSearchQuery, snippetAroundMatch } from "@rakazo/core";
|
import { extractLinksFromText, matchesSearchQuery, snippetAroundMatch } from "@rakazo/core";
|
||||||
import { Prisma, type PrismaClient } from "@rakazo/db";
|
import type { PrismaClient } from "@rakazo/db";
|
||||||
|
import { Prisma } from "@rakazo/db";
|
||||||
|
|
||||||
const SEARCH_LIMIT = 25;
|
const SEARCH_LIMIT = 25;
|
||||||
/** Cap name matches so content hits (messages/files/links/routines) keep most of the budget. */
|
/** Cap name matches so content hits (messages/files/links/routines) keep most of the budget. */
|
||||||
@ -64,7 +65,8 @@ export async function querySpaceSearch(
|
|||||||
bucket.push(hit);
|
bucket.push(hit);
|
||||||
}
|
}
|
||||||
|
|
||||||
const bots = await prisma.bot.findMany({
|
const [bots, groups, artifacts, groupArtifacts, routines] = await Promise.all([
|
||||||
|
prisma.bot.findMany({
|
||||||
where: {
|
where: {
|
||||||
spaceId: actor.spaceId,
|
spaceId: actor.spaceId,
|
||||||
userId: actor.userId,
|
userId: actor.userId,
|
||||||
@ -76,7 +78,52 @@ export async function querySpaceSearch(
|
|||||||
],
|
],
|
||||||
},
|
},
|
||||||
take: CONVERSATION_HIT_LIMIT,
|
take: CONVERSATION_HIT_LIMIT,
|
||||||
});
|
}),
|
||||||
|
prisma.chatGroup.findMany({
|
||||||
|
where: {
|
||||||
|
spaceId: actor.spaceId,
|
||||||
|
userId: actor.userId,
|
||||||
|
name: { contains: query, mode: "insensitive" },
|
||||||
|
},
|
||||||
|
take: CONVERSATION_HIT_LIMIT,
|
||||||
|
}),
|
||||||
|
prisma.artifact.findMany({
|
||||||
|
where: {
|
||||||
|
spaceId: actor.spaceId,
|
||||||
|
userId: actor.userId,
|
||||||
|
groupId: null,
|
||||||
|
botId: { not: null },
|
||||||
|
name: { contains: query, mode: "insensitive" },
|
||||||
|
bot: { archivedAt: null },
|
||||||
|
},
|
||||||
|
include: { bot: { select: { name: true } } },
|
||||||
|
take: SEARCH_LIMIT,
|
||||||
|
}),
|
||||||
|
prisma.artifact.findMany({
|
||||||
|
where: {
|
||||||
|
spaceId: actor.spaceId,
|
||||||
|
userId: actor.userId,
|
||||||
|
groupId: { not: null },
|
||||||
|
name: { contains: query, mode: "insensitive" },
|
||||||
|
},
|
||||||
|
include: { group: { select: { name: true } } },
|
||||||
|
take: SEARCH_LIMIT,
|
||||||
|
}),
|
||||||
|
prisma.routine.findMany({
|
||||||
|
where: {
|
||||||
|
spaceId: actor.spaceId,
|
||||||
|
userId: actor.userId,
|
||||||
|
OR: [
|
||||||
|
{ name: { contains: query, mode: "insensitive" } },
|
||||||
|
{ prompt: { contains: query, mode: "insensitive" } },
|
||||||
|
],
|
||||||
|
bot: { archivedAt: null },
|
||||||
|
},
|
||||||
|
include: { bot: { select: { name: true } } },
|
||||||
|
take: SEARCH_LIMIT,
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
for (const bot of bots) {
|
for (const bot of bots) {
|
||||||
pushInto(
|
pushInto(
|
||||||
conversationHits,
|
conversationHits,
|
||||||
@ -91,14 +138,6 @@ export async function querySpaceSearch(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const groups = await prisma.chatGroup.findMany({
|
|
||||||
where: {
|
|
||||||
spaceId: actor.spaceId,
|
|
||||||
userId: actor.userId,
|
|
||||||
name: { contains: query, mode: "insensitive" },
|
|
||||||
},
|
|
||||||
take: CONVERSATION_HIT_LIMIT,
|
|
||||||
});
|
|
||||||
for (const group of groups) {
|
for (const group of groups) {
|
||||||
pushInto(
|
pushInto(
|
||||||
conversationHits,
|
conversationHits,
|
||||||
@ -114,27 +153,72 @@ export async function querySpaceSearch(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const contentBudget = Math.max(0, SEARCH_LIMIT - conversationHits.length);
|
const contentBudget = Math.max(0, SEARCH_LIMIT - conversationHits.length);
|
||||||
|
const pattern = `%${query}%`;
|
||||||
|
|
||||||
const artifacts = await prisma.artifact.findMany({
|
const [botArtifactMessages, groupArtifactMessages, messageRows, groupMessageRows] =
|
||||||
where: {
|
await Promise.all([
|
||||||
spaceId: actor.spaceId,
|
findArtifactMessages(
|
||||||
userId: actor.userId,
|
|
||||||
groupId: null,
|
|
||||||
botId: { not: null },
|
|
||||||
name: { contains: query, mode: "insensitive" },
|
|
||||||
bot: { archivedAt: null },
|
|
||||||
},
|
|
||||||
include: { bot: { select: { name: true } } },
|
|
||||||
take: SEARCH_LIMIT,
|
|
||||||
});
|
|
||||||
const botArtifactMessages = await findArtifactMessages(
|
|
||||||
prisma,
|
prisma,
|
||||||
actor,
|
actor,
|
||||||
artifacts.flatMap((artifact) =>
|
artifacts.flatMap((artifact) =>
|
||||||
artifact.botId && artifact.bot ? [{ id: artifact.id, targetId: artifact.botId }] : [],
|
artifact.botId && artifact.bot ? [{ id: artifact.id, targetId: artifact.botId }] : [],
|
||||||
),
|
),
|
||||||
"bot",
|
"bot",
|
||||||
);
|
),
|
||||||
|
findArtifactMessages(
|
||||||
|
prisma,
|
||||||
|
actor,
|
||||||
|
groupArtifacts.flatMap((artifact) =>
|
||||||
|
artifact.groupId && artifact.group
|
||||||
|
? [{ id: artifact.id, targetId: artifact.groupId }]
|
||||||
|
: [],
|
||||||
|
),
|
||||||
|
"group",
|
||||||
|
),
|
||||||
|
prisma.$queryRaw<
|
||||||
|
Array<{
|
||||||
|
id: string;
|
||||||
|
threadId: string;
|
||||||
|
seq: number;
|
||||||
|
blocks: Prisma.JsonValue;
|
||||||
|
botId: string;
|
||||||
|
botName: string;
|
||||||
|
}>
|
||||||
|
>`
|
||||||
|
SELECT m.id, m."threadId", m.seq, m.blocks, b.id AS "botId", b.name AS "botName"
|
||||||
|
FROM messages m
|
||||||
|
INNER JOIN threads t ON t.id = m."threadId"
|
||||||
|
INNER JOIN bots b ON b.id = t."botId"
|
||||||
|
WHERE t."spaceId" = ${actor.spaceId}
|
||||||
|
AND t."userId" = ${actor.userId}
|
||||||
|
AND b."archivedAt" IS NULL
|
||||||
|
AND m.blocks::text ILIKE ${pattern}
|
||||||
|
ORDER BY m."createdAt" DESC
|
||||||
|
LIMIT ${SEARCH_LIMIT}
|
||||||
|
`,
|
||||||
|
prisma.$queryRaw<
|
||||||
|
Array<{
|
||||||
|
id: string;
|
||||||
|
threadId: string;
|
||||||
|
seq: number;
|
||||||
|
blocks: Prisma.JsonValue;
|
||||||
|
groupId: string;
|
||||||
|
groupName: string;
|
||||||
|
}>
|
||||||
|
>`
|
||||||
|
SELECT m.id, m."threadId", m.seq, m.blocks, g.id AS "groupId", g.name AS "groupName"
|
||||||
|
FROM messages m
|
||||||
|
INNER JOIN threads t ON t.id = m."threadId"
|
||||||
|
INNER JOIN chat_groups g ON g.id = t."groupId"
|
||||||
|
WHERE t."spaceId" = ${actor.spaceId}
|
||||||
|
AND t."userId" = ${actor.userId}
|
||||||
|
AND t."groupId" IS NOT NULL
|
||||||
|
AND m.blocks::text ILIKE ${pattern}
|
||||||
|
ORDER BY m."createdAt" DESC
|
||||||
|
LIMIT ${SEARCH_LIMIT}
|
||||||
|
`,
|
||||||
|
]);
|
||||||
|
|
||||||
for (const artifact of artifacts) {
|
for (const artifact of artifacts) {
|
||||||
if (!artifact.botId || !artifact.bot) continue;
|
if (!artifact.botId || !artifact.bot) continue;
|
||||||
const message = botArtifactMessages.get(artifact.id);
|
const message = botArtifactMessages.get(artifact.id);
|
||||||
@ -155,24 +239,6 @@ export async function querySpaceSearch(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const groupArtifacts = await prisma.artifact.findMany({
|
|
||||||
where: {
|
|
||||||
spaceId: actor.spaceId,
|
|
||||||
userId: actor.userId,
|
|
||||||
groupId: { not: null },
|
|
||||||
name: { contains: query, mode: "insensitive" },
|
|
||||||
},
|
|
||||||
include: { group: { select: { name: true } } },
|
|
||||||
take: SEARCH_LIMIT,
|
|
||||||
});
|
|
||||||
const groupArtifactMessages = await findArtifactMessages(
|
|
||||||
prisma,
|
|
||||||
actor,
|
|
||||||
groupArtifacts.flatMap((artifact) =>
|
|
||||||
artifact.groupId && artifact.group ? [{ id: artifact.id, targetId: artifact.groupId }] : [],
|
|
||||||
),
|
|
||||||
"group",
|
|
||||||
);
|
|
||||||
for (const artifact of groupArtifacts) {
|
for (const artifact of groupArtifacts) {
|
||||||
if (!artifact.groupId || !artifact.group) continue;
|
if (!artifact.groupId || !artifact.group) continue;
|
||||||
const message = groupArtifactMessages.get(artifact.id);
|
const message = groupArtifactMessages.get(artifact.id);
|
||||||
@ -193,19 +259,6 @@ export async function querySpaceSearch(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const routines = await prisma.routine.findMany({
|
|
||||||
where: {
|
|
||||||
spaceId: actor.spaceId,
|
|
||||||
userId: actor.userId,
|
|
||||||
OR: [
|
|
||||||
{ name: { contains: query, mode: "insensitive" } },
|
|
||||||
{ prompt: { contains: query, mode: "insensitive" } },
|
|
||||||
],
|
|
||||||
bot: { archivedAt: null },
|
|
||||||
},
|
|
||||||
include: { bot: { select: { name: true } } },
|
|
||||||
take: SEARCH_LIMIT,
|
|
||||||
});
|
|
||||||
for (const routine of routines) {
|
for (const routine of routines) {
|
||||||
pushInto(
|
pushInto(
|
||||||
contentHits,
|
contentHits,
|
||||||
@ -221,29 +274,6 @@ export async function querySpaceSearch(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const pattern = `%${query}%`;
|
|
||||||
const messageRows = await prisma.$queryRaw<
|
|
||||||
Array<{
|
|
||||||
id: string;
|
|
||||||
threadId: string;
|
|
||||||
seq: number;
|
|
||||||
blocks: Prisma.JsonValue;
|
|
||||||
botId: string;
|
|
||||||
botName: string;
|
|
||||||
}>
|
|
||||||
>`
|
|
||||||
SELECT m.id, m."threadId", m.seq, m.blocks, b.id AS "botId", b.name AS "botName"
|
|
||||||
FROM messages m
|
|
||||||
INNER JOIN threads t ON t.id = m."threadId"
|
|
||||||
INNER JOIN bots b ON b.id = t."botId"
|
|
||||||
WHERE t."spaceId" = ${actor.spaceId}
|
|
||||||
AND t."userId" = ${actor.userId}
|
|
||||||
AND b."archivedAt" IS NULL
|
|
||||||
AND m.blocks::text ILIKE ${pattern}
|
|
||||||
ORDER BY m."createdAt" DESC
|
|
||||||
LIMIT ${SEARCH_LIMIT}
|
|
||||||
`;
|
|
||||||
|
|
||||||
for (const row of messageRows) {
|
for (const row of messageRows) {
|
||||||
pushMessageHits(row.blocks as MessageBlock[], {
|
pushMessageHits(row.blocks as MessageBlock[], {
|
||||||
botId: row.botId,
|
botId: row.botId,
|
||||||
@ -255,28 +285,6 @@ export async function querySpaceSearch(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const groupMessageRows = await prisma.$queryRaw<
|
|
||||||
Array<{
|
|
||||||
id: string;
|
|
||||||
threadId: string;
|
|
||||||
seq: number;
|
|
||||||
blocks: Prisma.JsonValue;
|
|
||||||
groupId: string;
|
|
||||||
groupName: string;
|
|
||||||
}>
|
|
||||||
>`
|
|
||||||
SELECT m.id, m."threadId", m.seq, m.blocks, g.id AS "groupId", g.name AS "groupName"
|
|
||||||
FROM messages m
|
|
||||||
INNER JOIN threads t ON t.id = m."threadId"
|
|
||||||
INNER JOIN chat_groups g ON g.id = t."groupId"
|
|
||||||
WHERE t."spaceId" = ${actor.spaceId}
|
|
||||||
AND t."userId" = ${actor.userId}
|
|
||||||
AND t."groupId" IS NOT NULL
|
|
||||||
AND m.blocks::text ILIKE ${pattern}
|
|
||||||
ORDER BY m."createdAt" DESC
|
|
||||||
LIMIT ${SEARCH_LIMIT}
|
|
||||||
`;
|
|
||||||
|
|
||||||
for (const row of groupMessageRows) {
|
for (const row of groupMessageRows) {
|
||||||
pushMessageHits(row.blocks as MessageBlock[], {
|
pushMessageHits(row.blocks as MessageBlock[], {
|
||||||
groupId: row.groupId,
|
groupId: row.groupId,
|
||||||
|
|||||||
@ -69,7 +69,7 @@ test("Oo browser shows live screen, cursor, cowork click, and user context", asy
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
await page.getByRole("button", { name: "Browser", exact: true }).click();
|
await page.getByTitle("Agent computer").click();
|
||||||
const browser = page.getByTestId("in-app-browser");
|
const browser = page.getByTestId("in-app-browser");
|
||||||
await expect(browser.getByTestId("oo-computer-browser-screen")).toHaveAttribute(
|
await expect(browser.getByTestId("oo-computer-browser-screen")).toHaveAttribute(
|
||||||
"src",
|
"src",
|
||||||
|
|||||||
@ -74,7 +74,7 @@ export function DualSurfaceDivider({
|
|||||||
onDoubleClick={handleDoubleClick}
|
onDoubleClick={handleDoubleClick}
|
||||||
onKeyDown={handleKeyDown}
|
onKeyDown={handleKeyDown}
|
||||||
className={cn(
|
className={cn(
|
||||||
"group relative flex w-3 md:w-4 shrink-0 cursor-col-resize select-none touch-none items-center justify-center transition-colors bg-border/0 hover:bg-accent/40 active:bg-primary/20 focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring",
|
"group relative flex w-3 md:w-3 shrink-0 cursor-col-resize select-none touch-none items-center justify-center transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring",
|
||||||
disabled && "pointer-events-none cursor-default opacity-50",
|
disabled && "pointer-events-none cursor-default opacity-50",
|
||||||
className,
|
className,
|
||||||
)}
|
)}
|
||||||
@ -82,8 +82,8 @@ export function DualSurfaceDivider({
|
|||||||
>
|
>
|
||||||
<div
|
<div
|
||||||
className={cn(
|
className={cn(
|
||||||
"h-full w-1 rounded-full bg-border transition-colors group-hover:bg-accent group-active:bg-primary",
|
"h-12 w-1 rounded-full bg-border opacity-0 transition-opacity group-hover:opacity-100 group-active:bg-primary group-active:opacity-100",
|
||||||
isDragging && "bg-primary",
|
isDragging && "bg-primary opacity-100",
|
||||||
)}
|
)}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@ -5,8 +5,8 @@
|
|||||||
|
|
||||||
type AudioContextCtor = new () => AudioContext;
|
type AudioContextCtor = new () => AudioContext;
|
||||||
|
|
||||||
/** Master output ~−15 LUFS for short procedural peaks. */
|
/** Master output ~−12 LUFS for clear, warm procedural peaks. */
|
||||||
const SFX_MASTER_GAIN = 0.58;
|
const SFX_MASTER_GAIN = 0.85;
|
||||||
/** Duck depth when voice lane active (TTS or mic wash). */
|
/** Duck depth when voice lane active (TTS or mic wash). */
|
||||||
const SFX_DUCK_RATIO = 0.28;
|
const SFX_DUCK_RATIO = 0.28;
|
||||||
const DUCK_RAMP_S = 0.065;
|
const DUCK_RAMP_S = 0.065;
|
||||||
@ -22,8 +22,10 @@ class OoAudioBus {
|
|||||||
private ctx: AudioContext | null = null;
|
private ctx: AudioContext | null = null;
|
||||||
private sfxGain: GainNode | null = null;
|
private sfxGain: GainNode | null = null;
|
||||||
private voiceHold = 0;
|
private voiceHold = 0;
|
||||||
|
private unlockBound = false;
|
||||||
|
|
||||||
ensureContext(): AudioContext | null {
|
ensureContext(): AudioContext | null {
|
||||||
|
this.bindGestureUnlock();
|
||||||
if (this.ctx && this.ctx.state !== "closed") return this.ctx;
|
if (this.ctx && this.ctx.state !== "closed") return this.ctx;
|
||||||
const Ctor = audioContextCtor();
|
const Ctor = audioContextCtor();
|
||||||
if (!Ctor) return null;
|
if (!Ctor) return null;
|
||||||
@ -43,6 +45,30 @@ class OoAudioBus {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Unlocks Web Audio on the user's first click, key, or tap so async SSE events can play audio. */
|
||||||
|
bindGestureUnlock(): void {
|
||||||
|
if (this.unlockBound || typeof window === "undefined" || typeof window.addEventListener !== "function") {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.unlockBound = true;
|
||||||
|
const unlock = () => {
|
||||||
|
const ctx = this.ensureContext();
|
||||||
|
if (ctx && ctx.state === "suspended") {
|
||||||
|
void ctx.resume().catch(() => undefined);
|
||||||
|
}
|
||||||
|
if (typeof window !== "undefined" && "speechSynthesis" in window) {
|
||||||
|
try {
|
||||||
|
window.speechSynthesis.getVoices?.();
|
||||||
|
} catch {
|
||||||
|
// Ignore voice preload errors.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
window.addEventListener("pointerdown", unlock, { capture: true, passive: true });
|
||||||
|
window.addEventListener("keydown", unlock, { capture: true, passive: true });
|
||||||
|
window.addEventListener("touchstart", unlock, { capture: true, passive: true });
|
||||||
|
}
|
||||||
|
|
||||||
/** Route for oo-sound / oo-vocals / oo-voice-mark oscillators. */
|
/** Route for oo-sound / oo-vocals / oo-voice-mark oscillators. */
|
||||||
sfxDestination(ctx: AudioContext): AudioNode {
|
sfxDestination(ctx: AudioContext): AudioNode {
|
||||||
if (ctx !== this.ctx) {
|
if (ctx !== this.ctx) {
|
||||||
|
|||||||
@ -68,61 +68,62 @@ function playCueOnContext(ctx: AudioContext, cue: OoSoundCue) {
|
|||||||
|
|
||||||
switch (cue) {
|
switch (cue) {
|
||||||
case "app-start":
|
case "app-start":
|
||||||
sweep(ctx, master, 523.25, 659.25, t0, 0.14, { gain: 0.038, type: "sine" });
|
sweep(ctx, master, 523.25, 659.25, t0, 0.16, { gain: 0.11, type: "sine" });
|
||||||
sweep(ctx, master, 659.25, 493.88, t0 + 0.1, 0.18, { gain: 0.032, type: "triangle" });
|
sweep(ctx, master, 659.25, 783.99, t0 + 0.1, 0.2, { gain: 0.095, type: "triangle" });
|
||||||
break;
|
break;
|
||||||
case "interact":
|
case "interact":
|
||||||
tone(ctx, master, 1046.5, t0, 0.06, {
|
tone(ctx, master, 880, t0, 0.075, {
|
||||||
gain: 0.018,
|
gain: 0.065,
|
||||||
attack: 0.003,
|
attack: 0.004,
|
||||||
release: 0.04,
|
release: 0.045,
|
||||||
type: "triangle",
|
type: "triangle",
|
||||||
});
|
});
|
||||||
break;
|
break;
|
||||||
case "success-resolve":
|
case "success-resolve":
|
||||||
case "work-complete":
|
case "work-complete":
|
||||||
// mint → gold resolve (~270 ms)
|
// mint → gold resolve (~270 ms)
|
||||||
sweep(ctx, master, 523.25, 659.25, t0, 0.12, { gain: 0.042, type: "sine" });
|
sweep(ctx, master, 523.25, 659.25, t0, 0.13, { gain: 0.12, type: "sine" });
|
||||||
sweep(ctx, master, 659.25, 783.99, t0 + 0.1, 0.22, { gain: 0.036, type: "triangle" });
|
sweep(ctx, master, 659.25, 783.99, t0 + 0.1, 0.22, { gain: 0.105, type: "triangle" });
|
||||||
tone(ctx, master, 880, t0 + 0.2, 0.28, { gain: 0.03, type: "sine" });
|
tone(ctx, master, 880, t0 + 0.2, 0.28, { gain: 0.09, type: "sine" });
|
||||||
break;
|
break;
|
||||||
case "need-you":
|
case "need-you":
|
||||||
sweep(ctx, master, 220, 330, t0, 0.1, { gain: 0.042, type: "triangle" });
|
sweep(ctx, master, 261.63, 392, t0, 0.12, { gain: 0.125, type: "triangle" });
|
||||||
tone(ctx, master, 277.18, t0 + 0.09, 0.12, { gain: 0.035, release: 0.08 });
|
tone(ctx, master, 523.25, t0 + 0.1, 0.18, { gain: 0.11, release: 0.1 });
|
||||||
break;
|
break;
|
||||||
case "listening-start":
|
case "listening-start":
|
||||||
case "voice_listen":
|
case "voice_listen":
|
||||||
tone(ctx, master, 220, t0, 0.22, { gain: 0.045, type: "triangle" });
|
tone(ctx, master, 293.66, t0, 0.2, { gain: 0.11, type: "triangle" });
|
||||||
tone(ctx, master, 330, t0 + 0.04, 0.28, { gain: 0.035 });
|
tone(ctx, master, 440, t0 + 0.05, 0.24, { gain: 0.095 });
|
||||||
break;
|
break;
|
||||||
case "listening-stop":
|
case "listening-stop":
|
||||||
tone(ctx, master, 330, t0, 0.18, { gain: 0.03, release: 0.12 });
|
tone(ctx, master, 392, t0, 0.16, { gain: 0.085, release: 0.1 });
|
||||||
|
tone(ctx, master, 523.25, t0 + 0.06, 0.18, { gain: 0.075, release: 0.11 });
|
||||||
break;
|
break;
|
||||||
case "speaking-start":
|
case "speaking-start":
|
||||||
tone(ctx, master, 440, t0, 0.2, { gain: 0.04, type: "sine" });
|
sweep(ctx, master, 392, 523.25, t0, 0.14, { gain: 0.095, type: "sine" });
|
||||||
break;
|
break;
|
||||||
case "speaking-done":
|
case "speaking-done":
|
||||||
tone(ctx, master, 523.25, t0, 0.24, { gain: 0.035 });
|
tone(ctx, master, 523.25, t0, 0.2, { gain: 0.085 });
|
||||||
tone(ctx, master, 392, t0 + 0.08, 0.3, { gain: 0.028, release: 0.14 });
|
tone(ctx, master, 392, t0 + 0.08, 0.26, { gain: 0.075, release: 0.14 });
|
||||||
break;
|
break;
|
||||||
case "voice_bargein":
|
case "voice_bargein":
|
||||||
sweep(ctx, master, 523.25, 261.63, t0, 0.05, { gain: 0.032, type: "sine" });
|
sweep(ctx, master, 523.25, 261.63, t0, 0.07, { gain: 0.09, type: "sine" });
|
||||||
break;
|
break;
|
||||||
case "attention":
|
case "attention":
|
||||||
tone(ctx, master, 277.18, t0, 0.16, { gain: 0.05, type: "triangle" });
|
tone(ctx, master, 329.63, t0, 0.16, { gain: 0.13, type: "triangle" });
|
||||||
tone(ctx, master, 233.08, t0 + 0.12, 0.22, { gain: 0.045 });
|
tone(ctx, master, 261.63, t0 + 0.12, 0.22, { gain: 0.115 });
|
||||||
break;
|
break;
|
||||||
case "message-sent":
|
case "message-sent":
|
||||||
tone(ctx, master, 880, t0, 0.1, { gain: 0.022, attack: 0.004, release: 0.06 });
|
sweep(ctx, master, 587.33, 880, t0, 0.11, { gain: 0.08, type: "sine" });
|
||||||
break;
|
break;
|
||||||
case "handoff":
|
case "handoff":
|
||||||
tone(ctx, master, 349.23, t0, 0.2, { gain: 0.04 });
|
tone(ctx, master, 392, t0, 0.18, { gain: 0.11 });
|
||||||
tone(ctx, master, 466.16, t0 + 0.1, 0.24, { gain: 0.035, type: "triangle" });
|
tone(ctx, master, 523.25, t0 + 0.09, 0.22, { gain: 0.1, type: "triangle" });
|
||||||
break;
|
break;
|
||||||
case "group-spark":
|
case "group-spark":
|
||||||
tone(ctx, master, 587.33, t0, 0.11, { gain: 0.032, attack: 0.004, type: "triangle" });
|
tone(ctx, master, 587.33, t0, 0.11, { gain: 0.095, attack: 0.004, type: "triangle" });
|
||||||
tone(ctx, master, 739.99, t0 + 0.06, 0.14, { gain: 0.028, type: "sine" });
|
tone(ctx, master, 739.99, t0 + 0.06, 0.14, { gain: 0.085, type: "sine" });
|
||||||
tone(ctx, master, 880, t0 + 0.12, 0.1, { gain: 0.022, release: 0.05 });
|
tone(ctx, master, 880, t0 + 0.12, 0.12, { gain: 0.075, release: 0.06 });
|
||||||
break;
|
break;
|
||||||
case "routine_start":
|
case "routine_start":
|
||||||
tone(ctx, master, 392, t0, 0.08, { gain: 0.025, attack: 0.005, release: 0.03, type: "sine" });
|
tone(ctx, master, 392, t0, 0.08, { gain: 0.025, attack: 0.005, release: 0.03, type: "sine" });
|
||||||
|
|||||||
@ -3,8 +3,8 @@ import { readOoSoundEnabled } from "./oo-sound-pref";
|
|||||||
|
|
||||||
export type OoVocalReaction = "ouh" | "mhm" | "aaah" | "mm" | "nod-down" | "nod-up" | "hm";
|
export type OoVocalReaction = "ouh" | "mhm" | "aaah" | "mm" | "nod-down" | "nod-up" | "hm";
|
||||||
|
|
||||||
const COOLDOWN_MIN_MS = 8_000;
|
const COOLDOWN_MIN_MS = 1_600;
|
||||||
const COOLDOWN_MAX_MS = 12_000;
|
const COOLDOWN_MAX_MS = 2_800;
|
||||||
|
|
||||||
let lastPlayedAt = 0;
|
let lastPlayedAt = 0;
|
||||||
let nextCooldownMs = COOLDOWN_MIN_MS;
|
let nextCooldownMs = COOLDOWN_MIN_MS;
|
||||||
@ -28,17 +28,23 @@ function vocalFormant(
|
|||||||
) {
|
) {
|
||||||
const osc = ctx.createOscillator();
|
const osc = ctx.createOscillator();
|
||||||
const gain = ctx.createGain();
|
const gain = ctx.createGain();
|
||||||
|
const boostedPeak = Math.min(0.22, gainPeak * 3.2);
|
||||||
|
osc.type = "sine";
|
||||||
|
osc.frequency.setValueAtTime(baseHz, startAt);
|
||||||
|
osc.frequency.exponentialRampToValueAtTime(baseHz * 1.12, startAt + duration * 0.75);
|
||||||
|
gain.gain.setValueAtTime(0.0001, startAt);
|
||||||
|
gain.gain.exponentialRampToValueAtTime(boostedPeak, startAt + 0.02);
|
||||||
|
gain.gain.exponentialRampToValueAtTime(0.0001, startAt + duration);
|
||||||
|
if (typeof ctx.createBiquadFilter === "function") {
|
||||||
const filter = ctx.createBiquadFilter();
|
const filter = ctx.createBiquadFilter();
|
||||||
filter.type = "bandpass";
|
filter.type = "bandpass";
|
||||||
filter.frequency.value = baseHz * 2.4;
|
filter.frequency.value = baseHz * 1.6;
|
||||||
filter.Q.value = 4.5;
|
filter.Q.value = 1.8;
|
||||||
osc.type = "triangle";
|
|
||||||
osc.frequency.setValueAtTime(baseHz, startAt);
|
|
||||||
gain.gain.setValueAtTime(0.0001, startAt);
|
|
||||||
gain.gain.exponentialRampToValueAtTime(gainPeak, startAt + 0.025);
|
|
||||||
gain.gain.exponentialRampToValueAtTime(0.0001, startAt + duration);
|
|
||||||
osc.connect(filter);
|
osc.connect(filter);
|
||||||
filter.connect(gain);
|
filter.connect(gain);
|
||||||
|
} else {
|
||||||
|
osc.connect(gain);
|
||||||
|
}
|
||||||
gain.connect(destination);
|
gain.connect(destination);
|
||||||
osc.start(startAt);
|
osc.start(startAt);
|
||||||
osc.stop(startAt + duration + 0.02);
|
osc.stop(startAt + duration + 0.02);
|
||||||
|
|||||||
@ -215,6 +215,53 @@ describe("Speaker", () => {
|
|||||||
expect(speaker.state.status).toBe("idle");
|
expect(speaker.state.status).toBe("idle");
|
||||||
expect(speaker.state.error).toBeUndefined();
|
expect(speaker.state.error).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("filters out novelty voices and selects a natural German voice for German text", async () => {
|
||||||
|
stubSelectedSpace("space-voice");
|
||||||
|
const speak = vi.fn();
|
||||||
|
const cancel = vi.fn();
|
||||||
|
const eddy = { name: "Eddy (German (Germany))", lang: "de-DE", localService: true };
|
||||||
|
const grandma = { name: "Grandma (German (Germany))", lang: "de-DE", localService: true };
|
||||||
|
const anna = { name: "Anna", lang: "de-DE", localService: true };
|
||||||
|
vi.stubGlobal(
|
||||||
|
"SpeechSynthesisUtterance",
|
||||||
|
class {
|
||||||
|
onend: (() => void) | null = null;
|
||||||
|
onerror: (() => void) | null = null;
|
||||||
|
voice: unknown = null;
|
||||||
|
lang = "";
|
||||||
|
rate = 1;
|
||||||
|
pitch = 1;
|
||||||
|
constructor(public text: string) {}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
vi.stubGlobal("window", {
|
||||||
|
speechSynthesis: {
|
||||||
|
cancel,
|
||||||
|
getVoices: () => [eddy, grandma, anna],
|
||||||
|
speak,
|
||||||
|
},
|
||||||
|
localStorage: globalThis.localStorage,
|
||||||
|
});
|
||||||
|
|
||||||
|
const speaker = new Speaker();
|
||||||
|
vi.spyOn(
|
||||||
|
speaker as unknown as { prepare: () => Promise<string[]> },
|
||||||
|
"prepare",
|
||||||
|
).mockRejectedValue(new Error("offline"));
|
||||||
|
|
||||||
|
const pending = speaker.speak("Ich habe das Projekt für dich aktualisiert.");
|
||||||
|
await vi.waitFor(() => expect(speak).toHaveBeenCalledOnce());
|
||||||
|
const utterance = speak.mock.calls[0]?.[0] as {
|
||||||
|
voice?: { name: string };
|
||||||
|
lang?: string;
|
||||||
|
onend?: () => void;
|
||||||
|
};
|
||||||
|
expect(utterance.lang).toBe("de-DE");
|
||||||
|
expect(utterance.voice?.name).toBe("Anna");
|
||||||
|
utterance.onend?.();
|
||||||
|
await pending;
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("UtteranceStreamer", () => {
|
describe("UtteranceStreamer", () => {
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
import { ClauseSegmenter, readBoundedResponseBytes, speakable } from "@rakazo/core";
|
import { ClauseSegmenter, readBoundedResponseBytes, speakable } from "@rakazo/core";
|
||||||
|
import { ooAudioBus } from "./oo-audio-bus.js";
|
||||||
import { rpc, selectedSpaceId, withSpaceHeaders } from "./rpc.js";
|
import { rpc, selectedSpaceId, withSpaceHeaders } from "./rpc.js";
|
||||||
import { readUiLocale } from "./ui-locale.js";
|
import { readUiLocale } from "./ui-locale.js";
|
||||||
|
|
||||||
@ -77,7 +78,19 @@ export class Speaker {
|
|||||||
if (this.snapshot.status !== "idle" || this.snapshot.error) this.set(IDLE);
|
if (this.snapshot.status !== "idle" || this.snapshot.error) this.set(IDLE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private bufferSource: AudioBufferSourceNode | null = null;
|
||||||
|
|
||||||
private teardownAudio() {
|
private teardownAudio() {
|
||||||
|
if (this.bufferSource) {
|
||||||
|
try {
|
||||||
|
this.bufferSource.onended = null;
|
||||||
|
this.bufferSource.stop();
|
||||||
|
this.bufferSource.disconnect();
|
||||||
|
} catch {
|
||||||
|
// Ignore already-stopped buffer source errors.
|
||||||
|
}
|
||||||
|
this.bufferSource = null;
|
||||||
|
}
|
||||||
if (this.audio) {
|
if (this.audio) {
|
||||||
this.audio.pause();
|
this.audio.pause();
|
||||||
this.audio.src = "";
|
this.audio.src = "";
|
||||||
@ -226,14 +239,49 @@ export class Speaker {
|
|||||||
audio.onended = null;
|
audio.onended = null;
|
||||||
audio.onerror = null;
|
audio.onerror = null;
|
||||||
if (this.settlePlayback === done) this.settlePlayback = null;
|
if (this.settlePlayback === done) this.settlePlayback = null;
|
||||||
if (this.audio === audio) this.teardownAudio();
|
if (this.audio === audio || this.bufferSource) this.teardownAudio();
|
||||||
resolve(ok);
|
resolve(ok);
|
||||||
};
|
};
|
||||||
this.settlePlayback = done;
|
this.settlePlayback = done;
|
||||||
audio.onended = () => done(true);
|
audio.onended = () => done(true);
|
||||||
audio.onerror = () => done(false);
|
audio.onerror = () => {
|
||||||
audio.play().catch(() => done(false));
|
void this.playViaWebAudio(blob, live).then(done);
|
||||||
|
};
|
||||||
|
audio.play().catch(() => {
|
||||||
|
void this.playViaWebAudio(blob, live).then(done);
|
||||||
});
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async playViaWebAudio(blob: Blob, live: () => boolean): Promise<boolean> {
|
||||||
|
if (!live()) return false;
|
||||||
|
const ctx = ooAudioBus.ensureContext();
|
||||||
|
if (
|
||||||
|
!ctx ||
|
||||||
|
typeof ctx.decodeAudioData !== "function" ||
|
||||||
|
typeof ctx.createBufferSource !== "function"
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const arrayBuffer = await blob.arrayBuffer();
|
||||||
|
if (!live()) return false;
|
||||||
|
const audioBuffer = await ctx.decodeAudioData(arrayBuffer.slice(0));
|
||||||
|
if (!live()) return false;
|
||||||
|
return await new Promise<boolean>((resolve) => {
|
||||||
|
const source = ctx.createBufferSource();
|
||||||
|
source.buffer = audioBuffer;
|
||||||
|
source.connect(ctx.destination);
|
||||||
|
this.bufferSource = source;
|
||||||
|
source.onended = () => {
|
||||||
|
if (this.bufferSource === source) this.bufferSource = null;
|
||||||
|
resolve(true);
|
||||||
|
};
|
||||||
|
source.start(0);
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async tryBrowserFallback(
|
async tryBrowserFallback(
|
||||||
@ -256,6 +304,7 @@ export class Speaker {
|
|||||||
return new Promise((resolve) => {
|
return new Promise((resolve) => {
|
||||||
try {
|
try {
|
||||||
window.speechSynthesis.cancel();
|
window.speechSynthesis.cancel();
|
||||||
|
window.speechSynthesis.resume?.();
|
||||||
const utterance = new SpeechSynthesisUtterance(text);
|
const utterance = new SpeechSynthesisUtterance(text);
|
||||||
const voices = window.speechSynthesis.getVoices?.() ?? [];
|
const voices = window.speechSynthesis.getVoices?.() ?? [];
|
||||||
const uiLocale = readUiLocale();
|
const uiLocale = readUiLocale();
|
||||||
@ -269,21 +318,26 @@ export class Speaker {
|
|||||||
const langPrefix = isGerman ? "de" : uiLocale.split("-")[0] || "en";
|
const langPrefix = isGerman ? "de" : uiLocale.split("-")[0] || "en";
|
||||||
utterance.lang = isGerman ? "de-DE" : langPrefix === "en" ? "en-US" : uiLocale;
|
utterance.lang = isGerman ? "de-DE" : langPrefix === "en" ? "en-US" : uiLocale;
|
||||||
|
|
||||||
const langVoices = voices.filter((v) =>
|
const allLangVoices = voices.filter((v) =>
|
||||||
v.lang.toLowerCase().replace(/_/g, "-").startsWith(langPrefix),
|
(v.lang ?? "").toLowerCase().replace(/_/g, "-").startsWith(langPrefix),
|
||||||
);
|
);
|
||||||
|
const noveltyHint =
|
||||||
|
/\b(?:eddy|flo|grandma|grandpa|reed|rocko|sandy|shelley|albert|bad news|bahh|bells|boing|bubbles|cellos|deranged|good news|hysterical|junior|organ|superstar|trinoids|whisper|zarvox|fred|kathy|ralph)\b/i;
|
||||||
|
const naturalLangVoices = allLangVoices.filter((v) => !noveltyHint.test(v.name ?? ""));
|
||||||
|
const langVoices = naturalLangVoices.length > 0 ? naturalLangVoices : allLangVoices;
|
||||||
const wantsMale = /^dm_/i.test(voiceId);
|
const wantsMale = /^dm_/i.test(voiceId);
|
||||||
const wantsFemale = /^df_/i.test(voiceId);
|
const wantsFemale = /^df_/i.test(voiceId) || (!wantsMale && isGerman);
|
||||||
const maleHint = /yannick|martin|markus|stefan|viktor|hans|daniel|thomas|male/i;
|
const maleHint =
|
||||||
const femaleHint = /anna|petra|helena|marlene|vicki|katja|siri|eloquence|female/i;
|
/yannick|martin|markus|stefan|viktor|hans|daniel|thomas|florian|conrad|male/i;
|
||||||
|
const femaleHint = /anna|petra|helena|marlene|vicki|katja|seraphina|amala|siri|female/i;
|
||||||
|
|
||||||
const matchedVoice =
|
const matchedVoice =
|
||||||
(wantsMale &&
|
(wantsMale &&
|
||||||
(langVoices.find((v) => !v.localService && maleHint.test(v.name)) ||
|
(langVoices.find((v) => !v.localService && maleHint.test(v.name ?? "")) ||
|
||||||
langVoices.find((v) => maleHint.test(v.name)))) ||
|
langVoices.find((v) => maleHint.test(v.name ?? "")))) ||
|
||||||
(wantsFemale &&
|
(wantsFemale &&
|
||||||
(langVoices.find((v) => !v.localService && femaleHint.test(v.name)) ||
|
(langVoices.find((v) => !v.localService && femaleHint.test(v.name ?? "")) ||
|
||||||
langVoices.find((v) => femaleHint.test(v.name)))) ||
|
langVoices.find((v) => femaleHint.test(v.name ?? "")))) ||
|
||||||
langVoices.find((v) => !v.localService) ||
|
langVoices.find((v) => !v.localService) ||
|
||||||
langVoices[0] ||
|
langVoices[0] ||
|
||||||
voices[0];
|
voices[0];
|
||||||
|
|||||||
@ -75,7 +75,6 @@ import {
|
|||||||
ArrowLeft,
|
ArrowLeft,
|
||||||
ArrowRight,
|
ArrowRight,
|
||||||
Calendar,
|
Calendar,
|
||||||
Globe,
|
|
||||||
PanelRight,
|
PanelRight,
|
||||||
Pencil,
|
Pencil,
|
||||||
Plus,
|
Plus,
|
||||||
@ -1228,17 +1227,41 @@ export function ShellPage() {
|
|||||||
|
|
||||||
const { agentSkills, setAgentSkills, refreshAgentSkills } = useAgentSkills();
|
const { agentSkills, setAgentSkills, refreshAgentSkills } = useAgentSkills();
|
||||||
|
|
||||||
|
const resolveBotVoiceId = useCallback(
|
||||||
|
(botId: string | undefined, currentVoiceId?: string | null): string => {
|
||||||
|
if (currentVoiceId && /^d[fm]_/i.test(currentVoiceId)) return currentVoiceId;
|
||||||
|
const germanVoices = [
|
||||||
|
"df_eva",
|
||||||
|
"dm_thorsten",
|
||||||
|
"df_ramona",
|
||||||
|
"dm_karlsson",
|
||||||
|
"df_kerstin",
|
||||||
|
] as const;
|
||||||
|
if (!botId) return "df_eva";
|
||||||
|
let hash = 0;
|
||||||
|
for (let i = 0; i < botId.length; i += 1) {
|
||||||
|
hash = (hash * 31 + botId.charCodeAt(i)) >>> 0;
|
||||||
|
}
|
||||||
|
return germanVoices[hash % germanVoices.length] ?? "df_eva";
|
||||||
|
},
|
||||||
|
[],
|
||||||
|
);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
void rpc.voice
|
void rpc.voice
|
||||||
.status()
|
.status()
|
||||||
.then(async (status) => {
|
.then(async (status) => {
|
||||||
if (!status.configured) {
|
if (!status.configured) {
|
||||||
const defaultVoiceId =
|
|
||||||
typeof navigator !== "undefined" && /^de\b/i.test(navigator.language)
|
|
||||||
? "df_eva"
|
|
||||||
: "af_heart";
|
|
||||||
await rpc.voice
|
await rpc.voice
|
||||||
.connect({ provider: "kokoro", apiKey: "local-kokoro", voiceId: defaultVoiceId })
|
.connect({ provider: "kokoro", apiKey: "local-kokoro", voiceId: "df_eva" })
|
||||||
|
.catch(() => undefined);
|
||||||
|
const refreshed = await rpc.voice.status().catch(() => status);
|
||||||
|
setVoiceStatus(refreshed);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!status.voiceId || /^a[fm]_/i.test(status.voiceId)) {
|
||||||
|
await rpc.voice
|
||||||
|
.setVoice({ provider: status.provider ?? "kokoro", voiceId: "df_eva" })
|
||||||
.catch(() => undefined);
|
.catch(() => undefined);
|
||||||
const refreshed = await rpc.voice.status().catch(() => status);
|
const refreshed = await rpc.voice.status().catch(() => status);
|
||||||
setVoiceStatus(refreshed);
|
setVoiceStatus(refreshed);
|
||||||
@ -1255,6 +1278,34 @@ export function ShellPage() {
|
|||||||
};
|
};
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!bots.length) return;
|
||||||
|
const migratedKey = "rakazo:voice-de-default-v2";
|
||||||
|
try {
|
||||||
|
if (localStorage.getItem(migratedKey) === "1") return;
|
||||||
|
localStorage.setItem(migratedKey, "1");
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const staleBots = bots.filter(
|
||||||
|
(b) => b.autoSpeak === false || !b.voiceId || /^a[fm]_/i.test(b.voiceId),
|
||||||
|
);
|
||||||
|
if (!staleBots.length) return;
|
||||||
|
void Promise.all(
|
||||||
|
staleBots.map((b) =>
|
||||||
|
rpc.bots
|
||||||
|
.update({
|
||||||
|
botId: b.id,
|
||||||
|
autoSpeak: true,
|
||||||
|
voiceId: resolveBotVoiceId(b.id, b.voiceId),
|
||||||
|
})
|
||||||
|
.catch(() => undefined),
|
||||||
|
),
|
||||||
|
).then(() => {
|
||||||
|
void refreshBots().catch(() => undefined);
|
||||||
|
});
|
||||||
|
}, [bots, refreshBots, resolveBotVoiceId]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const currentSnapshot = inGroup
|
const currentSnapshot = inGroup
|
||||||
? snapshot?.groupId === groupId
|
? snapshot?.groupId === groupId
|
||||||
@ -1296,7 +1347,8 @@ export function ShellPage() {
|
|||||||
const text = spokenSummaryFromBlocks(lastBot.blocks);
|
const text = spokenSummaryFromBlocks(lastBot.blocks);
|
||||||
if (!text) return;
|
if (!text) return;
|
||||||
autoSpoken.current = lastBot.id;
|
autoSpoken.current = lastBot.id;
|
||||||
void speaker.speak(text, { botId: speakerBotId, messageId: lastBot.id });
|
const voiceId = resolveBotVoiceId(speakerBotId, speakerBot?.voiceId ?? active?.voiceId);
|
||||||
|
void speaker.speak(text, { botId: speakerBotId, voiceId, messageId: lastBot.id });
|
||||||
}, [
|
}, [
|
||||||
snapshot,
|
snapshot,
|
||||||
inGroup,
|
inGroup,
|
||||||
@ -1306,6 +1358,7 @@ export function ShellPage() {
|
|||||||
coordinatorBot?.id,
|
coordinatorBot?.id,
|
||||||
bots,
|
bots,
|
||||||
callOpen,
|
callOpen,
|
||||||
|
resolveBotVoiceId,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@ -2953,9 +3006,11 @@ export function ShellPage() {
|
|||||||
}
|
}
|
||||||
const text = spokenSummaryFromBlocks(message.blocks) || speechFromBlocks(message.blocks);
|
const text = spokenSummaryFromBlocks(message.blocks) || speechFromBlocks(message.blocks);
|
||||||
const id = message.botId ?? activeBotId.current ?? defaultSpeakerId ?? coordinatorBot?.id;
|
const id = message.botId ?? activeBotId.current ?? defaultSpeakerId ?? coordinatorBot?.id;
|
||||||
if (text && id) void speaker.speak(text, { botId: id, messageId: message.id });
|
const msgBot = id ? bots.find((b) => b.id === id) : undefined;
|
||||||
|
const voiceId = resolveBotVoiceId(id, msgBot?.voiceId);
|
||||||
|
if (text && id) void speaker.speak(text, { botId: id, voiceId, messageId: message.id });
|
||||||
},
|
},
|
||||||
[coordinatorBot?.id, defaultSpeakerId],
|
[bots, coordinatorBot?.id, defaultSpeakerId, resolveBotVoiceId],
|
||||||
);
|
);
|
||||||
|
|
||||||
async function createGroup(input: { name: string; botIds: string[] }) {
|
async function createGroup(input: { name: string; botIds: string[] }) {
|
||||||
@ -3322,8 +3377,16 @@ export function ShellPage() {
|
|||||||
{!botsSidebarCollapsed ? (
|
{!botsSidebarCollapsed ? (
|
||||||
<div
|
<div
|
||||||
onPointerDown={startResizingLeft}
|
onPointerDown={startResizingLeft}
|
||||||
className="absolute -right-2 top-0 bottom-0 w-4 cursor-col-resize z-50 hidden md:block"
|
className="group absolute -right-3 top-0 bottom-0 w-3 cursor-col-resize z-50 hidden md:flex items-center justify-center"
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
className={`h-12 w-1 rounded-full bg-border transition-opacity ${
|
||||||
|
isResizingLeft
|
||||||
|
? "bg-primary opacity-100"
|
||||||
|
: "opacity-0 group-hover:opacity-100 group-active:bg-primary group-active:opacity-100"
|
||||||
|
}`}
|
||||||
/>
|
/>
|
||||||
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
<SidebarHeader
|
<SidebarHeader
|
||||||
activityMode={activityMode}
|
activityMode={activityMode}
|
||||||
@ -3816,7 +3879,7 @@ export function ShellPage() {
|
|||||||
minWidth: 0,
|
minWidth: 0,
|
||||||
flexShrink: 1,
|
flexShrink: 1,
|
||||||
transition: "width 200ms ease, opacity 150ms ease",
|
transition: "width 200ms ease, opacity 150ms ease",
|
||||||
overflow: showCanvasOnDesktop ? ("visible" as const) : ("hidden" as const),
|
overflow: "hidden" as const,
|
||||||
}
|
}
|
||||||
: {
|
: {
|
||||||
width:
|
width:
|
||||||
@ -3860,7 +3923,7 @@ export function ShellPage() {
|
|||||||
aria-hidden={mobileSidebarOpen || undefined}
|
aria-hidden={mobileSidebarOpen || undefined}
|
||||||
inert={mobileSidebarOpen}
|
inert={mobileSidebarOpen}
|
||||||
style={streamStyle}
|
style={streamStyle}
|
||||||
className="flex min-w-0 flex-1 flex-col bg-background md:rounded-2xl md:border md:border-border md:shadow-2xl overflow-hidden transition-all duration-300 ease-[cubic-bezier(0.16,1,0.3,1)]"
|
className="relative flex min-w-0 flex-1 flex-col bg-background md:rounded-2xl md:border md:border-border md:shadow-2xl overflow-hidden transition-all duration-300 ease-[cubic-bezier(0.16,1,0.3,1)]"
|
||||||
>
|
>
|
||||||
{activeNavPillar === "heute" ? (
|
{activeNavPillar === "heute" ? (
|
||||||
<div className="flex h-full flex-1 flex-col overflow-hidden">
|
<div className="flex h-full flex-1 flex-col overflow-hidden">
|
||||||
@ -4057,29 +4120,6 @@ export function ShellPage() {
|
|||||||
disabled={dualSurface.isMobile}
|
disabled={dualSurface.isMobile}
|
||||||
className="hidden md:flex"
|
className="hidden md:flex"
|
||||||
/>
|
/>
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
title={t`Browser`}
|
|
||||||
aria-label={t`Browser`}
|
|
||||||
onClick={() => {
|
|
||||||
const isBrowserOpen =
|
|
||||||
panel === "browser" && dualSurface.layoutMode !== "stream";
|
|
||||||
if (isBrowserOpen) {
|
|
||||||
setPanel(null);
|
|
||||||
dualSurface.setLayoutMode("stream");
|
|
||||||
} else {
|
|
||||||
setCanvasTab("browser");
|
|
||||||
setPanel("browser");
|
|
||||||
dualSurface.setLayoutMode("split");
|
|
||||||
}
|
|
||||||
}}
|
|
||||||
data-active={
|
|
||||||
panel === "browser" && dualSurface.layoutMode !== "stream" ? "" : undefined
|
|
||||||
}
|
|
||||||
className="app-no-drag rk-touch-target grid size-control-sm min-h-control-sm min-w-control-sm place-items-center rounded-lg hover:bg-accent data-active:bg-accent"
|
|
||||||
>
|
|
||||||
<Globe strokeWidth={1.7} className="text-foreground/75 size-icon-sm" />
|
|
||||||
</button>
|
|
||||||
{active || groupComputerBotId ? (
|
{active || groupComputerBotId ? (
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
@ -4216,6 +4256,7 @@ export function ShellPage() {
|
|||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
|
<div className="relative flex min-h-0 flex-1 flex-col overflow-hidden">
|
||||||
<Transcript
|
<Transcript
|
||||||
key={activeSnapshot?.threadId}
|
key={activeSnapshot?.threadId}
|
||||||
encounterMode={encounterMode}
|
encounterMode={encounterMode}
|
||||||
@ -4261,19 +4302,11 @@ export function ShellPage() {
|
|||||||
onSendQueuedNow={sendQueuedMessageNow}
|
onSendQueuedNow={sendQueuedMessageNow}
|
||||||
onRemoveQueuedMessage={removeQueuedMessage}
|
onRemoveQueuedMessage={removeQueuedMessage}
|
||||||
/>
|
/>
|
||||||
</>
|
|
||||||
)}
|
|
||||||
{activeGroup?.kind === "main" && !inProject && !showProjectTimeline ? (
|
|
||||||
<MainInbox
|
|
||||||
bots={bots}
|
|
||||||
onOpenBot={openBot}
|
|
||||||
onOpenGroup={openGroup}
|
|
||||||
onOpenInSpace={openSpaceChat}
|
|
||||||
/>
|
|
||||||
) : null}
|
|
||||||
{recordingSkill ? (
|
{recordingSkill ? (
|
||||||
<div className="px-6 pb-2 text-center text-ui-body-sm text-destructive">
|
<div className="pointer-events-none absolute inset-x-0 bottom-24 z-30 px-6 pb-2 text-center text-ui-body-sm text-destructive">
|
||||||
<Trans>Teaching in progress. Stop teaching before sending a new message.</Trans>
|
<Trans>
|
||||||
|
Teaching in progress. Stop teaching before sending a new message.
|
||||||
|
</Trans>
|
||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
{(active || activeGroup) && !showProjectTimeline ? (
|
{(active || activeGroup) && !showProjectTimeline ? (
|
||||||
@ -4337,6 +4370,17 @@ export function ShellPage() {
|
|||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
{activeGroup?.kind === "main" && !inProject && !showProjectTimeline ? (
|
||||||
|
<MainInbox
|
||||||
|
bots={bots}
|
||||||
|
onOpenBot={openBot}
|
||||||
|
onOpenGroup={openGroup}
|
||||||
|
onOpenInSpace={openSpaceChat}
|
||||||
|
/>
|
||||||
|
) : null}
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
</main>
|
</main>
|
||||||
@ -4366,10 +4410,10 @@ export function ShellPage() {
|
|||||||
data-panel={panel ?? "closed"}
|
data-panel={panel ?? "closed"}
|
||||||
data-auto={panel !== null && panel === autoPanel ? "" : undefined}
|
data-auto={panel !== null && panel === autoPanel ? "" : undefined}
|
||||||
style={canvasStyle}
|
style={canvasStyle}
|
||||||
className={`absolute inset-0 z-30 flex min-h-0 shrink-0 flex-col overflow-hidden bg-background md:relative md:inset-auto md:z-20 md:border-0 md:rounded-none md:shadow-none ${
|
className={`absolute inset-0 z-30 flex min-h-0 shrink-0 flex-col overflow-hidden bg-sidebar md:relative md:inset-auto md:z-20 md:rounded-2xl md:border md:border-sidebar-border md:shadow-2xl ${
|
||||||
isPanelVisible
|
isPanelVisible
|
||||||
? "w-full max-w-none md:max-w-none opacity-100"
|
? "w-full max-w-none md:max-w-none opacity-100"
|
||||||
: "pointer-events-none w-0 opacity-0 md:opacity-0 md:w-0"
|
: "pointer-events-none w-0 opacity-0 md:opacity-0 md:w-0 md:border-0"
|
||||||
} ${!(workspacePaneOpen ? isResizingWorkspacePane : isResizingRight) ? "transition-[width,opacity] duration-300 ease-[cubic-bezier(0.16,1,0.3,1)] motion-reduce:transition-none" : ""}`}
|
} ${!(workspacePaneOpen ? isResizingWorkspacePane : isResizingRight) ? "transition-[width,opacity] duration-300 ease-[cubic-bezier(0.16,1,0.3,1)] motion-reduce:transition-none" : ""}`}
|
||||||
>
|
>
|
||||||
<div
|
<div
|
||||||
|
|||||||
@ -452,11 +452,11 @@ export const Composer = memo(function Composer({
|
|||||||
onDragOver={handleDragOver}
|
onDragOver={handleDragOver}
|
||||||
onDragLeave={handleDragLeave}
|
onDragLeave={handleDragLeave}
|
||||||
onDrop={handleDrop}
|
onDrop={handleDrop}
|
||||||
className={`relative z-30 m-0 min-w-0 max-w-full overflow-x-hidden border-0 bg-gradient-to-t from-background via-background/80 to-transparent px-3 pb-4 pt-3 md:px-6 md:pb-6 ${
|
className={`composer-bottom-fade pointer-events-none absolute inset-x-0 bottom-0 z-30 m-0 min-w-0 max-w-full overflow-x-hidden border-0 px-3 pb-4 pt-10 md:px-6 md:pb-6 md:pt-12 ${
|
||||||
draggingFiles ? "rounded-[14px] ring-2 ring-inset ring-ring" : ""
|
draggingFiles ? "rounded-[14px] ring-2 ring-inset ring-ring" : ""
|
||||||
}`}
|
}`}
|
||||||
>
|
>
|
||||||
<div className="mx-auto w-full max-w-4xl">
|
<div className="pointer-events-auto mx-auto w-full max-w-4xl">
|
||||||
{onConnectModel ? (
|
{onConnectModel ? (
|
||||||
<div
|
<div
|
||||||
data-testid="composer-model-needed"
|
data-testid="composer-model-needed"
|
||||||
|
|||||||
@ -629,8 +629,8 @@ export const Transcript = memo(function Transcript({
|
|||||||
</button>
|
</button>
|
||||||
) : null}
|
) : null}
|
||||||
<div
|
<div
|
||||||
className={`mx-auto flex w-full flex-1 flex-col pb-6 ${
|
className={`mx-auto flex w-full flex-1 flex-col ${
|
||||||
encounterMode ? "max-w-2xl gap-3" : "max-w-4xl gap-6 pb-8"
|
encounterMode ? "max-w-2xl gap-3 pb-28 md:pb-32" : "max-w-4xl gap-6 pb-28 md:pb-32"
|
||||||
}`}
|
}`}
|
||||||
>
|
>
|
||||||
{coalescedMessages.map((message) => {
|
{coalescedMessages.map((message) => {
|
||||||
@ -883,7 +883,7 @@ export const Transcript = memo(function Transcript({
|
|||||||
aria-hidden={!showJumpToLatest}
|
aria-hidden={!showJumpToLatest}
|
||||||
tabIndex={showJumpToLatest ? 0 : -1}
|
tabIndex={showJumpToLatest ? 0 : -1}
|
||||||
onClick={jumpToLatest}
|
onClick={jumpToLatest}
|
||||||
className={`absolute bottom-4 left-1/2 z-20 grid h-9 w-9 -translate-x-1/2 place-items-center rounded-full border border-border bg-muted/95 text-foreground/75 shadow-md backdrop-blur transition-[opacity,transform,background-color] duration-200 ease-[cubic-bezier(.22,1,.36,1)] hover:bg-border motion-reduce:transition-none ${
|
className={`absolute bottom-24 left-1/2 z-20 grid h-9 w-9 -translate-x-1/2 place-items-center rounded-full border border-border bg-muted/95 text-foreground/75 shadow-md backdrop-blur transition-[opacity,transform,background-color] duration-200 ease-[cubic-bezier(.22,1,.36,1)] hover:bg-border md:bottom-28 motion-reduce:transition-none ${
|
||||||
showJumpToLatest
|
showJumpToLatest
|
||||||
? "translate-y-0 opacity-100"
|
? "translate-y-0 opacity-100"
|
||||||
: "pointer-events-none translate-y-2 opacity-0"
|
: "pointer-events-none translate-y-2 opacity-0"
|
||||||
@ -896,7 +896,7 @@ export const Transcript = memo(function Transcript({
|
|||||||
type="button"
|
type="button"
|
||||||
data-testid="needs-you-jump"
|
data-testid="needs-you-jump"
|
||||||
onClick={jumpToAsk}
|
onClick={jumpToAsk}
|
||||||
className="absolute bottom-4 left-1/2 z-20 inline-flex h-9 -translate-x-1/2 items-center gap-1.5 rounded-full border border-border bg-muted/95 px-3.5 text-ui-body-sm font-medium text-foreground shadow-md backdrop-blur hover:bg-border"
|
className="absolute bottom-24 left-1/2 z-20 inline-flex h-9 -translate-x-1/2 items-center gap-1.5 rounded-full border border-border bg-muted/95 px-3.5 text-ui-body-sm font-medium text-foreground shadow-md backdrop-blur hover:bg-border md:bottom-28"
|
||||||
>
|
>
|
||||||
<span aria-hidden="true" className="size-2 rounded-full bg-warning" />
|
<span aria-hidden="true" className="size-2 rounded-full bg-warning" />
|
||||||
{t`Needs you`}
|
{t`Needs you`}
|
||||||
|
|||||||
@ -1,9 +1,11 @@
|
|||||||
import type { Bot, ThreadSnapshot } from "@rakazo/contracts";
|
import type { Bot, ThreadSnapshot } from "@rakazo/contracts";
|
||||||
import { useEffect, useRef } from "react";
|
import { useEffect, useRef } from "react";
|
||||||
|
import { ooAudioBus } from "../../lib/oo-audio-bus";
|
||||||
import { wireOoAudioVoiceDucking } from "../../lib/oo-audio-voice-wire";
|
import { wireOoAudioVoiceDucking } from "../../lib/oo-audio-voice-wire";
|
||||||
import { ooSound } from "../../lib/oo-sound";
|
import { ooSound } from "../../lib/oo-sound";
|
||||||
import { ooVocals } from "../../lib/oo-vocals";
|
import { ooVocals } from "../../lib/oo-vocals";
|
||||||
import { ooVoiceMark } from "../../lib/oo-voice-mark";
|
import { ooVoiceMark } from "../../lib/oo-voice-mark";
|
||||||
|
import { speaker } from "../../lib/tts";
|
||||||
|
|
||||||
export type ShellSoundsInput = {
|
export type ShellSoundsInput = {
|
||||||
bots: Bot[];
|
bots: Bot[];
|
||||||
@ -27,8 +29,25 @@ export function useShellSounds({
|
|||||||
const seenHandoffKeys = useRef(new Set<string>());
|
const seenHandoffKeys = useRef(new Set<string>());
|
||||||
const appSoundStarted = useRef(false);
|
const appSoundStarted = useRef(false);
|
||||||
const prevBotStatuses = useRef(new Map<string, string>());
|
const prevBotStatuses = useRef(new Map<string, string>());
|
||||||
|
const prevThreadId = useRef<string | null>(null);
|
||||||
|
const prevBotMsgCount = useRef<number>(0);
|
||||||
|
|
||||||
useEffect(() => wireOoAudioVoiceDucking(), []);
|
useEffect(() => {
|
||||||
|
ooAudioBus.bindGestureUnlock();
|
||||||
|
return wireOoAudioVoiceDucking();
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let prevStatus = speaker.state.status;
|
||||||
|
return speaker.subscribe((state) => {
|
||||||
|
if (prevStatus !== "speaking" && state.status === "speaking" && !callOpen) {
|
||||||
|
ooSound.play("speaking-start");
|
||||||
|
} else if (prevStatus === "speaking" && state.status === "idle" && !callOpen) {
|
||||||
|
ooSound.play("speaking-done");
|
||||||
|
}
|
||||||
|
prevStatus = state.status;
|
||||||
|
});
|
||||||
|
}, [callOpen]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (appSoundStarted.current) return;
|
if (appSoundStarted.current) return;
|
||||||
@ -37,6 +56,23 @@ export function useShellSounds({
|
|||||||
window.setTimeout(() => ooVoiceMark.maybePlayColdOpen(), 280);
|
window.setTimeout(() => ooVoiceMark.maybePlayColdOpen(), 280);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const threadId = activeSnapshot?.threadId ?? null;
|
||||||
|
if (!threadId) return;
|
||||||
|
if (prevThreadId.current && prevThreadId.current !== threadId) {
|
||||||
|
ooSound.play("interact");
|
||||||
|
ooVocals.reactForEvent("nod");
|
||||||
|
prevBotMsgCount.current = (activeSnapshot?.messages ?? []).filter(
|
||||||
|
(m) => m.role === "bot",
|
||||||
|
).length;
|
||||||
|
} else if (!prevThreadId.current) {
|
||||||
|
prevBotMsgCount.current = (activeSnapshot?.messages ?? []).filter(
|
||||||
|
(m) => m.role === "bot",
|
||||||
|
).length;
|
||||||
|
}
|
||||||
|
prevThreadId.current = threadId;
|
||||||
|
}, [activeSnapshot?.threadId, activeSnapshot?.messages]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
for (const bot of bots) {
|
for (const bot of bots) {
|
||||||
const prev = prevBotStatuses.current.get(bot.id);
|
const prev = prevBotStatuses.current.get(bot.id);
|
||||||
@ -52,13 +88,27 @@ export function useShellSounds({
|
|||||||
}, [bots]);
|
}, [bots]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (prevTranscriptRunning.current && !transcriptRunning && !callOpen) {
|
if (!prevTranscriptRunning.current && transcriptRunning && !callOpen) {
|
||||||
|
ooSound.play("listening-stop");
|
||||||
|
ooVocals.reactForEvent("curious");
|
||||||
|
} else if (prevTranscriptRunning.current && !transcriptRunning && !callOpen) {
|
||||||
ooSound.play("success-resolve");
|
ooSound.play("success-resolve");
|
||||||
ooVocals.reactForEvent("success");
|
ooVocals.reactForEvent("success");
|
||||||
}
|
}
|
||||||
prevTranscriptRunning.current = transcriptRunning;
|
prevTranscriptRunning.current = transcriptRunning;
|
||||||
}, [transcriptRunning, callOpen]);
|
}, [transcriptRunning, callOpen]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const botMsgs = (activeSnapshot?.messages ?? []).filter((m) => m.role === "bot").length;
|
||||||
|
if (prevThreadId.current === (activeSnapshot?.threadId ?? null) && botMsgs > prevBotMsgCount.current) {
|
||||||
|
if (transcriptRunning && !callOpen) {
|
||||||
|
ooSound.play("interact");
|
||||||
|
ooVocals.react("mhm");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
prevBotMsgCount.current = botMsgs;
|
||||||
|
}, [activeSnapshot?.messages, activeSnapshot?.threadId, transcriptRunning, callOpen]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const runId = displayedRunErrorId;
|
const runId = displayedRunErrorId;
|
||||||
if (runId && runId !== prevRunErrorId.current) {
|
if (runId && runId !== prevRunErrorId.current) {
|
||||||
|
|||||||
@ -34,15 +34,25 @@ body,
|
|||||||
}
|
}
|
||||||
|
|
||||||
.apple-island {
|
.apple-island {
|
||||||
backdrop-filter: blur(36px) saturate(210%);
|
backdrop-filter: blur(16px) saturate(180%);
|
||||||
-webkit-backdrop-filter: blur(36px) saturate(210%);
|
-webkit-backdrop-filter: blur(16px) saturate(180%);
|
||||||
background: transparent;
|
background: color-mix(in srgb, var(--background) 22%, transparent);
|
||||||
border: 1px solid color-mix(in srgb, var(--foreground) 13%, transparent);
|
border: 1px solid color-mix(in srgb, var(--foreground) 13%, transparent);
|
||||||
box-shadow:
|
box-shadow:
|
||||||
0 18px 44px -8px rgba(0, 0, 0, 0.75),
|
0 18px 44px -8px rgba(0, 0, 0, 0.75),
|
||||||
inset 0 1px 0 rgba(255, 255, 255, 0.16);
|
inset 0 1px 0 rgba(255, 255, 255, 0.16);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.composer-bottom-fade {
|
||||||
|
background: linear-gradient(
|
||||||
|
to top,
|
||||||
|
var(--background) 0%,
|
||||||
|
color-mix(in srgb, var(--background) 78%, transparent) 32%,
|
||||||
|
color-mix(in srgb, var(--background) 32%, transparent) 68%,
|
||||||
|
transparent 100%
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
.apple-glass-card {
|
.apple-glass-card {
|
||||||
backdrop-filter: blur(20px);
|
backdrop-filter: blur(20px);
|
||||||
-webkit-backdrop-filter: blur(20px);
|
-webkit-backdrop-filter: blur(20px);
|
||||||
|
|||||||
@ -45,4 +45,23 @@ describe("git shell policy", () => {
|
|||||||
it("parses refspec destinations", () => {
|
it("parses refspec destinations", () => {
|
||||||
expect(parseGitPushRef("git push origin HEAD:feature/x")).toBe("feature/x");
|
expect(parseGitPushRef("git push origin HEAD:feature/x")).toBe("feature/x");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("evaluates multiline gh pr create commands with long markdown bodies in linear time", () => {
|
||||||
|
const command = [
|
||||||
|
"cd /home/rakazo/project && gh pr create --repo owner/repo \\",
|
||||||
|
" --base main \\",
|
||||||
|
" --head feature/domain-migration \\",
|
||||||
|
' --title "chore: migrate domain" \\',
|
||||||
|
` --body "${"## Section\n\n- bullet item with words and details\n".repeat(40)}" 2>&1`,
|
||||||
|
].join("\n");
|
||||||
|
const started = performance.now();
|
||||||
|
expect(
|
||||||
|
evaluateGitShellPolicy({
|
||||||
|
command,
|
||||||
|
defaultBranch: "main",
|
||||||
|
gitPushApprovedThisRun: false,
|
||||||
|
}),
|
||||||
|
).toMatchObject({ action: "require_approval" });
|
||||||
|
expect(performance.now() - started).toBeLessThan(50);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@ -9,7 +9,10 @@ import type {
|
|||||||
VoiceVerifyResult,
|
VoiceVerifyResult,
|
||||||
} from "@rakazo/adapter-kit";
|
} from "@rakazo/adapter-kit";
|
||||||
import { formatVoiceLabel } from "@rakazo/core";
|
import { formatVoiceLabel } from "@rakazo/core";
|
||||||
import { listPiperVoices, synthesizePiper } from "./piper-voice.js";
|
import { listPiperVoices, PIPER_MODELS, piperBaseUrl, synthesizePiper } from "./piper-voice.js";
|
||||||
|
|
||||||
|
const GERMAN_TEXT_RE =
|
||||||
|
/[äöüß]|(?:\b(?:und|der|die|das|ist|nicht|für|mit|den|dem|des|ein|eine|ich|wir|hier|bitte|oder|hallo|projekt|habe|sind|wird)\b)/i;
|
||||||
import {
|
import {
|
||||||
readVoiceAudio,
|
readVoiceAudio,
|
||||||
readVoiceJson,
|
readVoiceJson,
|
||||||
@ -112,7 +115,16 @@ export class KokoroVoiceProvider implements VoiceProvider {
|
|||||||
|
|
||||||
async synthesize(request: VoiceSynthesizeRequest, context: AdapterContext): Promise<SpeechClip> {
|
async synthesize(request: VoiceSynthesizeRequest, context: AdapterContext): Promise<SpeechClip> {
|
||||||
const signal = voiceDeadline(request.signal ?? context.signal, 120_000);
|
const signal = voiceDeadline(request.signal ?? context.signal, 120_000);
|
||||||
const piper = await synthesizePiper(request.voiceId, request.text, signal);
|
const effectiveVoiceId =
|
||||||
|
request.voiceId &&
|
||||||
|
!PIPER_MODELS[request.voiceId] &&
|
||||||
|
piperBaseUrl() &&
|
||||||
|
GERMAN_TEXT_RE.test(request.text)
|
||||||
|
? /^[ab]m_/i.test(request.voiceId)
|
||||||
|
? "dm_thorsten"
|
||||||
|
: "df_eva"
|
||||||
|
: request.voiceId;
|
||||||
|
const piper = await synthesizePiper(effectiveVoiceId, request.text, signal);
|
||||||
if (piper) return piper;
|
if (piper) return piper;
|
||||||
let res: Response;
|
let res: Response;
|
||||||
try {
|
try {
|
||||||
|
|||||||
@ -11,10 +11,15 @@ export const PIPER_MODELS: Record<string, string> = {
|
|||||||
dm_karlsson: "de_DE-karlsson-low",
|
dm_karlsson: "de_DE-karlsson-low",
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Piper runs only where configured, so a server without it skips the request entirely. */
|
/** Piper runs on 127.0.0.1:5150 by default outside unit tests, or at PIPER_BASE_URL. */
|
||||||
export function piperBaseUrl(): string | undefined {
|
export function piperBaseUrl(): string | undefined {
|
||||||
const raw = process.env.PIPER_BASE_URL?.trim();
|
const raw = process.env.PIPER_BASE_URL;
|
||||||
return raw ? raw.replace(/\/+$/, "") : undefined;
|
if (raw !== undefined) {
|
||||||
|
const trimmed = raw.trim();
|
||||||
|
return trimmed ? trimmed.replace(/\/+$/, "") : undefined;
|
||||||
|
}
|
||||||
|
if (process.env.VITEST || process.env.NODE_ENV === "test") return undefined;
|
||||||
|
return "http://127.0.0.1:5150";
|
||||||
}
|
}
|
||||||
|
|
||||||
export function piperUnreachable(): string {
|
export function piperUnreachable(): string {
|
||||||
@ -61,5 +66,7 @@ export async function synthesizePiper(
|
|||||||
throw new Error(piperUnreachable());
|
throw new Error(piperUnreachable());
|
||||||
}
|
}
|
||||||
await requireOk(res, "Local voice", "speaking");
|
await requireOk(res, "Local voice", "speaking");
|
||||||
return { bytes: await readVoiceAudio(res, signal), mimeType: "audio/wav" };
|
const contentType = res.headers.get("content-type");
|
||||||
|
const mimeType = contentType?.startsWith("audio/mpeg") ? "audio/mpeg" : "audio/wav";
|
||||||
|
return { bytes: await readVoiceAudio(res, signal), mimeType };
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,9 +1,11 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { ActionApprovalRule } from "./action-approval.js";
|
||||||
import {
|
import {
|
||||||
type ActionApprovalRule,
|
|
||||||
applyJudgeDecision,
|
applyJudgeDecision,
|
||||||
|
classifyApprovalRiskTier,
|
||||||
connectorKindFromToolName,
|
connectorKindFromToolName,
|
||||||
connectorToolRequiresApproval,
|
connectorToolRequiresApproval,
|
||||||
|
groupApprovalRequests,
|
||||||
isApprovalAskBlock,
|
isApprovalAskBlock,
|
||||||
isSecretAskBlock,
|
isSecretAskBlock,
|
||||||
planActionGate,
|
planActionGate,
|
||||||
@ -120,6 +122,52 @@ describe("isApprovalAskBlock", () => {
|
|||||||
}),
|
}),
|
||||||
).toBe(false);
|
).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("batches pending approvals by connector and risk tier while deduplicating identical effect keys", () => {
|
||||||
|
expect(classifyApprovalRiskTier("create_space", false)).toBe("explicit_security");
|
||||||
|
expect(classifyApprovalRiskTier("github_create_issue", true)).toBe("consequential_write");
|
||||||
|
expect(classifyApprovalRiskTier("github_get_issue", true)).toBe("standard");
|
||||||
|
|
||||||
|
const batches = groupApprovalRequests([
|
||||||
|
{
|
||||||
|
id: "req-1",
|
||||||
|
toolName: "github_create_issue",
|
||||||
|
viaConnector: true,
|
||||||
|
effectKey: "issue:bug-42",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "req-2",
|
||||||
|
toolName: "github_create_issue",
|
||||||
|
viaConnector: true,
|
||||||
|
effectKey: "issue:bug-42",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "req-3",
|
||||||
|
toolName: "github_add_comment",
|
||||||
|
viaConnector: true,
|
||||||
|
effectKey: "comment:bug-42",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "req-4",
|
||||||
|
toolName: "create_space",
|
||||||
|
viaConnector: false,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(batches).toHaveLength(2);
|
||||||
|
expect(batches[0]).toMatchObject({
|
||||||
|
groupKey: "github:consequential_write",
|
||||||
|
connectorKind: "github",
|
||||||
|
riskTier: "consequential_write",
|
||||||
|
deduplicatedCount: 1,
|
||||||
|
});
|
||||||
|
expect(batches[0]?.items.map((item) => item.id)).toEqual(["req-1", "req-3"]);
|
||||||
|
expect(batches[1]).toMatchObject({
|
||||||
|
groupKey: "create:explicit_security",
|
||||||
|
riskTier: "explicit_security",
|
||||||
|
deduplicatedCount: 0,
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("isSecretAskBlock", () => {
|
describe("isSecretAskBlock", () => {
|
||||||
|
|||||||
@ -248,3 +248,80 @@ export function isApprovalAskBlock(block: {
|
|||||||
[...actionIds].every((id) => id === "allow" || id === "always" || id === "deny")
|
[...actionIds].every((id) => id === "allow" || id === "always" || id === "deny")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type ApprovalRiskTier = "explicit_security" | "consequential_write" | "standard";
|
||||||
|
|
||||||
|
export type PendingApprovalCandidate = {
|
||||||
|
id: string;
|
||||||
|
toolName: string;
|
||||||
|
viaConnector: boolean;
|
||||||
|
connectorKind?: string;
|
||||||
|
effectKey?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ApprovalBatchGroup = {
|
||||||
|
groupKey: string;
|
||||||
|
connectorKind: string;
|
||||||
|
riskTier: ApprovalRiskTier;
|
||||||
|
items: PendingApprovalCandidate[];
|
||||||
|
deduplicatedCount: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function classifyApprovalRiskTier(
|
||||||
|
toolName: string,
|
||||||
|
viaConnector: boolean,
|
||||||
|
): ApprovalRiskTier {
|
||||||
|
if (toolRequiresExplicitApproval(toolName)) return "explicit_security";
|
||||||
|
if (toolRequiresApproval(toolName, viaConnector)) return "consequential_write";
|
||||||
|
return "standard";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Groups and deduplicates pending tool approval requests by connector and risk tier so the user
|
||||||
|
* can review related actions in a single coherent batch rather than repetitive interruptions.
|
||||||
|
*/
|
||||||
|
export function groupApprovalRequests(
|
||||||
|
candidates: readonly PendingApprovalCandidate[],
|
||||||
|
connectorKinds: string[] = [],
|
||||||
|
): ApprovalBatchGroup[] {
|
||||||
|
const groups = new Map<string, ApprovalBatchGroup>();
|
||||||
|
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
const connectorKind = (
|
||||||
|
candidate.connectorKind ?? connectorKindFromToolName(candidate.toolName, connectorKinds)
|
||||||
|
).toLowerCase();
|
||||||
|
const riskTier = classifyApprovalRiskTier(candidate.toolName, candidate.viaConnector);
|
||||||
|
const groupKey = `${connectorKind}:${riskTier}`;
|
||||||
|
|
||||||
|
let group = groups.get(groupKey);
|
||||||
|
if (!group) {
|
||||||
|
group = {
|
||||||
|
groupKey,
|
||||||
|
connectorKind,
|
||||||
|
riskTier,
|
||||||
|
items: [],
|
||||||
|
deduplicatedCount: 0,
|
||||||
|
};
|
||||||
|
groups.set(groupKey, group);
|
||||||
|
}
|
||||||
|
|
||||||
|
const dedupeKey = candidate.effectKey?.trim()
|
||||||
|
? `${candidate.toolName.toLowerCase()}::${candidate.effectKey.trim()}`
|
||||||
|
: null;
|
||||||
|
const isDuplicate =
|
||||||
|
dedupeKey !== null &&
|
||||||
|
group.items.some(
|
||||||
|
(existing) =>
|
||||||
|
existing.toolName.toLowerCase() === candidate.toolName.toLowerCase() &&
|
||||||
|
existing.effectKey?.trim() === candidate.effectKey?.trim(),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (isDuplicate) {
|
||||||
|
group.deduplicatedCount += 1;
|
||||||
|
} else {
|
||||||
|
group.items.push(candidate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...groups.values()];
|
||||||
|
}
|
||||||
|
|||||||
@ -3,6 +3,7 @@ import {
|
|||||||
AttachmentValidationError,
|
AttachmentValidationError,
|
||||||
attachmentsForBot,
|
attachmentsForBot,
|
||||||
blocksToAgentHistoryText,
|
blocksToAgentHistoryText,
|
||||||
|
compactAgentHistoryMessages,
|
||||||
decodeAttachmentBase64,
|
decodeAttachmentBase64,
|
||||||
inferAttachmentMimeType,
|
inferAttachmentMimeType,
|
||||||
messageBlockForArtifact,
|
messageBlockForArtifact,
|
||||||
@ -102,6 +103,46 @@ describe("attachment helpers", () => {
|
|||||||
expect(attachmentsForBot(attachments, "bot-two")).toEqual([attachments[1]]);
|
expect(attachmentsForBot(attachments, "bot-two")).toEqual([attachments[1]]);
|
||||||
expect(attachmentsForBot(attachments, undefined)).toEqual([]);
|
expect(attachmentsForBot(attachments, undefined)).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("compacts older history messages while keeping recent messages and artifact IDs intact", () => {
|
||||||
|
const longText = "A".repeat(400);
|
||||||
|
const history = compactAgentHistoryMessages(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
id: "m-old",
|
||||||
|
role: "bot",
|
||||||
|
blocks: [
|
||||||
|
{ kind: "text", text: longText },
|
||||||
|
{
|
||||||
|
kind: "task",
|
||||||
|
taskId: "t-1",
|
||||||
|
assigneeBotId: "b-2",
|
||||||
|
assigneeBotName: "Researcher",
|
||||||
|
title: "Deep scan",
|
||||||
|
text: "Long task brief that is already finished",
|
||||||
|
status: "done",
|
||||||
|
hop: 1,
|
||||||
|
},
|
||||||
|
{ kind: "image", artifactId: "art-keep", mimeType: "image/png", name: "diagram.png" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "m-recent",
|
||||||
|
role: "user",
|
||||||
|
blocks: [{ kind: "text", text: longText }],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
{ keepRecentFull: 1, maxOlderTextChars: 80 },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(history[0]?.compacted).toBe(true);
|
||||||
|
expect(history[0]?.text).toContain("[…compacted…]");
|
||||||
|
expect(history[0]?.text).toContain("[task for Researcher: Deep scan (done)]");
|
||||||
|
expect(history[0]?.text).not.toContain("Long task brief that is already finished");
|
||||||
|
expect(history[0]?.text).toContain("(artifact art-keep)");
|
||||||
|
expect(history[1]?.compacted).toBe(false);
|
||||||
|
expect(history[1]?.text).toBe(longText);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("peer message history", () => {
|
describe("peer message history", () => {
|
||||||
|
|||||||
@ -262,3 +262,61 @@ export function userTurnBlocksForRun(
|
|||||||
(sourceMessageId ? message.id === sourceMessageId : message.runId === runId),
|
(sourceMessageId ? message.id === sourceMessageId : message.runId === runId),
|
||||||
)?.blocks;
|
)?.blocks;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type AgentHistoryMessageInput = {
|
||||||
|
id: string;
|
||||||
|
role: string;
|
||||||
|
blocks: MessageBlock[];
|
||||||
|
};
|
||||||
|
|
||||||
|
const DEFAULT_KEEP_RECENT_FULL = 6;
|
||||||
|
const DEFAULT_MAX_OLDER_TEXT_CHARS = 280;
|
||||||
|
|
||||||
|
function compactOlderBlock(block: MessageBlock, maxTextChars: number): MessageBlock {
|
||||||
|
if (block.kind === "task" && (block.status === "done" || block.status === "cancelled")) {
|
||||||
|
return { ...block, text: "" };
|
||||||
|
}
|
||||||
|
if (block.kind === "text" && block.text.length > maxTextChars) {
|
||||||
|
return {
|
||||||
|
...block,
|
||||||
|
text: `${block.text.slice(0, maxTextChars).trimEnd()} […compacted…]`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return block;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compacts older conversation turns in long-lived threads while keeping the most recent turns,
|
||||||
|
* artifact references, and bot attributions intact.
|
||||||
|
*/
|
||||||
|
export function compactAgentHistoryMessages(
|
||||||
|
messages: readonly AgentHistoryMessageInput[],
|
||||||
|
options?: { keepRecentFull?: number; maxOlderTextChars?: number },
|
||||||
|
): Array<{ id: string; role: string; text: string; compacted: boolean }> {
|
||||||
|
const keepRecentFull = Math.max(0, options?.keepRecentFull ?? DEFAULT_KEEP_RECENT_FULL);
|
||||||
|
const maxOlderTextChars = Math.max(
|
||||||
|
40,
|
||||||
|
options?.maxOlderTextChars ?? DEFAULT_MAX_OLDER_TEXT_CHARS,
|
||||||
|
);
|
||||||
|
const cutoff = Math.max(0, messages.length - keepRecentFull);
|
||||||
|
|
||||||
|
return messages.map((msg, index) => {
|
||||||
|
if (index >= cutoff) {
|
||||||
|
return {
|
||||||
|
id: msg.id,
|
||||||
|
role: msg.role,
|
||||||
|
text: blocksToAgentHistoryText(msg.blocks),
|
||||||
|
compacted: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const compactedBlocks = msg.blocks.map((block) => compactOlderBlock(block, maxOlderTextChars));
|
||||||
|
const fullText = blocksToAgentHistoryText(msg.blocks);
|
||||||
|
const compactText = blocksToAgentHistoryText(compactedBlocks).trimEnd();
|
||||||
|
return {
|
||||||
|
id: msg.id,
|
||||||
|
role: msg.role,
|
||||||
|
text: compactText,
|
||||||
|
compacted: compactText.length < fullText.length,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|||||||
@ -39,6 +39,40 @@ describe("autopilot", () => {
|
|||||||
).toBe("failing");
|
).toBe("failing");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("treats identical consecutive checkpoint texts as stagnation and pauses on loop", () => {
|
||||||
|
let state = advanceAutopilotGuard(
|
||||||
|
{ idleCount: 0, failCount: 0, lastCheckpointText: "Checked inbox" },
|
||||||
|
{ checkpointed: true, failed: false, checkpointText: "checked inbox" },
|
||||||
|
);
|
||||||
|
expect(state.idleCount).toBe(1);
|
||||||
|
expect(state.pause).toBeNull();
|
||||||
|
state = advanceAutopilotGuard(state, {
|
||||||
|
checkpointed: true,
|
||||||
|
failed: false,
|
||||||
|
checkpointText: " Checked inbox ",
|
||||||
|
});
|
||||||
|
expect(state.idleCount).toBe(2);
|
||||||
|
state = advanceAutopilotGuard(state, {
|
||||||
|
checkpointed: true,
|
||||||
|
failed: false,
|
||||||
|
checkpointText: "Checked inbox",
|
||||||
|
});
|
||||||
|
expect(state.pause).toBe("idle");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("includes a bounded recent checkpoint trail in continuation prompts when provided", () => {
|
||||||
|
const prompt = autopilotContinuationPrompt({
|
||||||
|
goal: "Audit modules",
|
||||||
|
focus: null,
|
||||||
|
until: null,
|
||||||
|
lastCheckpoint: "Module B audited",
|
||||||
|
recentCheckpoints: ["Module A audited", "Module B audited"],
|
||||||
|
nextStep: "Module C",
|
||||||
|
stepCount: 2,
|
||||||
|
});
|
||||||
|
expect(prompt).toContain("Recent checkpoints: 1. Module A audited | 2. Module B audited");
|
||||||
|
});
|
||||||
|
|
||||||
it("tells each continuation the order and where it stands", () => {
|
it("tells each continuation the order and where it stands", () => {
|
||||||
const prompt = autopilotContinuationPrompt({
|
const prompt = autopilotContinuationPrompt({
|
||||||
goal: "Tidy up the tickets",
|
goal: "Tidy up the tickets",
|
||||||
|
|||||||
@ -11,21 +11,35 @@ export interface AutopilotState {
|
|||||||
focus: string | null;
|
focus: string | null;
|
||||||
until: Date | null;
|
until: Date | null;
|
||||||
lastCheckpoint: string | null;
|
lastCheckpoint: string | null;
|
||||||
|
recentCheckpoints?: readonly string[];
|
||||||
nextStep: string | null;
|
nextStep: string | null;
|
||||||
stepCount: number;
|
stepCount: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const MAX_RECENT_CHECKPOINT_TRAIL = 5;
|
||||||
|
|
||||||
|
function normalizeCheckpointText(text: string | null | undefined): string {
|
||||||
|
return (text ?? "").replace(/\s+/g, " ").trim().toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The prompt of each continuation run: the order in the user's words and where the work stands,
|
* The prompt of each continuation run: the order in the user's words and where the work stands,
|
||||||
* so a fresh run picks up without the whole history.
|
* so a fresh run picks up without the whole history.
|
||||||
*/
|
*/
|
||||||
export function autopilotContinuationPrompt(state: AutopilotState, now = new Date()): string {
|
export function autopilotContinuationPrompt(state: AutopilotState, now = new Date()): string {
|
||||||
|
const trail = (state.recentCheckpoints ?? [])
|
||||||
|
.map((item) => item.trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
.slice(-MAX_RECENT_CHECKPOINT_TRAIL);
|
||||||
return [
|
return [
|
||||||
"Autopilot: continue the user's order on your own, one sensible step at a time.",
|
"Autopilot: continue the user's order on your own, one sensible step at a time.",
|
||||||
`Order: ${state.goal}`,
|
`Order: ${state.goal}`,
|
||||||
state.focus ? `Focus: ${state.focus}` : null,
|
state.focus ? `Focus: ${state.focus}` : null,
|
||||||
state.until ? `Until: ${state.until.toISOString()} (now ${now.toISOString()})` : "Until: done",
|
state.until ? `Until: ${state.until.toISOString()} (now ${now.toISOString()})` : "Until: done",
|
||||||
`Steps so far: ${state.stepCount}`,
|
`Steps so far: ${state.stepCount}`,
|
||||||
|
trail.length > 1
|
||||||
|
? `Recent checkpoints: ${trail.map((item, idx) => `${idx + 1}. ${item}`).join(" | ")}`
|
||||||
|
: null,
|
||||||
state.lastCheckpoint ? `Last checkpoint: ${state.lastCheckpoint}` : null,
|
state.lastCheckpoint ? `Last checkpoint: ${state.lastCheckpoint}` : null,
|
||||||
state.nextStep ? `Planned next: ${state.nextStep}` : null,
|
state.nextStep ? `Planned next: ${state.nextStep}` : null,
|
||||||
"Do the next step, then call autopilot_checkpoint with what changed. When the order is done, call autopilot_finish with the result. Actions that need the user's approval are parked as suggestions; continue with other work.",
|
"Do the next step, then call autopilot_checkpoint with what changed. When the order is done, call autopilot_finish with the result. Actions that need the user's approval are parked as suggestions; continue with other work.",
|
||||||
@ -36,10 +50,20 @@ export function autopilotContinuationPrompt(state: AutopilotState, now = new Dat
|
|||||||
|
|
||||||
/** Whether the run that just ended moved the autopilot on, and if it should pause now. */
|
/** Whether the run that just ended moved the autopilot on, and if it should pause now. */
|
||||||
export function advanceAutopilotGuard(
|
export function advanceAutopilotGuard(
|
||||||
counts: { idleCount: number; failCount: number },
|
counts: { idleCount: number; failCount: number; lastCheckpointText?: string | null },
|
||||||
run: { checkpointed: boolean; failed: boolean },
|
run: { checkpointed: boolean; failed: boolean; checkpointText?: string | null },
|
||||||
): { idleCount: number; failCount: number; pause: "idle" | "failing" | null } {
|
): {
|
||||||
const idleCount = run.checkpointed ? 0 : counts.idleCount + 1;
|
idleCount: number;
|
||||||
|
failCount: number;
|
||||||
|
pause: "idle" | "failing" | null;
|
||||||
|
lastCheckpointText?: string | null;
|
||||||
|
} {
|
||||||
|
const prevNorm = normalizeCheckpointText(counts.lastCheckpointText);
|
||||||
|
const nextNorm = normalizeCheckpointText(run.checkpointText);
|
||||||
|
const repeatedSameCheckpoint =
|
||||||
|
run.checkpointed && Boolean(prevNorm && nextNorm && prevNorm === nextNorm);
|
||||||
|
const effectiveProgress = run.checkpointed && !repeatedSameCheckpoint;
|
||||||
|
const idleCount = effectiveProgress ? 0 : counts.idleCount + 1;
|
||||||
const failCount = run.failed ? counts.failCount + 1 : 0;
|
const failCount = run.failed ? counts.failCount + 1 : 0;
|
||||||
const pause =
|
const pause =
|
||||||
failCount >= AUTOPILOT_FAIL_LIMIT
|
failCount >= AUTOPILOT_FAIL_LIMIT
|
||||||
@ -47,7 +71,18 @@ export function advanceAutopilotGuard(
|
|||||||
: idleCount >= AUTOPILOT_IDLE_LIMIT
|
: idleCount >= AUTOPILOT_IDLE_LIMIT
|
||||||
? "idle"
|
? "idle"
|
||||||
: null;
|
: null;
|
||||||
return { idleCount, failCount, pause };
|
const result: {
|
||||||
|
idleCount: number;
|
||||||
|
failCount: number;
|
||||||
|
pause: "idle" | "failing" | null;
|
||||||
|
lastCheckpointText?: string | null;
|
||||||
|
} = { idleCount, failCount, pause };
|
||||||
|
if (run.checkpointText !== undefined || counts.lastCheckpointText !== undefined) {
|
||||||
|
result.lastCheckpointText = run.checkpointed
|
||||||
|
? (run.checkpointText ?? counts.lastCheckpointText ?? null)
|
||||||
|
: (counts.lastCheckpointText ?? null);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The autopilot's time is up. */
|
/** The autopilot's time is up. */
|
||||||
|
|||||||
@ -1,5 +1,9 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { browserUserContextLine, maskBrowserUserDetail } from "./browser-cowork.js";
|
import {
|
||||||
|
browserUserContextLine,
|
||||||
|
coalesceBrowserUserActions,
|
||||||
|
maskBrowserUserDetail,
|
||||||
|
} from "./browser-cowork.js";
|
||||||
|
|
||||||
describe("browser cowork context", () => {
|
describe("browser cowork context", () => {
|
||||||
it("masks password-like labels and fields", () => {
|
it("masks password-like labels and fields", () => {
|
||||||
@ -13,4 +17,24 @@ describe("browser cowork context", () => {
|
|||||||
);
|
);
|
||||||
expect(browserUserContextLine("type", "hello", true)).toBe("User typed in a password field");
|
expect(browserUserContextLine("type", "hello", true)).toBe("User typed in a password field");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("coalesces repeated scrolls, typing bursts, and navigate-to-page_change echoes", () => {
|
||||||
|
const lines = coalesceBrowserUserActions([
|
||||||
|
{ kind: "navigate", detail: "https://example.com/login" },
|
||||||
|
{ kind: "page_change", detail: "https://example.com/login" },
|
||||||
|
{ kind: "scroll", detail: "down" },
|
||||||
|
{ kind: "scroll", detail: "down" },
|
||||||
|
{ kind: "scroll", detail: "down" },
|
||||||
|
{ kind: "type", detail: "al" },
|
||||||
|
{ kind: "type", detail: "ice" },
|
||||||
|
{ kind: "type", detail: "pw1", passwordField: true },
|
||||||
|
{ kind: "type", detail: "pw2", passwordField: true },
|
||||||
|
]);
|
||||||
|
expect(lines).toEqual([
|
||||||
|
"User navigated to https://example.com/login",
|
||||||
|
"User scrolled down (3x)",
|
||||||
|
"User typed alice",
|
||||||
|
"User typed in a password field",
|
||||||
|
]);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@ -33,3 +33,75 @@ export function browserUserContextLine(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type BrowserUserActionEvent = {
|
||||||
|
kind: BrowserUserActionKind;
|
||||||
|
detail: string;
|
||||||
|
passwordField?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Coalesces noisy browser cowork event streams (repeated scrolls, consecutive typing bursts,
|
||||||
|
* or a navigate immediately echoed by a matching page_change) into concise episode lines.
|
||||||
|
*/
|
||||||
|
export function coalesceBrowserUserActions(events: readonly BrowserUserActionEvent[]): string[] {
|
||||||
|
const lines: string[] = [];
|
||||||
|
let index = 0;
|
||||||
|
|
||||||
|
while (index < events.length) {
|
||||||
|
const current = events[index]!;
|
||||||
|
|
||||||
|
if (current.kind === "scroll") {
|
||||||
|
const dir = current.detail.trim() || "down";
|
||||||
|
let count = 1;
|
||||||
|
while (
|
||||||
|
index + count < events.length &&
|
||||||
|
events[index + count]?.kind === "scroll" &&
|
||||||
|
(events[index + count]?.detail.trim() || "down") === dir
|
||||||
|
) {
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
const base = browserUserContextLine("scroll", dir, false);
|
||||||
|
lines.push(count > 1 ? `${base} (${count}x)` : base);
|
||||||
|
index += count;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (current.kind === "type") {
|
||||||
|
const isPassword = Boolean(current.passwordField);
|
||||||
|
const chunks = [current.detail];
|
||||||
|
let count = 1;
|
||||||
|
while (
|
||||||
|
index + count < events.length &&
|
||||||
|
events[index + count]?.kind === "type" &&
|
||||||
|
Boolean(events[index + count]?.passwordField) === isPassword
|
||||||
|
) {
|
||||||
|
chunks.push(events[index + count]!.detail);
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
lines.push(browserUserContextLine("type", chunks.join(""), isPassword));
|
||||||
|
index += count;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
current.kind === "navigate" &&
|
||||||
|
index + 1 < events.length &&
|
||||||
|
events[index + 1]?.kind === "page_change" &&
|
||||||
|
events[index + 1]?.detail.trim() === current.detail.trim()
|
||||||
|
) {
|
||||||
|
lines.push(
|
||||||
|
browserUserContextLine("navigate", current.detail, Boolean(current.passwordField)),
|
||||||
|
);
|
||||||
|
index += 2;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
lines.push(
|
||||||
|
browserUserContextLine(current.kind, current.detail, Boolean(current.passwordField)),
|
||||||
|
);
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return lines;
|
||||||
|
}
|
||||||
|
|||||||
@ -11,7 +11,7 @@ export type GitShellPolicyDecision =
|
|||||||
|
|
||||||
const GIT_PUSH_RE = /\bgit\b[^\n;&|]*\bpush\b/i;
|
const GIT_PUSH_RE = /\bgit\b[^\n;&|]*\bpush\b/i;
|
||||||
const GIT_FORCE_PUSH_RE = /(?:--force-with-lease|--force(?:\s|$)|(?:^|\s)-f(?:\s|$))/i;
|
const GIT_FORCE_PUSH_RE = /(?:--force-with-lease|--force(?:\s|$)|(?:^|\s)-f(?:\s|$))/i;
|
||||||
const GH_PR_RE = /\bgh\b(?:(?:\s+[^;&|]+)*\s+pr\b|\s+pr\b)(?:\s+create\b|\s|$|[;&|])/i;
|
const GH_PR_RE = /\bgh\b[^\n;&|]*\bpr\b/i;
|
||||||
|
|
||||||
export function evaluateGitShellPolicy(input: GitShellPolicyInput): GitShellPolicyDecision {
|
export function evaluateGitShellPolicy(input: GitShellPolicyInput): GitShellPolicyDecision {
|
||||||
const command = input.command.trim();
|
const command = input.command.trim();
|
||||||
|
|||||||
@ -14,14 +14,17 @@ describe("search helpers", () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("matches case-insensitive substrings", () => {
|
it("matches case-insensitive substrings and multi-token queries", () => {
|
||||||
expect(matchesSearchQuery("Venue", "Research venues")).toBe(true);
|
expect(matchesSearchQuery("Venue", "Research venues")).toBe(true);
|
||||||
|
expect(matchesSearchQuery("research berlin", "Research top venues in Berlin")).toBe(true);
|
||||||
|
expect(matchesSearchQuery("research tokyo", "Research top venues in Berlin")).toBe(false);
|
||||||
expect(matchesSearchQuery("missing", "Research venues")).toBe(false);
|
expect(matchesSearchQuery("missing", "Research venues")).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("builds a short snippet around the query", () => {
|
it("builds a short snippet around the query or first matching token", () => {
|
||||||
const text = "alpha beta gamma delta epsilon zeta eta theta";
|
const text = "alpha beta gamma delta epsilon zeta eta theta";
|
||||||
expect(snippetAroundMatch(text, "gamma", 20)).toContain("gamma");
|
expect(snippetAroundMatch(text, "gamma", 20)).toContain("gamma");
|
||||||
|
expect(snippetAroundMatch(text, "zeta alpha", 25)).toContain("zeta");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("uses groupId for group search message jumps", () => {
|
it("uses groupId for group search message jumps", () => {
|
||||||
|
|||||||
@ -5,22 +5,42 @@ export function extractLinksFromText(text: string): string[] {
|
|||||||
return [...new Set(matches.map((url) => url.replace(/[.,;:!?)]+$/, "")))];
|
return [...new Set(matches.map((url) => url.replace(/[.,;:!?)]+$/, "")))];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function tokenizeSearchWords(query: string): string[] {
|
||||||
|
const tokens: string[] = [];
|
||||||
|
for (const match of query.toLowerCase().matchAll(/[\p{L}\p{N}_-]+/gu)) {
|
||||||
|
if (match[0]) tokens.push(match[0]);
|
||||||
|
}
|
||||||
|
return tokens;
|
||||||
|
}
|
||||||
|
|
||||||
export function matchesSearchQuery(
|
export function matchesSearchQuery(
|
||||||
query: string,
|
query: string,
|
||||||
...fields: Array<string | undefined | null>
|
...fields: Array<string | undefined | null>
|
||||||
): boolean {
|
): boolean {
|
||||||
const q = query.trim().toLowerCase();
|
const q = query.trim().toLowerCase();
|
||||||
if (!q) return false;
|
if (!q) return false;
|
||||||
return fields.some((field) => field?.toLowerCase().includes(q));
|
const lowerFields = fields
|
||||||
|
.filter((field): field is string => typeof field === "string" && field.length > 0)
|
||||||
|
.map((field) => field.toLowerCase());
|
||||||
|
if (lowerFields.some((field) => field.includes(q))) return true;
|
||||||
|
const tokens = tokenizeSearchWords(q);
|
||||||
|
if (tokens.length <= 1) return false;
|
||||||
|
const combined = lowerFields.join(" ");
|
||||||
|
return tokens.every((token) => combined.includes(token));
|
||||||
}
|
}
|
||||||
|
|
||||||
export function snippetAroundMatch(text: string, query: string, maxLen = 120): string {
|
export function snippetAroundMatch(text: string, query: string, maxLen = 120): string {
|
||||||
const q = query.trim().toLowerCase();
|
const q = query.trim().toLowerCase();
|
||||||
if (!q) return text.slice(0, maxLen);
|
if (!q) return text.slice(0, maxLen);
|
||||||
const lower = text.toLowerCase();
|
const lower = text.toLowerCase();
|
||||||
const index = lower.indexOf(q);
|
let index = lower.indexOf(q);
|
||||||
|
if (index < 0) {
|
||||||
|
const firstToken = tokenizeSearchWords(q).find((token) => lower.includes(token));
|
||||||
|
if (firstToken) index = lower.indexOf(firstToken);
|
||||||
|
}
|
||||||
if (index < 0) return text.slice(0, maxLen);
|
if (index < 0) return text.slice(0, maxLen);
|
||||||
const start = Math.max(0, index - 40);
|
const leadIn = Math.min(40, Math.floor(maxLen / 3));
|
||||||
|
const start = Math.max(0, index - leadIn);
|
||||||
const end = Math.min(text.length, start + maxLen);
|
const end = Math.min(text.length, start + maxLen);
|
||||||
const prefix = start > 0 ? "…" : "";
|
const prefix = start > 0 ? "…" : "";
|
||||||
const suffix = end < text.length ? "…" : "";
|
const suffix = end < text.length ? "…" : "";
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user