* media: check ffmpeg filter support before subtitle operations * sandbox: handle synchronous spawn errors safely in desktop sandbox * Voice-first streaming, non-blocking follow-ups, quiet routines and chat UX Why: questions blocked the chat, follow-up messages had no acknowledgement, message actions sat beside the bubble, and voice turn-taking added latency. What: waiting runs accept steering messages and the bot marks each claimed follow-up as seen; unclaimed follow-ups show as queued. Reaction, copy and reply controls sit under each message. Streaming voice (VAD endpointing, barge-in that stops the run, clause-based TTS), an optimistic input queue, procedural earcons and delta-based quiet routines were added. Onboarding can be skipped straight to the main conversation. Dual-surface layout and project pickers are included. Status colors use semantic tokens. Tested: unit suites for core, adapters, api and web pass, typecheck and Biome are clean. Browser end-to-end tests were not run locally. * Update translation catalogs Re-extracted catalogs for the new and moved strings; the catalog sync test passes. * Add project plan and test readiness notes Notes from the milestone work: feature inventory, test infrastructure and test readiness. No code changes. * fix(ci): align scale guards, space deletion, computer toggle, and mobile expo deps * fix(ci): restore space deletion guard, browser toggle, and run-steps contracts * fix(ci): restore model resolution errors, group stage, and onboarding flow * fix(ci): restore in-app-browser close button, disable stop button while sending, and sync lingui catalogs * fix(ci): address web e2e failures for bot-crud, choice cards, message quote, and voice onboarding * test(web): wait for avatar and rig visibility in oo-avatar motion spec * test(web): wait for persisted message IDs in message-hover-actions pagination spec * Onboarding: always drop the local voice when the voice step is skipped Skipping waited only for the latest voice load and disconnected only when the client saw the connect succeed. An earlier or client-failed connect could still persist the local credential, so a user who skipped voice was left with one. Wait for every load and always disconnect. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wy1md9QE1ayJbDCwJ4vAQs * Shell: hide the optimistic copy once the transcript shows the send A message sent while the bot works streamed into the transcript before its send request resolved, so the same text showed twice until the optimistic bubble was removed. Drop an optimistic message as soon as a persisted user message with the same text appears. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wy1md9QE1ayJbDCwJ4vAQs --------- Co-authored-by: Claude <noreply@anthropic.com>
290 lines
8.9 KiB
TypeScript
290 lines
8.9 KiB
TypeScript
// @vitest-environment jsdom
|
|
|
|
import { act } from "react";
|
|
import { createRoot } from "react-dom/client";
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
|
|
// @ts-expect-error react act environment
|
|
globalThis.IS_REACT_ACT_ENVIRONMENT = true;
|
|
|
|
const { formatTranslation } = vi.hoisted(() => {
|
|
const formatTranslation = (input: unknown, ...values: unknown[]) => {
|
|
if (Array.isArray(input)) {
|
|
return input.reduce(
|
|
(text, part, index) => text + part + (index < values.length ? values[index] : ""),
|
|
"",
|
|
);
|
|
}
|
|
if (input && typeof input === "object" && "message" in input) {
|
|
return (input as { message: string }).message;
|
|
}
|
|
return String(input ?? "");
|
|
};
|
|
return { formatTranslation };
|
|
});
|
|
|
|
vi.mock("@lingui/react/macro", () => ({
|
|
useLingui: () => ({ t: formatTranslation }),
|
|
Trans: ({ children }: { children: React.ReactNode }) => children,
|
|
}));
|
|
|
|
vi.mock("@lingui/core/macro", () => ({
|
|
t: formatTranslation,
|
|
msg: (parts: TemplateStringsArray, ...values: unknown[]) => ({
|
|
id: parts.join(""),
|
|
message: parts.reduce(
|
|
(text, part, index) => text + part + (index < values.length ? values[index] : ""),
|
|
"",
|
|
),
|
|
}),
|
|
}));
|
|
|
|
import { dictation } from "../lib/dictation.js";
|
|
import { rpc } from "../lib/rpc.js";
|
|
import { Speaker, speaker, UtteranceStreamer } from "../lib/tts.js";
|
|
import { CallView } from "../pages/CallView.js";
|
|
|
|
describe("Adversarial Stress Test: Full-Duplex Barge-In & Mute Timing", () => {
|
|
let container: HTMLDivElement | null = null;
|
|
let root: ReturnType<typeof createRoot> | null = null;
|
|
|
|
beforeEach(() => {
|
|
window.matchMedia = vi.fn().mockImplementation((query: string) => ({
|
|
matches: false,
|
|
media: query,
|
|
onchange: null,
|
|
addListener: vi.fn(),
|
|
removeListener: vi.fn(),
|
|
addEventListener: vi.fn(),
|
|
removeEventListener: vi.fn(),
|
|
dispatchEvent: vi.fn(),
|
|
}));
|
|
container = document.createElement("div");
|
|
document.body.appendChild(container);
|
|
root = createRoot(container);
|
|
});
|
|
|
|
afterEach(() => {
|
|
if (root) {
|
|
act(() => {
|
|
root?.unmount();
|
|
});
|
|
}
|
|
container?.remove();
|
|
vi.restoreAllMocks();
|
|
vi.unstubAllGlobals();
|
|
});
|
|
|
|
describe("Microsecond timing assertions (<15ms) on playback mute", () => {
|
|
it("Timing 1: speaker.stop() synchronous execution finishes well below 15ms threshold", () => {
|
|
const spk = new Speaker();
|
|
const latencies: number[] = [];
|
|
|
|
for (let i = 0; i < 50; i++) {
|
|
// Set up active simulated state
|
|
(spk as unknown as { snapshot: { status: string } }).snapshot = { status: "speaking" };
|
|
|
|
const t0 = performance.now();
|
|
spk.stop();
|
|
const t1 = performance.now();
|
|
|
|
const latencyMs = t1 - t0;
|
|
latencies.push(latencyMs);
|
|
expect(spk.state.status).toBe("idle");
|
|
expect(latencyMs).toBeLessThan(15); // Strictest requirement: < 15ms
|
|
}
|
|
|
|
const avgLatency = latencies.reduce((a, b) => a + b, 0) / latencies.length;
|
|
expect(avgLatency).toBeLessThan(5); // Typically < 0.5ms
|
|
});
|
|
|
|
it("Timing 2: Concurrent speech detection and playback chunk arrival mutes in <15ms", async () => {
|
|
const spk = new Speaker();
|
|
vi.spyOn(spk, "render").mockImplementation(
|
|
() => new Promise((resolve) => setTimeout(() => resolve(new Blob(["audio"])), 5)),
|
|
);
|
|
vi.spyOn(spk, "play").mockImplementation(
|
|
() => new Promise((resolve) => setTimeout(() => resolve(true), 20)),
|
|
);
|
|
|
|
// Start simulated speech
|
|
const speakPromise = spk.speak("First clause of response.");
|
|
|
|
// Concurrently simulate speech detection (barge-in event)
|
|
const t0 = performance.now();
|
|
spk.stop();
|
|
const muteElapsed = performance.now() - t0;
|
|
|
|
expect(muteElapsed).toBeLessThan(15);
|
|
expect(spk.state.status).toBe("idle");
|
|
|
|
await speakPromise;
|
|
expect(spk.state.status).toBe("idle");
|
|
});
|
|
});
|
|
|
|
describe("Concurrent speech detection vs UtteranceStreamer", () => {
|
|
it("Streamer Tracking: speaker.stop() synchronously aborts active UtteranceStreamer instances without audio leakage", async () => {
|
|
const spk = new Speaker();
|
|
vi.spyOn(spk, "render").mockResolvedValue(new Blob(["audio"]));
|
|
let playCount = 0;
|
|
vi.spyOn(spk, "play").mockImplementation(async () => {
|
|
playCount++;
|
|
return true;
|
|
});
|
|
|
|
const streamer = spk.createUtteranceStreamer();
|
|
streamer.push("First clause stream. Second clause stream. Third clause stream. ");
|
|
|
|
// Barge-in mute via speaker.stop() (as invoked in CallView)
|
|
spk.stop();
|
|
|
|
// Flush remaining streamer clauses
|
|
await streamer.flush();
|
|
|
|
// Verified: UtteranceStreamer is stopped immediately by speaker.stop()
|
|
expect(playCount).toBe(0);
|
|
|
|
// Explicit streamer.stop() is required to stop it
|
|
streamer.stop();
|
|
});
|
|
});
|
|
|
|
describe("CallView Barge-In Abort Signal Propagation", () => {
|
|
it("CallView Signal Propagation: voice barge-in mutes speaker AND signals backend cancellation", async () => {
|
|
const onStopMock = vi.fn();
|
|
const stopRpc = vi.fn().mockResolvedValue({} as never);
|
|
(rpc.threads as { stop?: unknown }).stop = stopRpc;
|
|
|
|
// Mount CallView
|
|
await act(async () => {
|
|
root!.render(
|
|
<CallView
|
|
botId="bot-1"
|
|
botName="Assistant"
|
|
botTitle="Helper"
|
|
botDescription="Test bot"
|
|
botColor="blue"
|
|
transcribe={true}
|
|
snapshot={null}
|
|
onSend={async () => {}}
|
|
onFollowUp={async () => {}}
|
|
onAnswer={async () => {}}
|
|
onStop={onStopMock}
|
|
onClose={() => {}}
|
|
/>,
|
|
);
|
|
});
|
|
|
|
// Put speaker into speaking phase
|
|
act(() => {
|
|
(speaker as unknown as { set: (s: unknown) => void }).set({
|
|
status: "speaking",
|
|
botId: "bot-1",
|
|
caption: "I am actively explaining the answer...",
|
|
});
|
|
});
|
|
|
|
// Verify speaker is speaking
|
|
expect(speaker.state.status).toBe("speaking");
|
|
|
|
// Now simulate user voice barge-in: mic energy spikes > 0.18 with user speech
|
|
act(() => {
|
|
(dictation as unknown as { set: (s: unknown) => void }).set({
|
|
status: "listening",
|
|
energy: 0.42,
|
|
transcript: "Wait, cancel that!",
|
|
});
|
|
});
|
|
|
|
// Speaker must be stopped immediately (<15ms barge-in mute)
|
|
expect(speaker.state.status).toBe("idle");
|
|
|
|
const abortSignalled = onStopMock.mock.calls.length > 0 || stopRpc.mock.calls.length > 0;
|
|
expect(abortSignalled).toBe(true);
|
|
});
|
|
|
|
it("Contrast: Keyboard spacebar interruption properly triggers onStop", async () => {
|
|
const onStopMock = vi.fn();
|
|
const stopRpc = vi.fn().mockResolvedValue({} as never);
|
|
(rpc.threads as { stop?: unknown }).stop = stopRpc;
|
|
|
|
await act(async () => {
|
|
root!.render(
|
|
<CallView
|
|
botId="bot-1"
|
|
botName="Assistant"
|
|
botTitle="Helper"
|
|
botDescription="Test bot"
|
|
botColor="blue"
|
|
transcribe={true}
|
|
snapshot={null}
|
|
onSend={async () => {}}
|
|
onFollowUp={async () => {}}
|
|
onAnswer={async () => {}}
|
|
onStop={onStopMock}
|
|
onClose={() => {}}
|
|
/>,
|
|
);
|
|
});
|
|
|
|
// Put into speaking phase
|
|
act(() => {
|
|
(speaker as unknown as { set: (s: unknown) => void }).set({
|
|
status: "speaking",
|
|
botId: "bot-1",
|
|
caption: "I am talking...",
|
|
});
|
|
});
|
|
|
|
// Press Space key to interrupt
|
|
act(() => {
|
|
window.dispatchEvent(new KeyboardEvent("keydown", { key: " " }));
|
|
});
|
|
|
|
// Space key calls interrupt() which triggers onStopMock
|
|
expect(onStopMock).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("Stress: Rapid consecutive voice interruptions do not freeze or leak state", async () => {
|
|
await act(async () => {
|
|
root!.render(
|
|
<CallView
|
|
botId="bot-1"
|
|
botName="Assistant"
|
|
botTitle="Helper"
|
|
botDescription="Test bot"
|
|
botColor="blue"
|
|
transcribe={true}
|
|
snapshot={null}
|
|
onSend={async () => {}}
|
|
onFollowUp={async () => {}}
|
|
onAnswer={async () => {}}
|
|
onClose={() => {}}
|
|
/>,
|
|
);
|
|
});
|
|
|
|
for (let i = 0; i < 30; i++) {
|
|
act(() => {
|
|
(speaker as unknown as { set: (s: unknown) => void }).set({
|
|
status: "speaking",
|
|
botId: "bot-1",
|
|
caption: `Burst utterance ${i}`,
|
|
});
|
|
});
|
|
expect(speaker.state.status).toBe("speaking");
|
|
|
|
act(() => {
|
|
(dictation as unknown as { set: (s: unknown) => void }).set({
|
|
status: "listening",
|
|
energy: 0.35,
|
|
transcript: `Interrupt ${i}`,
|
|
});
|
|
});
|
|
expect(speaker.state.status).toBe("idle");
|
|
}
|
|
});
|
|
});
|
|
});
|