* media: check ffmpeg filter support before subtitle operations * sandbox: handle synchronous spawn errors safely in desktop sandbox * Voice-first streaming, non-blocking follow-ups, quiet routines and chat UX Why: questions blocked the chat, follow-up messages had no acknowledgement, message actions sat beside the bubble, and voice turn-taking added latency. What: waiting runs accept steering messages and the bot marks each claimed follow-up as seen; unclaimed follow-ups show as queued. Reaction, copy and reply controls sit under each message. Streaming voice (VAD endpointing, barge-in that stops the run, clause-based TTS), an optimistic input queue, procedural earcons and delta-based quiet routines were added. Onboarding can be skipped straight to the main conversation. Dual-surface layout and project pickers are included. Status colors use semantic tokens. Tested: unit suites for core, adapters, api and web pass, typecheck and Biome are clean. Browser end-to-end tests were not run locally. * Update translation catalogs Re-extracted catalogs for the new and moved strings; the catalog sync test passes. * Add project plan and test readiness notes Notes from the milestone work: feature inventory, test infrastructure and test readiness. No code changes. * fix(ci): align scale guards, space deletion, computer toggle, and mobile expo deps * fix(ci): restore space deletion guard, browser toggle, and run-steps contracts * fix(ci): restore model resolution errors, group stage, and onboarding flow * fix(ci): restore in-app-browser close button, disable stop button while sending, and sync lingui catalogs * fix(ci): address web e2e failures for bot-crud, choice cards, message quote, and voice onboarding * test(web): wait for avatar and rig visibility in oo-avatar motion spec * test(web): wait for persisted message IDs in message-hover-actions pagination spec * Onboarding: always drop the local voice when the voice step is skipped Skipping waited only for the latest voice load and disconnected only when the client saw the connect succeed. An earlier or client-failed connect could still persist the local credential, so a user who skipped voice was left with one. Wait for every load and always disconnect. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wy1md9QE1ayJbDCwJ4vAQs * Shell: hide the optimistic copy once the transcript shows the send A message sent while the bot works streamed into the transcript before its send request resolved, so the same text showed twice until the optimistic bubble was removed. Drop an optimistic message as soon as a persisted user message with the same text appears. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wy1md9QE1ayJbDCwJ4vAQs --------- Co-authored-by: Claude <noreply@anthropic.com>
403 lines
14 KiB
TypeScript
403 lines
14 KiB
TypeScript
// @vitest-environment jsdom
|
|
|
|
import type { StreamingVoiceFrame } from "@rakazo/core";
|
|
import { describe, expect, it } from "vitest";
|
|
import {
|
|
DynamicActionQueueImpl,
|
|
OptimisticInputQueue,
|
|
RoutinePolicyEvaluator,
|
|
} from "./fixtures/agency-contracts";
|
|
import { detectUserEnvironment, STARTER_SPARKS } from "./fixtures/ux-contracts";
|
|
|
|
class StreamingVoiceClientImpl {
|
|
public isStreaming = false;
|
|
public transcriptListeners: ((delta: string, isFinal: boolean) => void)[] = [];
|
|
public playbackListeners: ((chunk: Uint8Array) => void)[] = [];
|
|
public framesSent: StreamingVoiceFrame[] = [];
|
|
public interrupted = false;
|
|
public isMuted = false;
|
|
|
|
async startStream(): Promise<void> {
|
|
this.isStreaming = true;
|
|
this.interrupted = false;
|
|
}
|
|
|
|
sendFrame(frame: StreamingVoiceFrame): void {
|
|
if (!this.isStreaming) {
|
|
throw new Error("Stream not started");
|
|
}
|
|
this.framesSent.push(frame);
|
|
}
|
|
|
|
onTranscriptDelta(cb: (delta: string, isFinal: boolean) => void): void {
|
|
this.transcriptListeners.push(cb);
|
|
}
|
|
|
|
onPlaybackChunk(cb: (chunk: Uint8Array) => void): void {
|
|
this.playbackListeners.push(cb);
|
|
}
|
|
|
|
interrupt(): void {
|
|
this.interrupted = true;
|
|
this.isMuted = true;
|
|
}
|
|
|
|
emitTranscript(delta: string, isFinal: boolean): void {
|
|
for (const listener of this.transcriptListeners) {
|
|
listener(delta, isFinal);
|
|
}
|
|
}
|
|
|
|
emitPlayback(chunk: Uint8Array): void {
|
|
if (this.isMuted) return;
|
|
for (const listener of this.playbackListeners) {
|
|
listener(chunk);
|
|
}
|
|
}
|
|
}
|
|
|
|
describe("Cross-Feature Interactions & End-to-End Workflows (Tiers 3 & 4)", () => {
|
|
describe("Tier 3: Pairwise Cross-Feature Interactions", () => {
|
|
it("Pair 1: Voice barge-in interrupt while action queue is actively running step", () => {
|
|
const voiceClient = new StreamingVoiceClientImpl();
|
|
voiceClient.startStream();
|
|
|
|
const actionQueue = new DynamicActionQueueImpl();
|
|
actionQueue.synthesizeInput("Run code analysis; Deploy to production");
|
|
actionQueue.advance(); // step-1 running
|
|
|
|
expect(actionQueue.steps[0]!.status).toBe("running");
|
|
|
|
// User speaks interrupt: "Wait, stop deployment"
|
|
voiceClient.interrupt();
|
|
expect(voiceClient.interrupted).toBe(true);
|
|
|
|
// Barge-in cancels active action queue step
|
|
actionQueue.cancelActive("voice_barge_in");
|
|
expect(actionQueue.steps[0]!.status).toBe("cancelled");
|
|
expect(actionQueue.activeStepId).toBeNull();
|
|
});
|
|
|
|
it("Pair 2: Optimistic queue burst while streaming audio TTS is playing", () => {
|
|
const voiceClient = new StreamingVoiceClientImpl();
|
|
voiceClient.startStream();
|
|
const ttsPlaying = true;
|
|
|
|
const inputQueue = new OptimisticInputQueue();
|
|
// User types rapid follow-up while listening to TTS
|
|
const m1 = inputQueue.enqueue("Also check integration tests");
|
|
const m2 = inputQueue.enqueue("And format with Biome");
|
|
|
|
expect(inputQueue.queue.length).toBe(2);
|
|
expect(ttsPlaying).toBe(true); // TTS continues uninterrupted by typing
|
|
expect(m1.status).toBe("queued");
|
|
expect(m2.status).toBe("queued");
|
|
});
|
|
|
|
it("Pair 3: Mid-step tool cancellation triggered via voice interruption event", () => {
|
|
const toolAbortController = new AbortController();
|
|
let toolExecutionRunning = true;
|
|
|
|
toolAbortController.signal.addEventListener("abort", () => {
|
|
toolExecutionRunning = false;
|
|
});
|
|
|
|
// User speech energy spike triggers voice interrupt
|
|
const micEnergy = 0.52;
|
|
if (micEnergy > 0.035) {
|
|
toolAbortController.abort("voice_barge_in");
|
|
}
|
|
|
|
expect(toolAbortController.signal.aborted).toBe(true);
|
|
expect(toolExecutionRunning).toBe(false);
|
|
});
|
|
|
|
it("Pair 4: Quiet background routine fires during active voice session", () => {
|
|
const voiceClient = new StreamingVoiceClientImpl();
|
|
voiceClient.startStream();
|
|
|
|
const routinePolicy = new RoutinePolicyEvaluator("on_change");
|
|
// Output unchanged
|
|
const shouldNotify = routinePolicy.evaluateDelta("hash_state_1", "hash_state_1");
|
|
|
|
expect(shouldNotify).toBe(false);
|
|
// Voice lane remains undisturbed, zero audio chime spam
|
|
expect(voiceClient.isMuted).toBe(false);
|
|
});
|
|
|
|
it("Pair 5: 30-second rapid onboarding directly initiates full-duplex voice stage", async () => {
|
|
const env = detectUserEnvironment();
|
|
expect(env.timezone).toBeDefined();
|
|
|
|
const voiceClient = new StreamingVoiceClientImpl();
|
|
await voiceClient.startStream();
|
|
|
|
expect(voiceClient.isStreaming).toBe(true);
|
|
expect(voiceClient.interrupted).toBe(false);
|
|
});
|
|
|
|
it("Pair 6: Action queue synthesis dynamically surfaces Living Canvas for artifact inspection", () => {
|
|
const actionQueue = new DynamicActionQueueImpl();
|
|
actionQueue.synthesizeInput("Generate API schema; Save to swagger.json");
|
|
actionQueue.advance();
|
|
|
|
let livingCanvasVisible = false;
|
|
const onArtifactGenerated = () => {
|
|
livingCanvasVisible = true;
|
|
};
|
|
|
|
onArtifactGenerated();
|
|
expect(livingCanvasVisible).toBe(true);
|
|
});
|
|
|
|
it("Pair 7: Tactile folding step reflects mid-step tool cancellation without UI glitch", () => {
|
|
const actionQueue = new DynamicActionQueueImpl();
|
|
actionQueue.synthesizeInput("Run slow docker build");
|
|
actionQueue.advance();
|
|
|
|
actionQueue.cancelActive("user_redirect");
|
|
const step = actionQueue.steps[0]!;
|
|
|
|
expect(step.status).toBe("cancelled");
|
|
// Clean folded representation
|
|
const foldLabel = `${actionQueue.steps.length} step (${step.status})`;
|
|
expect(foldLabel).toBe("1 step (cancelled)");
|
|
});
|
|
|
|
it("Pair 8: Voice streaming client reconnects after temporary network drop while action queue advances", async () => {
|
|
const voiceClient = new StreamingVoiceClientImpl();
|
|
await voiceClient.startStream();
|
|
|
|
const actionQueue = new DynamicActionQueueImpl();
|
|
actionQueue.synthesizeInput("Compile contracts; Run evals");
|
|
actionQueue.advance(); // step 1 running
|
|
|
|
// Network drops
|
|
voiceClient.isStreaming = false;
|
|
|
|
// Action queue advances internally on server
|
|
actionQueue.advance(); // completes step 1, runs step 2
|
|
expect(actionQueue.steps[0]!.status).toBe("completed");
|
|
expect(actionQueue.steps[1]!.status).toBe("running");
|
|
|
|
// Reconnect voice client
|
|
await voiceClient.startStream();
|
|
expect(voiceClient.isStreaming).toBe(true);
|
|
});
|
|
|
|
it("Pair 9: Audio earcon triggers on optimistic queue drain and background delta detection", () => {
|
|
const earconLog: string[] = [];
|
|
const triggerEarcon = (name: string) => earconLog.push(name);
|
|
|
|
const inputQueue = new OptimisticInputQueue();
|
|
inputQueue.enqueue("Step A");
|
|
triggerEarcon("interact");
|
|
|
|
const routinePolicy = new RoutinePolicyEvaluator("on_change");
|
|
const hasDelta = routinePolicy.evaluateDelta("hash_old", "hash_new");
|
|
if (hasDelta) {
|
|
triggerEarcon("attention");
|
|
}
|
|
|
|
expect(earconLog).toEqual(["interact", "attention"]);
|
|
});
|
|
|
|
it("Pair 10: Living Canvas orientation swap while action queue runs background steps", () => {
|
|
let orientation: "stream-first" | "canvas-first" = "stream-first";
|
|
const actionQueue = new DynamicActionQueueImpl();
|
|
actionQueue.synthesizeInput("Background sync 1; Background sync 2");
|
|
actionQueue.advance();
|
|
|
|
// User swaps layout orientation
|
|
orientation = orientation === "stream-first" ? "canvas-first" : "stream-first";
|
|
|
|
expect(orientation).toBe("canvas-first");
|
|
expect(actionQueue.steps[0]!.status).toBe("running");
|
|
actionQueue.advance();
|
|
expect(actionQueue.steps[0]!.status).toBe("completed");
|
|
});
|
|
|
|
it("Pair 11: Voice interrupt resets incremental utterance clause pipelining", () => {
|
|
let pendingClauses = ["Here is the summary of", "your project status report."];
|
|
const onInterrupt = () => {
|
|
pendingClauses = [];
|
|
};
|
|
|
|
onInterrupt();
|
|
expect(pendingClauses.length).toBe(0);
|
|
});
|
|
|
|
it("Pair 12: Delta routine evaluation suppresses notification while user is typing in optimistic composer", () => {
|
|
const composerActive = true;
|
|
const routinePolicy = new RoutinePolicyEvaluator("on_change");
|
|
const shouldNotify = routinePolicy.evaluateDelta("hash_a", "hash_a");
|
|
|
|
expect(composerActive).toBe(true);
|
|
expect(shouldNotify).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("Tier 4: Real-World Application Scenarios", () => {
|
|
it("Scenario 1: Ambient Hands-Free Meeting & Live Action Queue", async () => {
|
|
// 1. Hands-free user initiates session
|
|
const voiceClient = new StreamingVoiceClientImpl();
|
|
await voiceClient.startStream();
|
|
|
|
// 2. User dictates three action items
|
|
const spokenTranscript =
|
|
"First review the pull request, then run security scan, and build docker image";
|
|
const actionQueue = new DynamicActionQueueImpl();
|
|
actionQueue.synthesizeInput(spokenTranscript);
|
|
|
|
expect(actionQueue.steps.length).toBe(3);
|
|
expect(actionQueue.steps[0]!.intent).toContain("First review the pull request");
|
|
expect(actionQueue.steps[1]!.intent).toContain("run security scan");
|
|
expect(actionQueue.steps[2]!.intent).toContain("build docker image");
|
|
|
|
// 3. Execution advances step by step
|
|
actionQueue.advance();
|
|
expect(actionQueue.steps[0]!.status).toBe("running");
|
|
|
|
actionQueue.advance();
|
|
expect(actionQueue.steps[0]!.status).toBe("completed");
|
|
expect(actionQueue.steps[1]!.status).toBe("running");
|
|
|
|
actionQueue.advance();
|
|
expect(actionQueue.steps[1]!.status).toBe("completed");
|
|
expect(actionQueue.steps[2]!.status).toBe("running");
|
|
|
|
actionQueue.advance();
|
|
expect(actionQueue.steps[2]!.status).toBe("completed");
|
|
expect(actionQueue.activeStepId).toBeNull();
|
|
});
|
|
|
|
it("Scenario 2: Conversational Barge-In & Mid-Step Tool Cancellation", async () => {
|
|
// 1. Agent executing long-running tool
|
|
const toolController = new AbortController();
|
|
const toolCompleted = false;
|
|
|
|
const voiceClient = new StreamingVoiceClientImpl();
|
|
await voiceClient.startStream();
|
|
|
|
// 2. User detects wrong direction and barges in with speech
|
|
const userSpokeBargeIn = true;
|
|
if (userSpokeBargeIn) {
|
|
voiceClient.interrupt();
|
|
toolController.abort("user_redirect");
|
|
}
|
|
|
|
expect(voiceClient.interrupted).toBe(true);
|
|
expect(toolController.signal.aborted).toBe(true);
|
|
expect(toolCompleted).toBe(false);
|
|
});
|
|
|
|
it("Scenario 3: High-Frequency Multi-Prompt Burst & Optimistic Ordering", () => {
|
|
const queue = new OptimisticInputQueue();
|
|
|
|
// User sends 3 rapid follow-ups in <200ms
|
|
const m1 = queue.enqueue("Check linting");
|
|
const m2 = queue.enqueue("Fix formatting issues");
|
|
const m3 = queue.enqueue("Push commit");
|
|
|
|
expect(queue.queue.length).toBe(3);
|
|
expect(m1.id).not.toBe(m2.id);
|
|
expect(m2.id).not.toBe(m3.id);
|
|
|
|
// UI processes them in exact FIFO order
|
|
const processed = queue.drain();
|
|
expect(processed[0]?.text).toBe("Check linting");
|
|
expect(processed[1]?.text).toBe("Fix formatting issues");
|
|
expect(processed[2]?.text).toBe("Push commit");
|
|
});
|
|
|
|
it("Scenario 4: Quiet Background Routine with Delta Detection & No Nagging", () => {
|
|
const policy = new RoutinePolicyEvaluator("on_change");
|
|
let notificationsTriggered = 0;
|
|
|
|
// Routine executes periodically (10 cycles)
|
|
let currentHash = "initial_db_state";
|
|
for (let cycle = 0; cycle < 9; cycle++) {
|
|
const nextHash = "initial_db_state"; // Unchanged
|
|
if (policy.evaluateDelta(currentHash, nextHash)) {
|
|
notificationsTriggered++;
|
|
}
|
|
currentHash = nextHash;
|
|
}
|
|
|
|
// Cycle 10: Critical change occurs
|
|
const criticalHash = "new_db_anomaly_detected";
|
|
if (policy.evaluateDelta(currentHash, criticalHash)) {
|
|
notificationsTriggered++;
|
|
}
|
|
|
|
// Exactly 1 notification was sent across 10 runs! Zero nagging.
|
|
expect(notificationsTriggered).toBe(1);
|
|
});
|
|
|
|
it("Scenario 5: Rapid 30-Second Zero-Questionnaire Onboarding to Live Execution", () => {
|
|
// 1. User arrives at fresh app
|
|
const env = detectUserEnvironment();
|
|
expect(env.locale).toBeDefined();
|
|
|
|
// 2. System presents 3 immediate starter sparks without questionnaires
|
|
expect(STARTER_SPARKS.length).toBe(3);
|
|
const chosenSpark = STARTER_SPARKS[0]!;
|
|
|
|
// 3. 1-click bootstrap initiates immediate session
|
|
const actionQueue = new DynamicActionQueueImpl();
|
|
actionQueue.synthesizeInput(chosenSpark);
|
|
actionQueue.advance();
|
|
|
|
expect(actionQueue.steps[0]?.status).toBe("running");
|
|
expect(actionQueue.steps[0]?.intent).toBe(chosenSpark);
|
|
});
|
|
|
|
it("Scenario 6: Tactile Dual-Surface Progressive Disclosure & Step Folding Workflow", () => {
|
|
let canvasVisible = false;
|
|
const steps = [
|
|
{ id: "1", toolName: "git clone", status: "completed" },
|
|
{ id: "2", toolName: "npm install", status: "completed" },
|
|
{ id: "3", toolName: "generate report.md", status: "completed" },
|
|
];
|
|
|
|
// Step folding hides chrome bloat
|
|
const stepSummary = `${steps.length} steps`;
|
|
expect(stepSummary).toBe("3 steps");
|
|
|
|
// File artifact generated -> Living Canvas surfaces
|
|
const onArtifactProduced = () => {
|
|
canvasVisible = true;
|
|
};
|
|
|
|
onArtifactProduced();
|
|
expect(canvasVisible).toBe(true);
|
|
});
|
|
|
|
it("Scenario 7: Network Loss & Reconnection Resilience during Full-Duplex Voice Session", async () => {
|
|
const voiceClient = new StreamingVoiceClientImpl();
|
|
await voiceClient.startStream();
|
|
|
|
// Send initial frame
|
|
voiceClient.sendFrame({ pcm16: new Int16Array(1280), sampleRate: 16000, timestamp: 1000 });
|
|
expect(voiceClient.framesSent.length).toBe(1);
|
|
|
|
// Network disconnects
|
|
voiceClient.isStreaming = false;
|
|
const bufferedFrames: any[] = [];
|
|
|
|
// Outgoing frame buffered while offline
|
|
bufferedFrames.push({ pcm16: new Int16Array(1280), sampleRate: 16000, timestamp: 1080 });
|
|
|
|
// Network recovers
|
|
await voiceClient.startStream();
|
|
for (const frame of bufferedFrames) {
|
|
voiceClient.sendFrame(frame);
|
|
}
|
|
|
|
expect(voiceClient.isStreaming).toBe(true);
|
|
expect(voiceClient.framesSent.length).toBe(2);
|
|
expect(voiceClient.framesSent[1]?.timestamp).toBe(1080);
|
|
});
|
|
});
|
|
});
|