* media: check ffmpeg filter support before subtitle operations * sandbox: handle synchronous spawn errors safely in desktop sandbox * Voice-first streaming, non-blocking follow-ups, quiet routines and chat UX Why: questions blocked the chat, follow-up messages had no acknowledgement, message actions sat beside the bubble, and voice turn-taking added latency. What: waiting runs accept steering messages and the bot marks each claimed follow-up as seen; unclaimed follow-ups show as queued. Reaction, copy and reply controls sit under each message. Streaming voice (VAD endpointing, barge-in that stops the run, clause-based TTS), an optimistic input queue, procedural earcons and delta-based quiet routines were added. Onboarding can be skipped straight to the main conversation. Dual-surface layout and project pickers are included. Status colors use semantic tokens. Tested: unit suites for core, adapters, api and web pass, typecheck and Biome are clean. Browser end-to-end tests were not run locally. * Update translation catalogs Re-extracted catalogs for the new and moved strings; the catalog sync test passes. * Add project plan and test readiness notes Notes from the milestone work: feature inventory, test infrastructure and test readiness. No code changes. * fix(ci): align scale guards, space deletion, computer toggle, and mobile expo deps * fix(ci): restore space deletion guard, browser toggle, and run-steps contracts * fix(ci): restore model resolution errors, group stage, and onboarding flow * fix(ci): restore in-app-browser close button, disable stop button while sending, and sync lingui catalogs * fix(ci): address web e2e failures for bot-crud, choice cards, message quote, and voice onboarding * test(web): wait for avatar and rig visibility in oo-avatar motion spec * test(web): wait for persisted message IDs in message-hover-actions pagination spec * Onboarding: always drop the local voice when the voice step is skipped Skipping waited only for the latest voice load and disconnected only when the client saw the connect succeed. An earlier or client-failed connect could still persist the local credential, so a user who skipped voice was left with one. Wait for every load and always disconnect. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wy1md9QE1ayJbDCwJ4vAQs * Shell: hide the optimistic copy once the transcript shows the send A message sent while the bot works streamed into the transcript before its send request resolved, so the same text showed twice until the optimistic bubble was removed. Drop an optimistic message as soon as a persisted user message with the same text appears. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wy1md9QE1ayJbDCwJ4vAQs --------- Co-authored-by: Claude <noreply@anthropic.com>
315 lines
8.6 KiB
TypeScript
315 lines
8.6 KiB
TypeScript
/**
|
|
* Outcome-Driven Action Queue State Machine for autonomous agent execution.
|
|
* Manages compound instruction synthesis, state transitions, outcome verification,
|
|
* and dynamic execution steering.
|
|
*/
|
|
|
|
export type ActionStepStatus = "pending" | "running" | "completed" | "cancelled" | "failed";
|
|
|
|
export interface ActionQueueStep {
|
|
id: string;
|
|
intent: string;
|
|
outcomeCriteria: string;
|
|
status: ActionStepStatus;
|
|
startedAt?: number;
|
|
completedAt?: number;
|
|
}
|
|
|
|
export interface DynamicActionQueue {
|
|
steps: ActionQueueStep[];
|
|
activeStepId: string | null;
|
|
synthesizeInput(
|
|
rawText: string,
|
|
options?: { criteriaTemplate?: (intent: string) => string },
|
|
): ActionQueueStep[];
|
|
cancelActive(reason?: string): ActionQueueStep | null;
|
|
advance(): ActionQueueStep | null;
|
|
}
|
|
|
|
export interface ActionQueueSnapshot {
|
|
steps: ActionQueueStep[];
|
|
activeStepId: string | null;
|
|
}
|
|
|
|
export type StepVerificationGate = (
|
|
step: ActionQueueStep,
|
|
output?: unknown,
|
|
) => boolean | Promise<boolean>;
|
|
|
|
export const ALLOWED_STEP_TRANSITIONS: Record<ActionStepStatus, readonly ActionStepStatus[]> = {
|
|
pending: ["running", "cancelled"],
|
|
running: ["completed", "failed", "cancelled"],
|
|
completed: [],
|
|
failed: ["running", "cancelled"],
|
|
cancelled: [],
|
|
};
|
|
|
|
export function canTransitionStep(from: ActionStepStatus, to: ActionStepStatus): boolean {
|
|
return (ALLOWED_STEP_TRANSITIONS[from] as readonly ActionStepStatus[])?.includes(to) ?? false;
|
|
}
|
|
|
|
export function assertStepTransition(from: ActionStepStatus, to: ActionStepStatus): void {
|
|
if (!canTransitionStep(from, to)) {
|
|
throw new Error(`Illegal step transition from ${from} to ${to}`);
|
|
}
|
|
}
|
|
|
|
export function isTerminalStepStatus(status: ActionStepStatus): boolean {
|
|
return status === "completed" || status === "cancelled";
|
|
}
|
|
|
|
export function isActiveStepStatus(status: ActionStepStatus): boolean {
|
|
return status === "running";
|
|
}
|
|
|
|
const STEP_SPLIT_REGEX = /(?:;|\.|\n+|\r+|\bthen\b|\band then\b|\bnext\b|,\s*and\b|,)/i;
|
|
|
|
export class DynamicActionQueueImpl implements DynamicActionQueue {
|
|
public steps: ActionQueueStep[] = [];
|
|
public activeStepId: string | null = null;
|
|
private verificationGate?: StepVerificationGate;
|
|
|
|
constructor(initialSteps: ActionQueueStep[] = [], activeStepId: string | null = null) {
|
|
this.steps = initialSteps.map((s) => ({ ...s }));
|
|
this.activeStepId = activeStepId;
|
|
}
|
|
|
|
setVerificationGate(gate: StepVerificationGate | undefined): void {
|
|
this.verificationGate = gate;
|
|
}
|
|
|
|
getVerificationGate(): StepVerificationGate | undefined {
|
|
return this.verificationGate;
|
|
}
|
|
|
|
async verifyStep(stepId: string, output?: unknown): Promise<boolean> {
|
|
const step = this.steps.find((s) => s.id === stepId);
|
|
if (!step) return false;
|
|
if (!this.verificationGate) return true;
|
|
return Boolean(await this.verificationGate(step, output));
|
|
}
|
|
|
|
synthesizeInput(
|
|
rawText: string,
|
|
options?: { criteriaTemplate?: (intent: string) => string },
|
|
): ActionQueueStep[] {
|
|
const trimmed = rawText.trim();
|
|
if (!trimmed) return [];
|
|
|
|
const criteriaFn =
|
|
options?.criteriaTemplate ?? ((intent: string) => `Verify ${intent} completed successfully`);
|
|
|
|
const parts = trimmed
|
|
.split(STEP_SPLIT_REGEX)
|
|
.map((s) => s.trim())
|
|
.filter((s) => s.length > 0);
|
|
|
|
const created: ActionQueueStep[] = [];
|
|
for (const part of parts) {
|
|
const step: ActionQueueStep = {
|
|
id: `step-${this.steps.length + 1}`,
|
|
intent: part,
|
|
outcomeCriteria: criteriaFn(part),
|
|
status: "pending",
|
|
};
|
|
this.steps.push(step);
|
|
created.push(step);
|
|
}
|
|
return created;
|
|
}
|
|
|
|
cancelActive(_reason?: string): ActionQueueStep | null {
|
|
if (!this.activeStepId) return null;
|
|
const step = this.steps.find((s) => s.id === this.activeStepId);
|
|
if (!step) {
|
|
this.activeStepId = null;
|
|
return null;
|
|
}
|
|
|
|
if (step.status === "running" || step.status === "pending") {
|
|
step.status = "cancelled";
|
|
step.completedAt = Date.now();
|
|
}
|
|
this.activeStepId = null;
|
|
return step;
|
|
}
|
|
|
|
advance(): ActionQueueStep | null {
|
|
// If a step is currently running, complete it
|
|
if (this.activeStepId) {
|
|
const active = this.steps.find((s) => s.id === this.activeStepId);
|
|
if (active && active.status === "running") {
|
|
active.status = "completed";
|
|
active.completedAt = Date.now();
|
|
}
|
|
}
|
|
|
|
// Do not auto-advance past a failed step
|
|
const hasFailed = this.steps.some((s) => s.status === "failed");
|
|
if (hasFailed) {
|
|
this.activeStepId = null;
|
|
return null;
|
|
}
|
|
|
|
// Advance to next pending step
|
|
const next = this.steps.find((s) => s.status === "pending");
|
|
if (next) {
|
|
next.status = "running";
|
|
next.startedAt = Date.now();
|
|
this.activeStepId = next.id;
|
|
return next;
|
|
}
|
|
|
|
this.activeStepId = null;
|
|
return null;
|
|
}
|
|
|
|
updateStepStatus(stepId: string, status: ActionStepStatus, _reason?: string): ActionQueueStep {
|
|
const step = this.steps.find((s) => s.id === stepId);
|
|
if (!step) {
|
|
throw new Error(`Step not found: ${stepId}`);
|
|
}
|
|
|
|
assertStepTransition(step.status, status);
|
|
|
|
step.status = status;
|
|
const now = Date.now();
|
|
|
|
if (status === "running") {
|
|
step.startedAt = step.startedAt ?? now;
|
|
this.activeStepId = step.id;
|
|
} else {
|
|
if (status === "completed" || status === "cancelled" || status === "failed") {
|
|
step.completedAt = now;
|
|
}
|
|
if (this.activeStepId === step.id) {
|
|
this.activeStepId = null;
|
|
}
|
|
}
|
|
|
|
return step;
|
|
}
|
|
|
|
reorderSteps(fromIndex: number, toIndex: number): void {
|
|
if (
|
|
fromIndex < 0 ||
|
|
fromIndex >= this.steps.length ||
|
|
toIndex < 0 ||
|
|
toIndex >= this.steps.length
|
|
) {
|
|
throw new RangeError(
|
|
`Invalid step reorder indices: ${fromIndex} -> ${toIndex} (queue length: ${this.steps.length})`,
|
|
);
|
|
}
|
|
if (fromIndex === toIndex) return;
|
|
|
|
const [moved] = this.steps.splice(fromIndex, 1);
|
|
if (moved) {
|
|
this.steps.splice(toIndex, 0, moved);
|
|
}
|
|
}
|
|
|
|
insertStep(
|
|
stepData: {
|
|
id?: string;
|
|
intent: string;
|
|
outcomeCriteria?: string;
|
|
status?: ActionStepStatus;
|
|
startedAt?: number;
|
|
completedAt?: number;
|
|
},
|
|
atIndex?: number,
|
|
): ActionQueueStep {
|
|
const id = stepData.id ?? `step-${this.steps.length + 1}`;
|
|
const step: ActionQueueStep = {
|
|
id,
|
|
intent: stepData.intent,
|
|
outcomeCriteria:
|
|
stepData.outcomeCriteria ?? `Verify ${stepData.intent} completed successfully`,
|
|
status: stepData.status ?? "pending",
|
|
startedAt: stepData.startedAt,
|
|
completedAt: stepData.completedAt,
|
|
};
|
|
|
|
if (typeof atIndex === "number" && atIndex >= 0 && atIndex <= this.steps.length) {
|
|
this.steps.splice(atIndex, 0, step);
|
|
} else {
|
|
this.steps.push(step);
|
|
}
|
|
|
|
return step;
|
|
}
|
|
|
|
removeStep(stepId: string): boolean {
|
|
const index = this.steps.findIndex((s) => s.id === stepId);
|
|
if (index === -1) return false;
|
|
|
|
if (this.activeStepId === stepId) {
|
|
this.activeStepId = null;
|
|
}
|
|
this.steps.splice(index, 1);
|
|
return true;
|
|
}
|
|
|
|
retryStep(stepId: string): ActionQueueStep {
|
|
const step = this.steps.find((s) => s.id === stepId);
|
|
if (!step) {
|
|
throw new Error(`Step not found: ${stepId}`);
|
|
}
|
|
if (step.status !== "failed") {
|
|
throw new Error(
|
|
`Cannot retry step with status "${step.status}"; only failed steps can be retried`,
|
|
);
|
|
}
|
|
|
|
step.status = "pending";
|
|
step.completedAt = undefined;
|
|
return step;
|
|
}
|
|
|
|
getStep(stepId: string): ActionQueueStep | undefined {
|
|
return this.steps.find((s) => s.id === stepId);
|
|
}
|
|
|
|
getActiveStep(): ActionQueueStep | null {
|
|
if (!this.activeStepId) return null;
|
|
return this.steps.find((s) => s.id === this.activeStepId) ?? null;
|
|
}
|
|
|
|
getPendingSteps(): ActionQueueStep[] {
|
|
return this.steps.filter((s) => s.status === "pending");
|
|
}
|
|
|
|
isComplete(): boolean {
|
|
return (
|
|
this.steps.length > 0 &&
|
|
this.steps.every(
|
|
(s) => s.status === "completed" || s.status === "cancelled" || s.status === "failed",
|
|
)
|
|
);
|
|
}
|
|
|
|
hasFailed(): boolean {
|
|
return this.steps.some((s) => s.status === "failed");
|
|
}
|
|
|
|
clear(): void {
|
|
this.steps = [];
|
|
this.activeStepId = null;
|
|
}
|
|
|
|
toJSON(): ActionQueueSnapshot {
|
|
return {
|
|
steps: this.steps.map((s) => ({ ...s })),
|
|
activeStepId: this.activeStepId,
|
|
};
|
|
}
|
|
|
|
static fromJSON(snapshot: ActionQueueSnapshot): DynamicActionQueueImpl {
|
|
return new DynamicActionQueueImpl(snapshot.steps, snapshot.activeStepId);
|
|
}
|
|
}
|
|
|
|
/** Convenient alias */
|
|
export const ActionQueue = DynamicActionQueueImpl;
|