diff --git a/apps/mobile/app/voice.tsx b/apps/mobile/app/voice.tsx index 6ceaa84..d323103 100644 --- a/apps/mobile/app/voice.tsx +++ b/apps/mobile/app/voice.tsx @@ -196,9 +196,7 @@ export default function VoiceSettings() { <> {isKokoro ? ( - {t( - "No API key. Start Kokoro-FastAPI on port 8880, then tap Connect local Kokoro.", - )} + {t("Local voice runs on your server. Tap Connect when it is ready.")} ) : ( {credential ? ( diff --git a/apps/web/e2e/helpers.ts b/apps/web/e2e/helpers.ts index 1142e0b..d7d7911 100644 --- a/apps/web/e2e/helpers.ts +++ b/apps/web/e2e/helpers.ts @@ -51,6 +51,12 @@ export async function completeOnboarding(page: Page, testInfo?: TestInfo) { } } + // Voice step: skip when shown + const voiceHeading = page.getByRole("heading", { name: /Pick a voice/i }); + if (await voiceHeading.isVisible({ timeout: 5000 }).catch(() => false)) { + await page.getByRole("button", { name: /^Skip$/i }).click(); + } + // Optional Server integrations step (needsSetup). Skip when shown, then the // first bot is created automatically — land in Chief's chat with no form. const integrations = page.getByRole("heading", { name: "Server integrations", exact: true }); diff --git a/apps/web/scripts/translations-de.json b/apps/web/scripts/translations-de.json index d26197a..7426408 100644 --- a/apps/web/scripts/translations-de.json +++ b/apps/web/scripts/translations-de.json @@ -606,5 +606,21 @@ "Nothing is still running": "Nichts läuft mehr", "Credentials are encrypted and never sent to the model.": "Zugangsdaten werden verschlüsselt und dem Modell nie gesendet.", "Rename section": "Abschnitt umbenennen", - "Could not rename section": "Abschnitt konnte nicht umbenannt werden" + "Could not rename section": "Abschnitt konnte nicht umbenannt werden", + "Can speak": "Kann sprechen", + "Does not speak": "Spricht nicht", + "Your Oo reads replies aloud when this is on.": "Dein Oo liest Antworten laut vor, wenn das an ist.", + "Speaking is on.": "Sprechen ist an.", + "Speaking is off.": "Sprechen ist aus.", + "Could not enable speaking": "Sprechen konnte nicht aktiviert werden", + "Could not update speaking": "Sprechen konnte nicht aktualisiert werden", + "Other voice providers": "Andere Sprachanbieter", + "Provider": "Anbieter", + "Loading voice settings…": "Spracheinstellungen werden geladen…", + "Pick a voice": "Stimme wählen", + "How should your Oo sound when reading replies aloud?": "Wie soll dein Oo klingen, wenn er Antworten vorliest?", + "Loading voices…": "Stimmen werden geladen…", + "Voice is not available yet. You can turn it on later in settings.": "Stimme ist noch nicht verfügbar. Du kannst sie später in den Einstellungen aktivieren.", + "Voice set to {0}.": "Stimme auf {0} gesetzt.", + "Could not update voice": "Stimme konnte nicht aktualisiert werden" } diff --git a/apps/web/src/components/onboarding/OnboardingVoiceStep.tsx b/apps/web/src/components/onboarding/OnboardingVoiceStep.tsx new file mode 100644 index 0000000..cfaf5fd --- /dev/null +++ b/apps/web/src/components/onboarding/OnboardingVoiceStep.tsx @@ -0,0 +1,107 @@ +import { Trans } from "@lingui/react/macro"; +import type { VoiceInfo } from "@rakazo/contracts"; +import { defaultVoiceForLocale, preferVoicesForLocale } from "@rakazo/core"; +import { Button } from "@rakazo/ui-web"; +import { useEffect, useState } from "react"; +import { rpc } from "../../lib/rpc"; +import { readUiLocale } from "../../lib/ui-locale"; + +/** Must match packages/adapters KokoroVoiceProvider placeholder. */ +export const KOKORO_LOCAL_TOKEN = "local-kokoro"; + +type OnboardingVoiceStepProps = { + selectedVoiceId: string; + onSelectedVoiceIdChange: (voiceId: string) => void; + onBack: () => void; + onSkip: () => void; + onContinue: () => void; +}; + +export function OnboardingVoiceStep({ + selectedVoiceId, + onSelectedVoiceIdChange, + onBack, + onSkip, + onContinue, +}: OnboardingVoiceStepProps) { + const [voiceOptions, setVoiceOptions] = useState([]); + const [voiceLoading, setVoiceLoading] = useState(true); + + useEffect(() => { + let cancelled = false; + setVoiceLoading(true); + void (async () => { + try { + await rpc.voice.connect({ provider: "kokoro", apiKey: KOKORO_LOCAL_TOKEN }); + const listed = await rpc.voice.voices({ provider: "kokoro" }); + if (cancelled) return; + const locale = readUiLocale(); + const ordered = preferVoicesForLocale(listed, locale).slice(0, 6); + setVoiceOptions(ordered); + const defaultVoice = defaultVoiceForLocale(listed, locale); + if (defaultVoice) onSelectedVoiceIdChange(defaultVoice.id); + } catch { + if (!cancelled) setVoiceOptions([]); + } finally { + if (!cancelled) setVoiceLoading(false); + } + })(); + return () => { + cancelled = true; + }; + }, [onSelectedVoiceIdChange]); + + return ( +
+

+ Pick a voice +

+

+ How should your Oo sound when reading replies aloud? +

+ {voiceLoading ? ( +

+ Loading voices… +

+ ) : voiceOptions.length > 0 ? ( +
+ {voiceOptions.map((voice) => ( + + ))} +
+ ) : ( +

+ Voice is not available yet. You can turn it on later in settings. +

+ )} +
+ +
+ + +
+
+
+ ); +} diff --git a/apps/web/src/lib/ui-locale.ts b/apps/web/src/lib/ui-locale.ts index 774cabb..8fc9ff7 100644 --- a/apps/web/src/lib/ui-locale.ts +++ b/apps/web/src/lib/ui-locale.ts @@ -121,6 +121,20 @@ export function resolveUiLocale(options: ResolveUiLocaleOptions = {}): UiLocale return "en"; } +export function readUiLocale( + storage: Pick | null = typeof localStorage !== "undefined" + ? localStorage + : null, +): UiLocale { + if (!storage) return "en"; + try { + const stored = storage.getItem(UI_LOCALE_STORAGE_KEY); + return isUiLocale(stored) ? stored : "en"; + } catch { + return "en"; + } +} + export function persistUiLocale( locale: UiLocale, storage: Pick | null = typeof localStorage !== "undefined" diff --git a/apps/web/src/pages/Onboarding.tsx b/apps/web/src/pages/Onboarding.tsx index 128e300..3d126dd 100644 --- a/apps/web/src/pages/Onboarding.tsx +++ b/apps/web/src/pages/Onboarding.tsx @@ -1,5 +1,5 @@ import { Trans, useLingui } from "@lingui/react/macro"; -import type { IntegrationSetupState, Me } from "@rakazo/contracts"; +import type { IntegrationSetupState } from "@rakazo/contracts"; import { Button, Input } from "@rakazo/ui-web"; import { useEffect, useId, useMemo, useRef, useState } from "react"; import { useNavigate } from "react-router-dom"; @@ -9,6 +9,10 @@ import { type OnboardingEmojiConfig, OnboardingOoDesigner, } from "../components/onboarding/OnboardingOoDesigner"; +import { + KOKORO_LOCAL_TOKEN, + OnboardingVoiceStep, +} from "../components/onboarding/OnboardingVoiceStep"; import { computeDiscoveryItems, countConfiguredIntegrations, @@ -142,8 +146,10 @@ export function OnboardingPage() { const navigate = useNavigate(); const fieldId = useId(); const [step, setStep] = useState< - "loading" | "emoji" | "preferences" | "language" | "integrations" | "bot" + "loading" | "emoji" | "preferences" | "language" | "voice" | "integrations" | "bot" >("loading"); + const [selectedVoiceId, setSelectedVoiceId] = useState(""); + const [voiceSkipped, setVoiceSkipped] = useState(false); const [integrationSetup, setIntegrationSetup] = useState(null); const needsIntegrationSetup = integrationSetup?.needsSetup ?? false; const [integrationServers, setIntegrationServers] = useState([]); @@ -184,7 +190,10 @@ export function OnboardingPage() { const discoveryPhase: OnboardingDiscoveryPhase = step === "loading" ? "loading" - : step === "emoji" || step === "preferences" || step === "language" + : step === "emoji" || + step === "preferences" || + step === "language" || + step === "voice" ? "emoji" : step === "integrations" ? "integrations" @@ -231,6 +240,16 @@ export function OnboardingPage() { ctxParts.length > 0 ? `## User Context\n${ctxParts.map((p) => `- ${p}`).join("\n")}` : ""; await rpc.bots.update({ botId: bot.id, instructions: userCtx }).catch(() => null); + if (selectedVoiceId && !voiceSkipped) { + await rpc.voice + .connect({ provider: "kokoro", apiKey: KOKORO_LOCAL_TOKEN, voiceId: selectedVoiceId }) + .catch(() => null); + await rpc.voice.setVoice({ voiceId: selectedVoiceId, provider: "kokoro" }).catch(() => null); + await rpc.bots + .update({ botId: bot.id, voiceId: selectedVoiceId, autoSpeak: true }) + .catch(() => null); + } + const me = await rpc.me().catch(() => null); if (me) setBootstrapMe(me); const harvested = await persistUserProfileHarvest( @@ -263,6 +282,11 @@ export function OnboardingPage() { void createFirstBot(); }, [step]); + function goAfterVoice(skipped: boolean) { + setVoiceSkipped(skipped); + setStep(needsIntegrationSetup ? "integrations" : "bot"); + } + return (
{error && step !== "bot" ?

{error}

: null} - {/* Step: Emoji Configuration */} {step === "emoji" ? ( ) : null} - {/* Step: Preferences */} {step === "preferences" ? (

@@ -412,7 +435,6 @@ export function OnboardingPage() {

) : null} - {/* Step: Language */} {step === "language" ? (

@@ -429,7 +451,7 @@ export function OnboardingPage() { onClick={() => { persistUiLocale(code as UiLocale); setCurrentLocale(code); - setStep(needsIntegrationSetup ? "integrations" : "bot"); + setStep("voice"); }} className={`rounded-xl border-2 px-3 py-3 text-sm font-medium text-left transition-all ${ currentLocale === code @@ -445,14 +467,23 @@ export function OnboardingPage() { -

) : null} - {/* Step: Integrations */} + {step === "voice" ? ( + setStep("language")} + onSkip={() => goAfterVoice(true)} + onContinue={() => goAfterVoice(false)} + /> + ) : null} + {step === "integrations" ? (

@@ -469,7 +500,6 @@ export function OnboardingPage() {

) : null} - {/* Step: Bot creation error */} {step === "bot" && error ? (

{error}

diff --git a/apps/web/src/pages/SettingsOverlay.tsx b/apps/web/src/pages/SettingsOverlay.tsx index 97492ab..8562d4d 100644 --- a/apps/web/src/pages/SettingsOverlay.tsx +++ b/apps/web/src/pages/SettingsOverlay.tsx @@ -263,7 +263,12 @@ export function SettingsOverlay({ /> ) : null} {section === "voice" ? ( - + ) : null}
diff --git a/apps/web/src/pages/Shell.tsx b/apps/web/src/pages/Shell.tsx index 7a4992f..4aa0ee5 100644 --- a/apps/web/src/pages/Shell.tsx +++ b/apps/web/src/pages/Shell.tsx @@ -57,6 +57,8 @@ import { serializeComposerPrompt, speechFromBlocks, truncateSlashDescription, + matchVoiceByName, + parseVoiceChatCommand, userVisibleMessages, } from "@rakazo/core"; import { @@ -2339,6 +2341,50 @@ export function ShellPage() { if (permissionRequest) void permissionRequest.then(flushPendingBrowserNotifications); } const trimmed = plan.trimmed; + if (botTarget && !groupTarget && plan.shouldSend && attachments.length === 0) { + const voiceCmd = parseVoiceChatCommand(trimmed); + if (voiceCmd) { + setSendError(null); + setAttachmentNotice(null); + try { + const localVoiceToken = "local-kokoro"; + if (voiceCmd.kind === "speak_off") { + await rpc.bots.update({ botId: botTarget, autoSpeak: false }); + await refreshBots(); + setAttachmentNotice(t`Speaking is off.`); + return; + } + if (voiceCmd.kind === "speak_on") { + await rpc.voice + .connect({ provider: "kokoro", apiKey: localVoiceToken }) + .catch(() => undefined); + await rpc.bots.update({ botId: botTarget, autoSpeak: true }); + await refreshBots(); + void rpc.voice.status().then(setVoiceStatus).catch(() => undefined); + setAttachmentNotice(t`Speaking is on.`); + return; + } + const voices = await rpc.voice.voices({}); + const match = matchVoiceByName(voices, voiceCmd.voiceName); + if (!match) { + setSendError(t`Voice "${voiceCmd.voiceName}" not found.`); + return; + } + await rpc.voice + .connect({ provider: "kokoro", apiKey: localVoiceToken }) + .catch(() => undefined); + await rpc.voice.setVoice({ voiceId: match.id, provider: "kokoro" }); + await rpc.bots.update({ botId: botTarget, voiceId: match.id, autoSpeak: true }); + await refreshBots(); + void rpc.voice.status().then(setVoiceStatus).catch(() => undefined); + setAttachmentNotice(t`Voice set to ${match.label}.`); + return; + } catch (error) { + setSendError(error instanceof Error ? error.message : t`Could not update voice`); + return; + } + } + } setSending(true); setSendError(null); const dropDelayedSetup = () => { @@ -2463,6 +2509,7 @@ export function ShellPage() { flushPendingBrowserNotifications, navigate, pendingAttachments, + refreshBots, sending, t, ], diff --git a/apps/web/src/pages/VoiceSettingsOverlay.tsx b/apps/web/src/pages/VoiceSettingsOverlay.tsx index bca50d5..8911160 100644 --- a/apps/web/src/pages/VoiceSettingsOverlay.tsx +++ b/apps/web/src/pages/VoiceSettingsOverlay.tsx @@ -1,5 +1,6 @@ import { Trans, useLingui } from "@lingui/react/macro"; import type { VoiceCatalogEntry, VoiceCredential, VoiceInfo, VoiceStatus } from "@rakazo/contracts"; +import { preferVoicesForLocale } from "@rakazo/core"; import { Button, Dialog, @@ -11,25 +12,31 @@ import { Input, NativeSelect, NativeSelectOption, + Switch, } from "@rakazo/ui-web"; -import { XIcon } from "lucide-react"; +import { ChevronDownIcon, XIcon } from "lucide-react"; import { useEffect, useId, useMemo, useState } from "react"; import { ooSound } from "../lib/oo-sound"; import { readOoSoundEnabled } from "../lib/oo-sound-pref"; import { rpc } from "../lib/rpc"; +import { readUiLocale } from "../lib/ui-locale"; /** Must match packages/adapters KokoroVoiceProvider placeholder. */ const KOKORO_LOCAL_TOKEN = "local-kokoro"; +const LOCAL_PROVIDER_ID = "kokoro"; export function VoiceSettingsOverlay({ onClose, embedded = false, onBusyChange, + activeBotId, }: { onClose: () => void; /** Render panel body only for the shared Settings shell. */ embedded?: boolean; onBusyChange?: (busy: boolean) => void; + /** Active bot whose autoSpeak toggle is controlled here. */ + activeBotId?: string; }) { const { t } = useLingui(); const apiKeyId = useId(); @@ -38,47 +45,64 @@ export function VoiceSettingsOverlay({ const [credentials, setCredentials] = useState([]); const [status, setStatus] = useState(null); const [voices, setVoices] = useState([]); - const [provider, setProvider] = useState(""); - const [apiKey, setApiKey] = useState(""); const [voiceId, setVoiceId] = useState(""); + const [autoSpeak, setAutoSpeak] = useState(false); const [loading, setLoading] = useState(true); - const [pending, setPending] = useState<"connect" | "disconnect" | "voice" | "test" | null>(null); + const [pending, setPending] = useState< + "connect" | "disconnect" | "voice" | "test" | "speak" | null + >(null); const [error, setError] = useState(null); const [notice, setNotice] = useState(null); const [soundEnabled, setSoundEnabled] = useState(() => readOoSoundEnabled()); + const [advancedOpen, setAdvancedOpen] = useState(false); + const [advancedProvider, setAdvancedProvider] = useState(""); + const [advancedApiKey, setAdvancedApiKey] = useState(""); const busy = pending !== null; + const uiLocale = readUiLocale(); + const localConnected = credentials.some((entry) => entry.provider === LOCAL_PROVIDER_ID); + const advancedProviders = catalog.filter((entry) => entry.id !== LOCAL_PROVIDER_ID); + const selectedAdvanced = + advancedProviders.find((entry) => entry.id === advancedProvider) ?? advancedProviders[0]; + const advancedCredential = credentials.find((entry) => entry.provider === advancedProvider); useEffect(() => { return () => onBusyChange?.(false); }, [onBusyChange]); - function markPending(next: "connect" | "disconnect" | "voice" | "test" | null) { + function markPending(next: "connect" | "disconnect" | "voice" | "test" | "speak" | null) { setPending(next); onBusyChange?.(next !== null); } - async function refresh(nextProvider?: string) { + async function refresh() { const [nextCatalog, nextCredentials, nextStatus] = await Promise.all([ rpc.voice.catalog(), rpc.voice.credentials(), rpc.voice.status(), ]); - const selected = nextProvider || provider || nextStatus.provider || nextCatalog[0]?.id || ""; setCatalog(nextCatalog); setCredentials(nextCredentials); setStatus(nextStatus); - setProvider(selected); - const cred = nextCredentials.find((entry) => entry.provider === selected); - const activeVoice = cred?.voiceId ?? ""; + const localCred = nextCredentials.find((entry) => entry.provider === LOCAL_PROVIDER_ID); + const activeVoice = localCred?.voiceId ?? nextStatus.voiceId ?? ""; setVoiceId(activeVoice); - if (cred) { - const listed = await rpc.voice.voices({ provider: selected }); + if (localCred || nextStatus.ready) { + const listed = await rpc.voice.voices({ provider: LOCAL_PROVIDER_ID }).catch(() => []); setVoices(listed); if (!activeVoice && listed[0]) setVoiceId(listed[0].id); } else { setVoices([]); } + if (activeBotId) { + const bots = await rpc.bots.list(); + const bot = bots.find((entry) => entry.id === activeBotId); + setAutoSpeak(bot?.autoSpeak ?? false); + } + if (!advancedProvider) { + const firstAdvanced = nextCatalog.find((entry) => entry.id !== LOCAL_PROVIDER_ID); + if (firstAdvanced) setAdvancedProvider(firstAdvanced.id); + } } useEffect(() => { @@ -87,49 +111,31 @@ export function VoiceSettingsOverlay({ setError(err instanceof Error ? err.message : t`Could not load voice settings`), ) .finally(() => setLoading(false)); - }, []); + }, [activeBotId]); - const selected = catalog.find((entry) => entry.id === provider) ?? catalog[0]; - const isKokoro = selected?.id === "kokoro"; - const credential = credentials.find((entry) => entry.provider === provider); - const canConnect = isKokoro || apiKey.trim().length >= 8; - const voiceOptions = useMemo( - () => (voices.length ? voices : voiceId ? [{ id: voiceId, label: voiceId }] : []), - [voices, voiceId], - ); + const voiceOptions = useMemo(() => { + const listed = preferVoicesForLocale(voices, uiLocale); + if (listed.length) return listed; + return voiceId ? [{ id: voiceId, label: voiceId }] : []; + }, [voices, voiceId, uiLocale]); - async function connectKey() { - if (!selected || !canConnect) return; - setError(null); - setNotice(null); + const speakingReady = Boolean(localConnected && voiceId && status?.ready); + + async function ensureLocalVoice(): Promise { + if (localConnected) return true; markPending("connect"); + setError(null); try { await rpc.voice.connect({ - provider: selected.id, - apiKey: isKokoro ? KOKORO_LOCAL_TOKEN : apiKey.trim(), + provider: LOCAL_PROVIDER_ID, + apiKey: KOKORO_LOCAL_TOKEN, voiceId: voiceId || undefined, }); - setApiKey(""); - await refresh(selected.id); - setNotice(t`Connected ${selected.name}.`); + await refresh(); + return true; } catch (err) { - setError(err instanceof Error ? err.message : t`Could not connect this voice provider`); - } finally { - markPending(null); - } - } - - async function disconnectProvider() { - if (!credential) return; - setError(null); - setNotice(null); - markPending("disconnect"); - try { - await rpc.voice.disconnect({ provider: credential.provider }); - setApiKey(""); - await refresh(credential.provider); - } catch (err) { - setError(err instanceof Error ? err.message : t`Could not disconnect this voice provider`); + setError(err instanceof Error ? err.message : t`Could not enable speaking`); + return false; } finally { markPending(null); } @@ -137,12 +143,12 @@ export function VoiceSettingsOverlay({ async function chooseVoice(nextVoiceId: string) { setVoiceId(nextVoiceId); - if (!credential) return; + if (!localConnected) return; markPending("voice"); setError(null); try { - await rpc.voice.setVoice({ voiceId: nextVoiceId, provider: selected?.id }); - await refresh(selected?.id); + await rpc.voice.setVoice({ voiceId: nextVoiceId, provider: LOCAL_PROVIDER_ID }); + await refresh(); } catch (err) { setError(err instanceof Error ? err.message : t`Could not save that voice`); } finally { @@ -150,6 +156,73 @@ export function VoiceSettingsOverlay({ } } + async function setSpeakingEnabled(next: boolean) { + setAutoSpeak(next); + markPending("speak"); + setError(null); + setNotice(null); + try { + if (next) { + const connected = await ensureLocalVoice(); + if (!connected) { + setAutoSpeak(false); + return; + } + const listed = await rpc.voice.voices({ provider: LOCAL_PROVIDER_ID }).catch(() => []); + const nextVoice = voiceId || listed[0]?.id; + if (nextVoice) { + await rpc.voice.setVoice({ voiceId: nextVoice, provider: LOCAL_PROVIDER_ID }); + setVoiceId(nextVoice); + } + } + if (activeBotId) { + await rpc.bots.update({ botId: activeBotId, autoSpeak: next }); + } + await refresh(); + setNotice(next ? t`Speaking is on.` : t`Speaking is off.`); + } catch (err) { + setAutoSpeak(!next); + setError(err instanceof Error ? err.message : t`Could not update speaking`); + } finally { + markPending(null); + } + } + + async function connectAdvanced() { + if (!selectedAdvanced || advancedApiKey.trim().length < 8) return; + markPending("connect"); + setError(null); + setNotice(null); + try { + await rpc.voice.connect({ + provider: selectedAdvanced.id, + apiKey: advancedApiKey.trim(), + }); + setAdvancedApiKey(""); + await refresh(); + setNotice(t`Connected ${selectedAdvanced.name}.`); + } catch (err) { + setError(err instanceof Error ? err.message : t`Could not connect this voice provider`); + } finally { + markPending(null); + } + } + + async function disconnectAdvanced() { + if (!advancedCredential) return; + markPending("disconnect"); + setError(null); + try { + await rpc.voice.disconnect({ provider: advancedCredential.provider }); + setAdvancedApiKey(""); + await refresh(); + } catch (err) { + setError(err instanceof Error ? err.message : t`Could not disconnect this voice provider`); + } finally { + markPending(null); + } + } + async function testVoice() { setError(null); setNotice(null); @@ -188,203 +261,187 @@ export function VoiceSettingsOverlay({ ) : null} -
-
-
- Providers -
-
- {catalog.map((entry) => { - const connected = credentials.some((cred) => cred.provider === entry.id); - return ( - - ); - })} +
+ {loading ? ( +

+ Loading voice settings… +

+ ) : null} + {error ?

{error}

: null} + {notice ?

{notice}

: null} + +
+
+
+

+ {autoSpeak ? Can speak : Does not speak} +

+

+ Your Oo reads replies aloud when this is on. +

+
+ void setSpeakingEnabled(checked)} + />
-
- {loading ? ( -

- Loading voice providers… -

- ) : null} - {error ?

{error}

: null} - {notice ?

{notice}

: null} - {selected ? ( - <> - {isKokoro ? ( -
-

- Local Kokoro (free) -

-

- - No paid API key. EvOo talks to Kokoro-FastAPI on your server (default - http://127.0.0.1:8880). Set KOKORO_BASE_URL if it runs elsewhere. - -

-

- docker run -p 8880:8880 ghcr.io/remsky/kokoro-fastapi-cpu:latest -

-
- ) : ( - + {autoSpeak && speakingReady ? ( + <> + + + Voice + + void chooseVoice(event.target.value)} + > + {voiceOptions.map((voice) => ( + + {voice.label} + {voice.description ? ` · ${voice.description}` : ""} + + ))} + + + + + ) : null} + +
+
+
+

+ Sound feedback +

+

+ + Calm audio cues for finished work, voice phases, and handoffs. Respects reduced + motion. + +

+
+ +
+
+ + {advancedProviders.length > 0 ? ( +
+ + {advancedOpen && selectedAdvanced ? ( +
+ + Provider + + { + setAdvancedProvider(event.target.value); + setAdvancedApiKey(""); + setError(null); + setNotice(null); + }} + > + {advancedProviders.map((entry) => ( + + {entry.name} + + ))} + + + + API key setApiKey(event.target.value)} - placeholder={credential ? t`Paste a replacement key` : t`Paste your API key`} + value={advancedApiKey} + onChange={(event) => setAdvancedApiKey(event.target.value)} + placeholder={ + advancedCredential ? t`Paste a replacement key` : t`Paste your API key` + } /> - )} - - {credential ? ( - ) : null} - -
-
-
-

- Sound feedback -

-

- - Calm audio cues for finished work, voice phases, and handoffs. Respects - reduced motion. - -

-
- -
-
- - {credential ? ( - <> - - - Voice - - void chooseVoice(event.target.value)} - > - {voiceOptions.map((voice) => ( - - {voice.label} - {voice.description ? ` · ${voice.description}` : ""} - - ))} - - + {advancedCredential ? ( - - ) : null} - - ) : null} -
+ ) : null} +
+ ) : null} +
+ ) : null}
); @@ -413,7 +470,7 @@ export function VoiceSettingsOverlay({ data-testid="voice-settings" aria-describedby={undefined} showCloseButton={false} - className="flex h-[min(680px,calc(100%-2rem))] w-[920px] flex-col gap-0 overflow-hidden rounded-2xl p-0 sm:h-[min(680px,calc(100%-5rem))] sm:max-w-[calc(100%-5rem)]" + className="flex h-[min(680px,calc(100%-2rem))] w-[min(720px,calc(100%-2rem))] flex-col gap-0 overflow-hidden rounded-2xl p-0 sm:h-[min(680px,calc(100%-5rem))] sm:max-w-[calc(100%-5rem)]" > {body} diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index c0cd0a5..cc22529 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -805,7 +805,7 @@ export const builtinAgentTools: ConnectorTool[] = [ { name: "update_bot", description: - "Update this bot's own profile fields that the user sees in chat: name (header and list label), title (short role line), and description. Call this when the user asks you to rename yourself or change your title/description. Do not claim you updated the profile without calling this tool.", + "Update this bot's own profile fields that the user sees in chat: name (header and list label), title (short role line), description, speaking voice, and whether replies are read aloud. Call this when the user asks you to rename yourself, change your title/description, pick a different voice (e.g. Eva, Bernd), or turn speaking on/off. Do not claim you updated the profile without calling this tool.", inputSchema: { type: "object", properties: { @@ -821,6 +821,15 @@ export const builtinAgentTools: ConnectorTool[] = [ type: "string", description: "Longer blurb describing what this bot does.", }, + voice_id: { + type: "string", + description: + "Voice id from the connected provider (e.g. df_eva, dm_bernd). Match the user's requested voice name.", + }, + auto_speak: { + type: "boolean", + description: "When true, read replies aloud. When false, stay silent.", + }, }, }, }, diff --git a/packages/adapters/src/executor.ts b/packages/adapters/src/executor.ts index 258c8bd..368481e 100644 --- a/packages/adapters/src/executor.ts +++ b/packages/adapters/src/executor.ts @@ -3151,13 +3151,24 @@ export function createRunExecutor(deps: ExecutorDeps) { return spawned; } if (name === "update_bot") { - const patch: { name?: string; title?: string; description?: string } = {}; + const patch: { + name?: string; + title?: string; + description?: string; + voiceId?: string | null; + autoSpeak?: boolean; + } = {}; if (args.name !== undefined) patch.name = String(args.name); if (args.title !== undefined) patch.title = String(args.title); if (args.description !== undefined) patch.description = String(args.description); + if (args.voice_id !== undefined) { + const nextVoiceId = String(args.voice_id).trim(); + patch.voiceId = nextVoiceId.length > 0 ? nextVoiceId : null; + } + if (args.auto_speak !== undefined) patch.autoSpeak = Boolean(args.auto_speak); if (Object.keys(patch).length === 0) { return finish({ - error: "Provide at least one of name, title, or description.", + error: "Provide at least one of name, title, description, voice_id, or auto_speak.", }); } if (patch.name !== undefined) { @@ -3198,7 +3209,14 @@ export function createRunExecutor(deps: ExecutorDeps) { const updated = await deps.prisma.bot.update({ where: { id: bot.id }, data: patch, - select: { id: true, name: true, title: true, description: true }, + select: { + id: true, + name: true, + title: true, + description: true, + voiceId: true, + autoSpeak: true, + }, }); try { await deps.events.append({ @@ -3223,6 +3241,8 @@ export function createRunExecutor(deps: ExecutorDeps) { name: updated.name, title: updated.title, description: updated.description, + voiceId: updated.voiceId, + autoSpeak: updated.autoSpeak, }); } if (name === "message_user") { @@ -3509,7 +3529,7 @@ export function createRunExecutor(deps: ExecutorDeps) { "create_space proposes a new privacy boundary inside the current organization. Use it when the user asks to create a space or separate data between teams or projects. It always pauses for explicit user approval; never claim the space exists before the tool succeeds.", "create_project creates a shared project group for 2–6 agents when the user asks for a new project. Include yourself plus relevant teammates in bot_ids. Creating the project is the whole action unless the user also asked to start work in it.", "spawn_bot creates a lasting regular bot (own chat, computer, memory) that appears in the user's bot list. If the user asked to create a bot, call spawn_bot once and stop. Do not run_subagent to demo it.", - "update_bot updates this bot's own name (chat header / list label), title, and description. When the user asks you to rename yourself or change your title or description, call update_bot — do not claim you changed them without the tool.", + "update_bot updates this bot's own name (chat header / list label), title, description, voice_id, and auto_speak. When the user asks you to rename yourself, change your title or description, pick a voice, or turn speaking on/off, call update_bot — do not claim you changed them without the tool.", "run_subagent is a short helper inside this turn only. It is not a bot, has no thread, and does not show in the list. Use it for parallel work you will summarize here.", botDirectory, "archive_bot safely archives a bot this bot created, and only that bot. Use it when the user asks to remove that bot or when it is finished and unused. The user can restore it or permanently delete it later. confirm_name must exactly match its name.", diff --git a/packages/adapters/src/kokoro-voice.ts b/packages/adapters/src/kokoro-voice.ts index aa54c43..0252986 100644 --- a/packages/adapters/src/kokoro-voice.ts +++ b/packages/adapters/src/kokoro-voice.ts @@ -8,6 +8,7 @@ import type { VoiceSynthesizeRequest, VoiceVerifyResult, } from "@rakazo/adapter-kit"; +import { formatVoiceLabel, GERMAN_VOICE_PRESETS } from "@rakazo/core"; import { readVoiceAudio, readVoiceJson, @@ -24,6 +25,7 @@ const MODEL = "kokoro"; /** Curated presets when the server is down or returns bare voice ids. */ export const KOKORO_FALLBACK_VOICES: VoiceInfo[] = [ + ...GERMAN_VOICE_PRESETS, { id: "af_heart", label: "Heart", description: "Warm · multilingual" }, { id: "af_bella", label: "Bella", description: "Expressive · multilingual" }, { id: "af_sky", label: "Sky", description: "Bright · multilingual" }, @@ -43,10 +45,7 @@ export function kokoroBaseUrl(): string { } export function kokoroUnreachable(): string { - return ( - "Kokoro lokal nicht erreichbar — starte Kokoro-FastAPI auf Port 8880, z. B. " + - "docker run -p 8880:8880 ghcr.io/remsky/kokoro-fastapi-cpu:latest" - ); + return "Lokaler Sprachdienst nicht erreichbar. Prüfe, ob der Sprachserver auf Port 8880 läuft."; } export class KokoroVoiceProvider implements VoiceProvider { @@ -67,9 +66,9 @@ export class KokoroVoiceProvider implements VoiceProvider { if (res.ok) return { ok: true }; return { ok: false, - message: voiceHttpError( + message: voiceHttpError( res.status, - "Kokoro", + "Local voice", "checking the local server", await readVoiceJson(res), ), @@ -86,7 +85,7 @@ export class KokoroVoiceProvider implements VoiceProvider { }); const body = await readVoiceJson(res, { requireValid: res.ok }); if (!res.ok) { - throw new Error(voiceHttpError(res.status, "Kokoro", "listing voices", body)); + throw new Error(voiceHttpError(res.status, "Local voice", "listing voices", body)); } const voices = voicesFrom(body); if (voices.length === 0) return [...KOKORO_FALLBACK_VOICES]; @@ -118,7 +117,7 @@ export class KokoroVoiceProvider implements VoiceProvider { } catch { throw new Error(kokoroUnreachable()); } - await requireOk(res, "Kokoro", "speaking"); + await requireOk(res, "Local voice", "speaking"); return { bytes: await readVoiceAudio(res, signal), mimeType: "audio/mpeg" }; } } @@ -150,8 +149,3 @@ function extractVoiceIds(body: unknown): string[] { .filter(Boolean); } -function formatVoiceLabel(id: string): string { - const suffix = id.replace(/^[a-z]{2}_/, ""); - if (!suffix) return id; - return suffix.charAt(0).toUpperCase() + suffix.slice(1).replace(/_/g, " "); -} diff --git a/packages/adapters/src/voice-factory.test.ts b/packages/adapters/src/voice-factory.test.ts index a43a5e3..e3f1d3b 100644 --- a/packages/adapters/src/voice-factory.test.ts +++ b/packages/adapters/src/voice-factory.test.ts @@ -168,7 +168,7 @@ describe("KokoroVoiceProvider", () => { const provider = new KokoroVoiceProvider(); const result = await provider.verify(KOKORO_LOCAL_TOKEN, ctx); expect(result.ok).toBe(false); - expect(result.message).toMatch(/Kokoro lokal nicht erreichbar/i); + expect(result.message).toMatch(/Lokaler Sprachdienst nicht erreichbar/i); }); it("maps plain-string and object voice payloads", async () => { diff --git a/packages/adapters/src/voice-factory.ts b/packages/adapters/src/voice-factory.ts index 4e5a63d..304d511 100644 --- a/packages/adapters/src/voice-factory.ts +++ b/packages/adapters/src/voice-factory.ts @@ -9,8 +9,8 @@ import { SCRIPTED_VOICE_CATALOG_ENTRY, ScriptedVoiceProvider } from "./scripted- export const VOICE_CATALOG = [ { id: "kokoro", - name: "Kokoro", - description: "Free local TTS (Kokoro-82M). No paid API key — run Kokoro-FastAPI on :8880.", + name: "Local voice", + description: "Free local speech on your server.", transcribe: false, }, { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index cdd7efb..83d9f49 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -45,3 +45,5 @@ export * from "./text-direction.js"; export * from "./thread-message-updates.js"; export * from "./thread-subscription.js"; export * from "./tool-activity.js"; +export * from "./voice-chat-command.js"; +export * from "./voice-labels.js"; diff --git a/packages/core/src/voice-chat-command.test.ts b/packages/core/src/voice-chat-command.test.ts new file mode 100644 index 0000000..50f6e0b --- /dev/null +++ b/packages/core/src/voice-chat-command.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { parseVoiceChatCommand } from "./voice-chat-command.js"; +import { defaultVoiceForLocale, matchVoiceByName } from "./voice-labels.js"; + +describe("parseVoiceChatCommand", () => { + it("parses slash voice commands", () => { + expect(parseVoiceChatCommand("/stimme Eva")).toEqual({ + kind: "set_voice", + voiceName: "Eva", + }); + expect(parseVoiceChatCommand("/voice bernd")).toEqual({ + kind: "set_voice", + voiceName: "bernd", + }); + }); + + it("parses natural German voice change requests", () => { + expect(parseVoiceChatCommand("Wechsel deine Stimme zu Bernd")).toEqual({ + kind: "set_voice", + voiceName: "Bernd", + }); + expect(parseVoiceChatCommand("Sprich mit der Stimme Eva")).toEqual({ + kind: "set_voice", + voiceName: "Eva", + }); + }); + + it("parses speak on/off toggles", () => { + expect(parseVoiceChatCommand("Sprich nicht mehr")).toEqual({ kind: "speak_off" }); + expect(parseVoiceChatCommand("/sprechen")).toEqual({ kind: "speak_on" }); + expect(parseVoiceChatCommand("Read replies aloud")).toEqual({ kind: "speak_on" }); + }); + + it("returns null for normal chat", () => { + expect(parseVoiceChatCommand("Summarize my inbox")).toBeNull(); + }); +}); + +describe("matchVoiceByName", () => { + const voices = [ + { id: "df_eva", label: "Eva", description: "Deutsch · warm" }, + { id: "dm_bernd", label: "Bernd", description: "Deutsch · klar" }, + ]; + + it("matches label, id, and partial names", () => { + expect(matchVoiceByName(voices, "Eva")?.id).toBe("df_eva"); + expect(matchVoiceByName(voices, "df_eva")?.id).toBe("df_eva"); + expect(matchVoiceByName(voices, "bernd")?.id).toBe("dm_bernd"); + }); +}); + +describe("defaultVoiceForLocale", () => { + it("prefers Eva for German locale", () => { + const voices = [ + { id: "af_heart", label: "Heart" }, + { id: "df_eva", label: "Eva" }, + { id: "dm_bernd", label: "Bernd" }, + ]; + expect(defaultVoiceForLocale(voices, "de")?.id).toBe("df_eva"); + }); +}); diff --git a/packages/core/src/voice-chat-command.ts b/packages/core/src/voice-chat-command.ts new file mode 100644 index 0000000..f2db652 --- /dev/null +++ b/packages/core/src/voice-chat-command.ts @@ -0,0 +1,57 @@ +export type VoiceChatCommand = + | { kind: "set_voice"; voiceName: string } + | { kind: "speak_on" } + | { kind: "speak_off" }; + +const SET_VOICE_PATTERNS = [ + /^(?:\/stimme|\/voice)\s+(.+)$/i, + /^(?:ändere|aendere|wechsel|nimm)\s+(?:deine\s+)?stimme\s+(?:zu|auf|in)\s+(.+)$/i, + /^(?:change|switch|use)\s+(?:your\s+)?voice\s+(?:to\s+)?(.+)$/i, + /^sprich\s+(?:mit\s+)?(?:der\s+)?stimme\s+(.+)$/i, + /^(?:nutze|verwende|nimm)\s+(?:die\s+)?stimme\s+(.+)$/i, +] as const; + +const SPEAK_ON_PATTERNS = [ + /^(?:\/sprechen|\/speak)$/i, + /^(?:kann\s+wieder\s+)?sprechen$/i, + /^(?:lies|lese)\s+(?:antworten\s+)?(?:wieder\s+)?(?:laut\s+)?vor$/i, + /^(?:turn\s+)?speak(?:ing)?\s+on$/i, + /^read\s+(?:replies\s+)?aloud$/i, +] as const; + +const SPEAK_OFF_PATTERNS = [ + /^(?:\/schweigen|\/mute)$/i, + /^(?:sprich\s+)?(?:nicht\s+mehr|nicht\s+wieder)$/i, + /^sprich\s+nicht$/i, + /^(?:turn\s+)?speak(?:ing)?\s+off$/i, + /^stop\s+speaking$/i, + /^schweig(?:e)?$/i, +] as const; + +function firstCapture(patterns: readonly RegExp[], text: string): string | undefined { + for (const pattern of patterns) { + const match = pattern.exec(text.trim()); + const captured = match?.[1]?.trim(); + if (captured) return captured; + } + return undefined; +} + +function matchesAny(patterns: readonly RegExp[], text: string): boolean { + const trimmed = text.trim(); + return patterns.some((pattern) => pattern.test(trimmed)); +} + +/** Parse a chat message for local voice control before sending to the agent. */ +export function parseVoiceChatCommand(text: string): VoiceChatCommand | null { + const trimmed = text.trim(); + if (!trimmed) return null; + + if (matchesAny(SPEAK_OFF_PATTERNS, trimmed)) return { kind: "speak_off" }; + if (matchesAny(SPEAK_ON_PATTERNS, trimmed)) return { kind: "speak_on" }; + + const voiceName = firstCapture(SET_VOICE_PATTERNS, trimmed); + if (voiceName) return { kind: "set_voice", voiceName }; + + return null; +} diff --git a/packages/core/src/voice-labels.ts b/packages/core/src/voice-labels.ts new file mode 100644 index 0000000..eef5380 --- /dev/null +++ b/packages/core/src/voice-labels.ts @@ -0,0 +1,57 @@ +import type { VoiceInfo } from "@rakazo/contracts"; + +/** Friendly German presets when Kokoro exposes df_/dm_ voices. */ +export const GERMAN_VOICE_PRESETS: VoiceInfo[] = [ + { id: "df_eva", label: "Eva", description: "Deutsch · warm" }, + { id: "dm_bernd", label: "Bernd", description: "Deutsch · klar" }, + { id: "df_victoria", label: "Victoria", description: "Deutsch · freundlich" }, + { id: "dm_martin", label: "Martin", description: "Deutsch · ruhig" }, +]; + +const PRESET_BY_ID = new Map(GERMAN_VOICE_PRESETS.map((voice) => [voice.id, voice])); + +/** Strip engine prefix (df_eva → Eva) for display when no preset exists. */ +export function formatVoiceLabel(id: string): string { + const preset = PRESET_BY_ID.get(id); + if (preset) return preset.label; + const suffix = id.replace(/^[a-z]{2}_/, ""); + if (!suffix) return id; + return suffix.charAt(0).toUpperCase() + suffix.slice(1).replace(/_/g, " "); +} + +export function voiceDescription(id: string, fallback?: string): string | undefined { + return PRESET_BY_ID.get(id)?.description ?? fallback; +} + +/** Prefer German voices when UI locale is de. */ +export function preferVoicesForLocale(voices: VoiceInfo[], locale: string): VoiceInfo[] { + if (locale !== "de") return voices; + const german = voices.filter((voice) => /^d[fm]_/.test(voice.id)); + if (german.length === 0) return voices; + const rest = voices.filter((voice) => !/^d[fm]_/.test(voice.id)); + return [...german, ...rest]; +} + +export function defaultVoiceForLocale(voices: VoiceInfo[], locale: string): VoiceInfo | undefined { + const ordered = preferVoicesForLocale(voices, locale); + if (locale === "de") { + return ordered.find((voice) => voice.id === "df_eva") ?? ordered[0]; + } + return ordered[0]; +} + +/** Match a user-provided name (Eva, bernd, df_eva) against available voices. */ +export function matchVoiceByName(voices: VoiceInfo[], query: string): VoiceInfo | undefined { + const needle = query.trim().toLowerCase(); + if (!needle) return undefined; + const byId = voices.find((voice) => voice.id.toLowerCase() === needle); + if (byId) return byId; + const byLabel = voices.find((voice) => voice.label.toLowerCase() === needle); + if (byLabel) return byLabel; + const suffix = needle.replace(/^[a-z]{2}_/, ""); + return voices.find((voice) => { + const label = voice.label.toLowerCase(); + const idSuffix = voice.id.replace(/^[a-z]{2}_/, "").toLowerCase(); + return label.includes(suffix) || idSuffix.includes(suffix) || suffix.includes(idSuffix); + }); +}