feat: lots of speech stuff

2026-05-01 19:28:02 -05:00
parent 9f3d393a4d
commit d5d7b5c6dc
41 changed files with 1405 additions and 361 deletions
@@ -1,4 +1,3 @@
-import type { Unstable_TriggerItem } from '@assistant-ui/core'
 import type { Unstable_IconComponent } from '@assistant-ui/react'
 import { FileText, FolderOpen, ImageIcon, Link, type LucideIcon } from 'lucide-react'
 import type { CSSProperties } from 'react'
@@ -37,7 +36,7 @@ export const DIRECTIVE_ICONS: Record<string, Unstable_IconComponent> = {
 }

 export const DIRECTIVE_POPOVER_CLASS =
-  'absolute bottom-24 left-1/2 z-50 w-[min(calc(100vw-1.5rem),28rem)] max-h-[min(28rem,calc(100vh-8rem))] -translate-x-1/2 overflow-y-auto overscroll-contain rounded-2xl border border-border/70 bg-popover p-1.5 text-popover-foreground shadow-2xl'
+  'absolute bottom-24 left-1/2 z-50 w-[min(calc(100vw-1.5rem),26rem)] max-h-[min(24rem,calc(100vh-8rem))] -translate-x-1/2 overflow-y-auto overscroll-contain rounded-2xl border border-border/60 bg-popover/95 p-1.5 text-popover-foreground shadow-2xl backdrop-blur-md ring-1 ring-black/5'

 export const PROMPT_SNIPPETS = [
  {
@@ -64,37 +63,6 @@ export const ASK_PLACEHOLDERS = [
  'Duck mode: gentle debugging, together.'
 ]

-export const REF_ITEMS: Unstable_TriggerItem[] = [
-  {
-    id: 'file:',
-    type: 'file',
-    label: 'File',
-    description: 'Attach a file path',
-    metadata: { icon: 'file' }
-  },
-  {
-    id: 'folder:',
-    type: 'folder',
-    label: 'Folder',
-    description: 'Attach a folder path',
-    metadata: { icon: 'folder' }
-  },
-  {
-    id: 'url:',
-    type: 'url',
-    label: 'URL',
-    description: 'Attach a web page',
-    metadata: { icon: 'url' }
-  },
-  {
-    id: 'image:',
-    type: 'image',
-    label: 'Image',
-    description: 'Attach an image path',
-    metadata: { icon: 'image' }
-  }
-]
-
 export const EDGE_NEWLINES_RE = /^[\t ]*(?:\r\n|\r|\n)+|(?:\r\n|\r|\n)+[\t ]*$/g
 export const DEFAULT_MAX_RECORDING_SECONDS = 120

@@ -15,11 +15,10 @@ import {
 import { cn } from '@/lib/utils'

 import { GHOST_ICON_BTN, PROMPT_SNIPPETS } from './constants'
-import type { ChatBarState, ContextSuggestion } from './types'
+import type { ChatBarState } from './types'

 export function ContextMenu({
  state,
-  onAddContextRef,
  onInsertText,
  onOpenUrlDialog,
  onPasteClipboardImage,
@@ -28,7 +27,6 @@ export function ContextMenu({
  onPickImages
 }: {
  state: ChatBarState
-  onAddContextRef?: (refText: string, label?: string, detail?: string) => void
  onInsertText: (text: string) => void
  onOpenUrlDialog: () => void
  onPasteClipboardImage?: () => void
@@ -36,11 +34,6 @@ export function ContextMenu({
  onPickFolders?: () => void
  onPickImages?: () => void
 }) {
-  const choose = (item: ContextSuggestion) =>
-    onAddContextRef ? onAddContextRef(item.text, item.display, item.meta) : onInsertText(item.text)
-
-  const suggestions = state.tools.suggestions?.slice(0, 8) ?? []
-
  return (
    <DropdownMenu>
      <DropdownMenuTrigger asChild>
@@ -56,48 +49,28 @@ export function ContextMenu({
          <Plus size={18} />
        </Button>
      </DropdownMenuTrigger>
-      <DropdownMenuContent align="start" className="w-64" side="top" sideOffset={10}>
-        <DropdownMenuLabel className="text-xs text-muted-foreground">Add context</DropdownMenuLabel>
+      <DropdownMenuContent align="start" className="w-60" side="top" sideOffset={10}>
+        <DropdownMenuLabel className="text-[0.7rem] font-medium uppercase tracking-wide text-muted-foreground/85">
+          Attach
+        </DropdownMenuLabel>
        <ContextMenuItem disabled={!onPickFiles} icon={FileText} onSelect={onPickFiles}>
-          Files
+          Files…
        </ContextMenuItem>
        <ContextMenuItem disabled={!onPickFolders} icon={FolderOpen} onSelect={onPickFolders}>
-          Folders
+          Folder…
        </ContextMenuItem>
        <ContextMenuItem disabled={!onPickImages} icon={ImageIcon} onSelect={onPickImages}>
-          Images
+          Images…
        </ContextMenuItem>
        <ContextMenuItem disabled={!onPasteClipboardImage} icon={Clipboard} onSelect={onPasteClipboardImage}>
-          Image from clipboard
+          Paste image
        </ContextMenuItem>
        <ContextMenuItem icon={Link} onSelect={onOpenUrlDialog}>
-          URL
+          URL…
        </ContextMenuItem>

        <DropdownMenuSeparator />

-        <DropdownMenuSub>
-          <DropdownMenuSubTrigger>
-            <FileText />
-            <span>Suggested files</span>
-          </DropdownMenuSubTrigger>
-          <DropdownMenuSubContent className="w-72">
-            {suggestions.length === 0 ? (
-              <DropdownMenuItem disabled>
-                <span className="text-muted-foreground">No suggestions</span>
-              </DropdownMenuItem>
-            ) : (
-              suggestions.map(item => (
-                <DropdownMenuItem key={item.text} onSelect={() => choose(item)}>
-                  <FileText />
-                  <span className="min-w-0 flex-1 truncate">{item.display}</span>
-                  {item.meta && <span className="max-w-28 truncate text-xs text-muted-foreground">{item.meta}</span>}
-                </DropdownMenuItem>
-              ))
-            )}
-          </DropdownMenuSubContent>
-        </DropdownMenuSub>
-
        <DropdownMenuSub>
          <DropdownMenuSubTrigger>
            <MessageSquareText />
@@ -111,6 +84,13 @@ export function ContextMenu({
            ))}
          </DropdownMenuSubContent>
        </DropdownMenuSub>
+
+        <DropdownMenuSeparator />
+
+        <div className="px-2 py-1 text-[0.7rem] text-muted-foreground/80">
+          Tip: type <kbd className="rounded bg-muted/70 px-1 py-px font-mono text-[0.65rem]">@</kbd> to reference files
+          inline.
+        </div>
      </DropdownMenuContent>
    </DropdownMenu>
  )
@@ -15,6 +15,7 @@ interface ConversationProps {
  status: ConversationStatus
  onEnd: () => void
  onStart: () => void
+  onStopTurn: () => void
  onToggleMute: () => void
 }

@@ -80,6 +81,7 @@ function ConversationPill({
  level,
  muted,
  onEnd,
+  onStopTurn,
  onToggleMute,
  status
 }: ConversationProps & { disabled: boolean }) {
@@ -104,10 +106,10 @@ function ConversationPill({
        aria-pressed={muted}
        className={cn(GHOST_ICON_BTN, 'p-0', muted && 'bg-muted text-muted-foreground')}
        disabled={disabled}
-      onClick={() => {
-        triggerHaptic('selection')
-        onToggleMute()
-      }}
+        onClick={() => {
+          triggerHaptic('selection')
+          onToggleMute()
+        }}
        size="icon"
        title={muted ? 'Unmute microphone' : 'Mute microphone'}
        type="button"
@@ -115,6 +117,23 @@ function ConversationPill({
      >
        {muted ? <MicOff size={16} /> : <Mic size={16} />}
      </Button>
+      {listening && (
+        <Button
+          aria-label="Stop listening and send"
+          className="h-8 shrink-0 gap-1.5 rounded-full px-2.5 text-xs text-muted-foreground hover:bg-accent hover:text-foreground"
+          disabled={disabled}
+          onClick={() => {
+            triggerHaptic('submit')
+            onStopTurn()
+          }}
+          title="Stop listening and send"
+          type="button"
+          variant="ghost"
+        >
+          <Square className="fill-current" size={11} />
+          <span>Stop</span>
+        </Button>
+      )}
      <Button
        aria-label="End voice conversation"
        className="h-8 gap-1.5 rounded-full bg-primary px-3 text-xs font-medium text-primary-foreground hover:bg-primary/90"
@@ -5,9 +5,9 @@ import {
  type Unstable_MentionCategory,
  type Unstable_MentionDirective
 } from '@assistant-ui/react'
-import { ChevronDown } from 'lucide-react'
+import { FileText } from 'lucide-react'

-import { DIRECTIVE_POPOVER_CLASS, REF_ITEMS } from './constants'
+import { DIRECTIVE_POPOVER_CLASS } from './constants'
 import type { ContextSuggestion } from './types'

 export function DirectivePopover({
@@ -24,80 +24,73 @@ export function DirectivePopover({
  return (
    <ComposerPrimitive.Unstable_TriggerPopover adapter={adapter} char="@" className={DIRECTIVE_POPOVER_CLASS}>
      <ComposerPrimitive.Unstable_TriggerPopover.Directive {...directive} />
-      <ComposerPrimitive.Unstable_TriggerPopoverCategories>
-        {categories => (
-          <div className="grid gap-1">
-            {categories.map(c => (
-              <ComposerPrimitive.Unstable_TriggerPopoverCategoryItem
-                categoryId={c.id}
-                className="flex w-full items-center justify-between rounded-xl px-3 py-2 text-left text-sm hover:bg-accent data-highlighted:bg-accent"
-                key={c.id}
-              >
-                <span>{c.label}</span>
-                <ChevronDown className="-rotate-90 size-3.5 text-muted-foreground" />
-              </ComposerPrimitive.Unstable_TriggerPopoverCategoryItem>
-            ))}
-          </div>
-        )}
-      </ComposerPrimitive.Unstable_TriggerPopoverCategories>
      <ComposerPrimitive.Unstable_TriggerPopoverItems>
        {items => (
-          <div className="grid gap-1">
-            <ComposerPrimitive.Unstable_TriggerPopoverBack className="mb-1 text-xs text-muted-foreground hover:text-foreground">
-              Back
-            </ComposerPrimitive.Unstable_TriggerPopoverBack>
-            {items.map((item, index) => {
-              const Icon = directiveIcon(item, iconMap, Fallback)
+          <div className="grid gap-0.5">
+            <div className="px-2 pb-1 pt-0.5 text-[0.7rem] font-medium uppercase tracking-wide text-muted-foreground/80">
+              Reference a file
+            </div>
+            {items.length === 0 ? (
+              <div className="px-3 py-3 text-sm text-muted-foreground">
+                <p>No file suggestions yet.</p>
+                <p className="mt-1 text-xs text-muted-foreground/80">
+                  Keep typing to filter, or click <span className="font-medium text-foreground/80">+</span> to attach
+                  files, folders, or a URL.
+                </p>
+              </div>
+            ) : (
+              items.map((item, index) => {
+                const Icon = directiveIcon(item, iconMap, Fallback)

-              return (
-                <ComposerPrimitive.Unstable_TriggerPopoverItem
-                  className="flex w-full items-center gap-2 rounded-xl px-3 py-2 text-left text-sm hover:bg-accent data-highlighted:bg-accent"
-                  index={index}
-                  item={item}
-                  key={`${item.type}:${item.id}`}
-                >
-                  <Icon className="size-4 shrink-0 text-muted-foreground" />
-                  <span className="grid min-w-0 flex-1 gap-0.5">
-                    <span className="truncate font-medium">{item.label}</span>
-                    {item.description && (
-                      <span className="truncate text-xs text-muted-foreground">{item.description}</span>
-                    )}
-                  </span>
-                </ComposerPrimitive.Unstable_TriggerPopoverItem>
-              )
-            })}
+                return (
+                  <ComposerPrimitive.Unstable_TriggerPopoverItem
+                    className="flex w-full items-center gap-2 rounded-xl px-2.5 py-1.5 text-left text-sm transition-colors hover:bg-accent/70 data-highlighted:bg-accent"
+                    index={index}
+                    item={item}
+                    key={`${item.type}:${item.id}`}
+                  >
+                    <Icon className="size-4 shrink-0 text-muted-foreground/80" />
+                    <span className="grid min-w-0 flex-1 gap-0.5">
+                      <span className="truncate font-medium text-foreground">{item.label}</span>
+                      {item.description && (
+                        <span className="truncate text-[0.72rem] text-muted-foreground/85">{item.description}</span>
+                      )}
+                    </span>
+                  </ComposerPrimitive.Unstable_TriggerPopoverItem>
+                )
+              })
+            )}
          </div>
        )}
      </ComposerPrimitive.Unstable_TriggerPopoverItems>
    </ComposerPrimitive.Unstable_TriggerPopover>
  )
 }
+
 export function buildMentionCategories(suggestions: ContextSuggestion[] | undefined): Unstable_MentionCategory[] {
-  const items = (suggestions ?? [])
-    .map(s => {
-      const match = s.text.match(/^@(file|folder|url|image):(.+)$/)
+  const items: Unstable_TriggerItem[] = []

-      if (!match) {
-        return null
-      }
+  for (const s of suggestions ?? []) {
+    const match = s.text.match(/^@(file|folder|url|image):(.+)$/)

-      const [, type, id] = match
+    if (!match) {
+      continue
+    }

-      return {
-        id,
-        type,
-        label: s.display || id,
-        description: s.meta,
-        metadata: { icon: type }
-      }
+    const [, type, id] = match
+
+    items.push({
+      id,
+      type,
+      label: s.display || id,
+      description: s.meta,
+      metadata: { icon: type }
    })
-    .filter((item): item is NonNullable<typeof item> => Boolean(item))
+  }

-  return [
-    { id: 'refs', label: 'Hermes refs', items: REF_ITEMS },
-    ...(items.length ? [{ id: 'context', label: 'Suggested files', items }] : [])
-  ]
+  return [{ id: 'context', label: 'References', items }]
 }
+
 function directiveIcon(
  item: Unstable_TriggerItem,
  iconMap: Record<string, Unstable_IconComponent>,
@@ -106,5 +99,5 @@ function directiveIcon(
  const meta = item.metadata as Record<string, unknown> | undefined
  const key = typeof meta?.icon === 'string' ? meta.icon : item.type

-  return iconMap[key] ?? iconMap[item.type] ?? fallback
+  return iconMap[key] ?? iconMap[item.type] ?? fallback ?? FileText
 }
@@ -1,6 +1,6 @@
 import { useCallback, useEffect, useRef, useState } from 'react'

-import { speakText } from '@/hermes'
+import { playSpeechText, stopVoicePlayback } from '@/lib/voice-playback'
 import { notify, notifyError } from '@/store/notifications'

 import {
@@ -14,13 +14,19 @@ import { useMicRecorder } from './use-mic-recorder'

 export type ConversationStatus = 'idle' | 'listening' | 'transcribing' | 'thinking' | 'speaking'

+interface PendingVoiceResponse {
+  id: string
+  pending: boolean
+  text: string
+}
+
 interface VoiceConversationOptions {
  busy: boolean
  enabled: boolean
  onFatalError?: () => void
-  onSubmit: (text: string) => void
+  onSubmit: (text: string) => Promise<void> | void
  onTranscribeAudio?: (audio: Blob) => Promise<string>
-  pendingResponseText: () => string | null
+  pendingResponse: () => PendingVoiceResponse | null
  consumePendingResponse: () => void
 }

@@ -30,16 +36,19 @@ export function useVoiceConversation({
  onFatalError,
  onSubmit,
  onTranscribeAudio,
-  pendingResponseText,
+  pendingResponse,
  consumePendingResponse
 }: VoiceConversationOptions) {
  const { handle, level } = useMicRecorder()
  const [status, setStatus] = useState<ConversationStatus>('idle')
  const [muted, setMuted] = useState(false)
-  const audioRef = useRef<HTMLAudioElement | null>(null)
  const turnTimeoutRef = useRef<number | null>(null)
  const pendingStartRef = useRef(false)
-  const lastSpokenRef = useRef<string | null>(null)
+  const turnClosingRef = useRef(false)
+  const awaitingSpokenResponseRef = useRef(false)
+  const responseIdRef = useRef<string | null>(null)
+  const spokenSourceLengthRef = useRef(0)
+  const speechBufferRef = useRef('')
  const enabledRef = useRef(enabled)
  const mutedRef = useRef(muted)
  const busyRef = useRef(busy)
@@ -69,36 +78,74 @@ export function useVoiceConversation({
    }
  }

-  const stopAudio = useCallback(() => {
-    const audio = audioRef.current
+  const resetSpeechBuffer = () => {
+    responseIdRef.current = null
+    spokenSourceLengthRef.current = 0
+    speechBufferRef.current = ''
+  }

-    if (audio) {
-      audio.pause()
-      audio.src = ''
-      audioRef.current = null
-    }
-  }, [])
-
-  const handleTurn = useCallback(async () => {
-    clearTurnTimeout()
-    setStatus('transcribing')
-    const result = await handle.stop()
-
-    if (!result || !result.heardSpeech || !onTranscribeAudio) {
-      if (enabledRef.current && !mutedRef.current && !busyRef.current && statusRef.current !== 'speaking') {
-        pendingStartRef.current = true
-      }
-
-      setStatus('idle')
+  const appendSpeechText = (text: string) => {
+    const cleaned = text

+    if (!cleaned) {
      return
    }

-    try {
-      const transcript = (await onTranscribeAudio(result.audio)).trim()
+    speechBufferRef.current = `${speechBufferRef.current} ${cleaned}`.trim()
+  }

-      if (!transcript) {
-        if (enabledRef.current) {
+  const takeSpeechChunk = (force = false): string | null => {
+    const buffer = speechBufferRef.current.replace(/\s+/g, ' ').trim()
+
+    if (!buffer) {
+      speechBufferRef.current = ''
+
+      return null
+    }
+
+    const sentence = buffer.match(/^(.+?[.!?。！？])(?:\s+|$)/)
+
+    if (sentence?.[1] && (sentence[1].length >= 8 || force)) {
+      const chunk = sentence[1].trim()
+      speechBufferRef.current = buffer.slice(sentence[1].length).trim()
+
+      return chunk
+    }
+
+    if (!force && buffer.length > 220) {
+      const softBoundary = Math.max(buffer.lastIndexOf(', ', 180), buffer.lastIndexOf('; ', 180), buffer.lastIndexOf(': ', 180))
+
+      if (softBoundary > 80) {
+        const chunk = buffer.slice(0, softBoundary + 1).trim()
+        speechBufferRef.current = buffer.slice(softBoundary + 1).trim()
+
+        return chunk
+      }
+    }
+
+    if (!force) {
+      return null
+    }
+
+    speechBufferRef.current = ''
+
+    return buffer
+  }
+
+  const handleTurn = useCallback(async (forceTranscribe = false) => {
+    if (turnClosingRef.current) {
+      return
+    }
+
+    turnClosingRef.current = true
+    clearTurnTimeout()
+    setStatus('transcribing')
+
+    try {
+      const result = await handle.stop()
+
+      if (!result || (!result.heardSpeech && !forceTranscribe) || !onTranscribeAudio) {
+        if (enabledRef.current && !mutedRef.current && !busyRef.current && statusRef.current !== 'speaking') {
          pendingStartRef.current = true
        }

@@ -107,16 +154,34 @@ export function useVoiceConversation({
        return
      }

-      onSubmit(transcript)
-      setStatus('thinking')
-    } catch (error) {
-      notifyError(error, 'Voice transcription failed')
+      try {
+        const transcript = (await onTranscribeAudio(result.audio)).trim()

-      if (enabledRef.current && !mutedRef.current && !busyRef.current) {
-        pendingStartRef.current = true
+        if (!transcript) {
+          if (enabledRef.current) {
+            pendingStartRef.current = true
+          }
+
+          setStatus('idle')
+
+          return
+        }
+
+        awaitingSpokenResponseRef.current = true
+        resetSpeechBuffer()
+        await onSubmit(transcript)
+        setStatus('thinking')
+      } catch (error) {
+        notifyError(error, 'Voice transcription failed')
+
+        if (enabledRef.current && !mutedRef.current && !busyRef.current) {
+          pendingStartRef.current = true
+        }
+
+        setStatus('idle')
      }
-
-      setStatus('idle')
+    } finally {
+      turnClosingRef.current = false
    }
  }, [handle, onSubmit, onTranscribeAudio])

@@ -158,24 +223,13 @@ export function useVoiceConversation({

  const speak = useCallback(
    async (text: string) => {
-      stopAudio()
      setStatus('speaking')

      try {
-        const response = await speakText(text)
-        const audio = new Audio(response.data_url)
-        audioRef.current = audio
-
-        await new Promise<void>((resolve, reject) => {
-          audio.addEventListener('ended', () => resolve(), { once: true })
-          audio.addEventListener('error', () => reject(new Error('Playback failed')), { once: true })
-          void audio.play().catch(reject)
-        })
+        await playSpeechText(text, { source: 'voice-conversation' })
      } catch (error) {
        notifyError(error, 'Voice playback failed')
      } finally {
-        audioRef.current = null
-
        if (enabledRef.current) {
          pendingStartRef.current = true
          setStatus('idle')
@@ -184,7 +238,7 @@ export function useVoiceConversation({
        }
      }
    },
-    [stopAudio]
+    []
  )

  const start = useCallback(async () => {
@@ -200,20 +254,31 @@ export function useVoiceConversation({
    }

    setMuted(false)
-    lastSpokenRef.current = null
+    awaitingSpokenResponseRef.current = false
+    resetSpeechBuffer()
+    consumePendingResponse()
    pendingStartRef.current = true
-  }, [onFatalError, onTranscribeAudio])
+    await startListening()
+  }, [consumePendingResponse, onFatalError, onTranscribeAudio, startListening])

  const end = useCallback(async () => {
    pendingStartRef.current = false
    clearTurnTimeout()
-    stopAudio()
+    stopVoicePlayback()
    handle.cancel()
-    lastSpokenRef.current = null
+    turnClosingRef.current = false
+    awaitingSpokenResponseRef.current = false
+    resetSpeechBuffer()
    consumePendingResponse()
    setMuted(false)
    setStatus('idle')
-  }, [consumePendingResponse, handle, stopAudio])
+  }, [consumePendingResponse, handle])
+
+  const stopTurn = useCallback(() => {
+    if (statusRef.current === 'listening') {
+      void handleTurn(true)
+    }
+  }, [handleTurn])

  const toggleMute = useCallback(() => {
    setMuted(value => {
@@ -231,22 +296,77 @@ export function useVoiceConversation({
    })
  }, [handle])

-  // Drive the loop: speak any new assistant response, otherwise start listening
-  // when the agent is idle and we're between turns.
+  useEffect(() => {
+    if (!enabled) {
+      return
+    }
+
+    const onKeyDown = (event: KeyboardEvent) => {
+      if (event.code !== 'Space' || event.repeat || event.metaKey || event.ctrlKey || event.altKey) {
+        return
+      }
+
+      if (statusRef.current !== 'listening') {
+        return
+      }
+
+      event.preventDefault()
+      stopTurn()
+    }
+
+    window.addEventListener('keydown', onKeyDown, { capture: true })
+
+    return () => window.removeEventListener('keydown', onKeyDown, { capture: true })
+  }, [enabled, stopTurn])
+
+  // Drive the loop: after a voice-submitted turn, speak stable chunks as the
+  // assistant stream grows. Otherwise start listening when idle between turns.
  useEffect(() => {
    if (!enabled || muted) {
      return
    }

-    const text = pendingResponseText()
-    const trimmed = text?.trim() ?? ''
+    if (awaitingSpokenResponseRef.current && status !== 'speaking') {
+      const response = pendingResponse()

-    if (trimmed && trimmed !== lastSpokenRef.current && status !== 'speaking') {
-      lastSpokenRef.current = trimmed
-      consumePendingResponse()
-      void speak(trimmed)
+      if (response) {
+        if (response.id !== responseIdRef.current) {
+          resetSpeechBuffer()
+          responseIdRef.current = response.id
+        }

-      return
+        if (response.text.length > spokenSourceLengthRef.current) {
+          appendSpeechText(response.text.slice(spokenSourceLengthRef.current))
+          spokenSourceLengthRef.current = response.text.length
+        }
+
+        const chunk = takeSpeechChunk(!response.pending && !busy)
+
+        if (chunk) {
+          void speak(chunk)
+
+          return
+        }
+
+        if (!response.pending && !busy) {
+          awaitingSpokenResponseRef.current = false
+          consumePendingResponse()
+          resetSpeechBuffer()
+          pendingStartRef.current = true
+          setStatus('idle')
+
+          return
+        }
+      }
+
+      if (!busy && status === 'thinking') {
+        awaitingSpokenResponseRef.current = false
+        resetSpeechBuffer()
+        pendingStartRef.current = true
+        setStatus('idle')
+
+        return
+      }
    }

    if (busy || status !== 'idle') {
@@ -256,7 +376,7 @@ export function useVoiceConversation({
    if (pendingStartRef.current) {
      void startListening()
    }
-  }, [busy, consumePendingResponse, enabled, muted, pendingResponseText, speak, startListening, status])
+  }, [busy, consumePendingResponse, enabled, muted, pendingResponse, speak, startListening, status])

  useEffect(() => {
    if (enabled && !wasEnabledRef.current) {
@@ -270,5 +390,5 @@ export function useVoiceConversation({
    wasEnabledRef.current = enabled
  }, [enabled, end, start])

-  return { end, level, muted, start, status, toggleMute }
+  return { end, level, muted, start, status, stopTurn, toggleMute }
 }
@@ -32,7 +32,7 @@ import { useVoiceConversation } from './hooks/use-voice-conversation'
 import { useVoiceRecorder } from './hooks/use-voice-recorder'
 import type { ChatBarProps } from './types'
 import { UrlDialog } from './url-dialog'
-import { VoiceActivity } from './voice-activity'
+import { VoiceActivity, VoicePlaybackActivity } from './voice-activity'

 function trimPastedEdgeNewlines(text: string): string {
  return text.replace(EDGE_NEWLINES_RE, '')
@@ -45,7 +45,6 @@ export function ChatBar({
  maxRecordingSeconds = DEFAULT_MAX_RECORDING_SECONDS,
  state,
  onCancel,
-  onAddContextRef,
  onAddUrl,
  onPasteClipboardImage,
  onPickFiles,
@@ -203,7 +202,7 @@ export function ChatBar({
      onCancel()
    } else if (draft.trim() || attachments.length > 0) {
      triggerHaptic('submit')
-      onSubmit(draft)
+      void onSubmit(draft)
      aui.composer().setText('')
    }

@@ -235,9 +234,9 @@ export function ChatBar({
    onTranscribeAudio
  })

-  const pendingResponseText = () => {
+  const pendingResponse = () => {
    const messages = $messages.get()
-    const last = messages.findLast(m => m.role === 'assistant' && !m.pending && !m.hidden)
+    const last = messages.findLast(m => m.role === 'assistant' && !m.hidden)

    if (!last || last.id === lastSpokenIdRef.current) {
      return null
@@ -249,9 +248,11 @@ export function ChatBar({
      return null
    }

-    lastSpokenIdRef.current = last.id
-
-    return text
+    return {
+      id: last.id,
+      pending: Boolean(last.pending),
+      text
+    }
  }

  const consumePendingResponse = () => {
@@ -263,13 +264,13 @@ export function ChatBar({
    }
  }

-  const submitVoiceTurn = (text: string) => {
+  const submitVoiceTurn = async (text: string) => {
    if (busy) {
      return
    }

    triggerHaptic('submit')
-    onSubmit(text)
+    await onSubmit(text)
    aui.composer().setText('')
    draftRef.current = ''
  }
@@ -281,12 +282,11 @@ export function ChatBar({
    onFatalError: () => setVoiceConversationActive(false),
    onSubmit: submitVoiceTurn,
    onTranscribeAudio,
-    pendingResponseText
+    pendingResponse
  })

  const contextMenu = (
    <ContextMenu
-      onAddContextRef={onAddContextRef}
      onInsertText={insertText}
      onOpenUrlDialog={() => {
        triggerHaptic('open')
@@ -313,6 +313,7 @@ export function ChatBar({
          void conversation.end()
        },
        onStart: () => setVoiceConversationActive(true),
+        onStopTurn: conversation.stopTurn,
        onToggleMute: conversation.toggleMute,
        status: conversation.status
      }}
@@ -343,14 +344,12 @@ export function ChatBar({
  return (
    <>
      <ComposerPrimitive.Unstable_TriggerPopoverRoot>
-        {mentionCategories.length > 0 && (
-          <DirectivePopover
-            adapter={mention.adapter}
-            directive={mention.directive}
-            fallbackIcon={mention.fallbackIcon ?? FileText}
-            iconMap={mention.iconMap ?? DIRECTIVE_ICONS}
-          />
-        )}
+        <DirectivePopover
+          adapter={mention.adapter}
+          directive={mention.directive}
+          fallbackIcon={mention.fallbackIcon ?? FileText}
+          iconMap={mention.iconMap ?? DIRECTIVE_ICONS}
+        />
        <ComposerPrimitive.Root
          className={cn(SHELL, 'group/composer pb-8 pt-2')}
          onSubmit={e => {
@@ -407,6 +406,7 @@ export function ChatBar({
              style={{ ...COMPOSER_BACKDROP_STYLE, borderRadius: `${glassTweaks.liquid.cornerRadius}px` }}
            >
              <VoiceActivity state={voiceActivityState} />
+              <VoicePlaybackActivity />
              {attachments.length > 0 && <AttachmentList attachments={attachments} onRemove={onRemoveAttachment} />}
              {stacked ? (
                <>
@@ -36,7 +36,7 @@ export interface ChatBarProps {
  onPickFolders?: () => void
  onPickImages?: () => void
  onRemoveAttachment?: (id: string) => void
-  onSubmit: (value: string) => void
+  onSubmit: (value: string) => Promise<void> | void
  onTranscribeAudio?: (audio: Blob) => Promise<string>
 }

@@ -1,9 +1,12 @@
+import { Globe } from 'lucide-react'
 import type * as React from 'react'

 import { Button } from '@/components/ui/button'
 import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog'
 import { Input } from '@/components/ui/input'

+const URL_HINT = /^https?:\/\//i
+
 export function UrlDialog({
  inputRef,
  onChange,
@@ -19,14 +22,23 @@ export function UrlDialog({
  open: boolean
  value: string
 }) {
+  const trimmed = value.trim()
+  const looksLikeUrl = trimmed.length > 0 && URL_HINT.test(trimmed)
+
  return (
    <Dialog onOpenChange={onOpenChange} open={open}>
-      <DialogContent className="max-w-md">
-        <DialogHeader>
-          <DialogTitle>Add URL Context</DialogTitle>
-          <DialogDescription>
-            Hermes will fetch this URL via the existing @url context resolver when you send the prompt.
-          </DialogDescription>
+      <DialogContent className="max-w-md gap-5">
+        <DialogHeader className="flex-row items-center gap-3 sm:items-center">
+          <span
+            aria-hidden
+            className="grid size-9 shrink-0 place-items-center rounded-xl bg-[color-mix(in_srgb,var(--dt-primary)_14%,transparent)] text-primary ring-1 ring-inset ring-primary/15"
+          >
+            <Globe className="size-4" />
+          </span>
+          <div className="grid gap-0.5 text-left">
+            <DialogTitle>Attach a URL</DialogTitle>
+            <DialogDescription>Hermes will fetch the page and include it as context for this turn.</DialogDescription>
+          </div>
        </DialogHeader>
        <form
          className="grid gap-4"
@@ -35,18 +47,29 @@ export function UrlDialog({
            onSubmit()
          }}
        >
-          <Input
-            onChange={e => onChange(e.target.value)}
-            placeholder="https://example.com"
-            ref={inputRef}
-            value={value}
-          />
+          <div className="grid gap-1.5">
+            <Input
+              autoComplete="off"
+              autoCorrect="off"
+              inputMode="url"
+              onChange={e => onChange(e.target.value)}
+              placeholder="https://example.com/post"
+              ref={inputRef}
+              spellCheck={false}
+              value={value}
+            />
+            {trimmed.length > 0 && !looksLikeUrl && (
+              <p className="text-xs text-muted-foreground/85">
+                Include the full URL, e.g. <span className="font-mono">https://…</span>
+              </p>
+            )}
+          </div>
          <DialogFooter>
            <Button onClick={() => onOpenChange(false)} type="button" variant="ghost">
              Cancel
            </Button>
-            <Button disabled={!value.trim()} type="submit">
-              Add URL
+            <Button disabled={!looksLikeUrl} type="submit">
+              Attach
            </Button>
          </DialogFooter>
        </form>
@@ -1,6 +1,10 @@
-import { Loader2, Mic } from 'lucide-react'
+import { useStore } from '@nanostores/react'
+import { Loader2, Mic, Volume2, VolumeX } from 'lucide-react'

+import { Button } from '@/components/ui/button'
 import { cn } from '@/lib/utils'
+import { stopVoicePlayback } from '@/lib/voice-playback'
+import { $voicePlayback } from '@/store/voice-playback'

 import type { VoiceActivityState } from './types'

@@ -36,6 +40,25 @@ function VoiceLevelBars({ level, active }: { active: boolean; level: number }) {
  )
 }

+function PlaybackBars() {
+  const bars = [820, 940, 760, 880, 700, 980, 790]
+
+  return (
+    <div aria-hidden="true" className="flex h-4 items-center gap-0.75">
+      {bars.map((duration, index) => (
+        <span
+          className="voice-wave-bar h-full w-0.5 rounded-full bg-current"
+          key={index}
+          style={{
+            animationDelay: `${index * -110}ms`,
+            animationDuration: `${duration}ms`
+          }}
+        />
+      ))}
+    </div>
+  )
+}
+
 export function VoiceActivity({
  state
 }: {
@@ -75,3 +98,50 @@ export function VoiceActivity({
    </div>
  )
 }
+
+export function VoicePlaybackActivity() {
+  const playback = useStore($voicePlayback)
+
+  if (playback.status === 'idle') {
+    return null
+  }
+
+  const preparing = playback.status === 'preparing'
+
+  const title = preparing
+    ? 'Preparing audio'
+    : playback.source === 'voice-conversation'
+      ? 'Speaking response'
+      : 'Reading aloud'
+
+  return (
+    <div
+      aria-live="polite"
+      className={cn(
+        'flex h-8 items-center gap-2 rounded-xl border border-primary/20 bg-primary/10 px-2.5 text-xs text-primary',
+        'shadow-[inset_0_1px_0_rgba(255,255,255,0.35)] backdrop-blur-sm'
+      )}
+      role="status"
+    >
+      <div className="flex size-5 shrink-0 items-center justify-center rounded-full bg-primary/15 text-primary">
+        {preparing ? <Loader2 className="animate-spin" size={12} /> : <Volume2 size={12} />}
+      </div>
+
+      <div className="flex min-w-0 flex-1 items-center gap-2">
+        <span className="truncate font-medium text-foreground/85">{title}</span>
+        {!preparing && <PlaybackBars />}
+      </div>
+
+      <Button
+        className="h-6 shrink-0 gap-1 rounded-full px-2 text-[0.6875rem]"
+        onClick={stopVoicePlayback}
+        size="sm"
+        type="button"
+        variant="ghost"
+      >
+        <VolumeX size={12} />
+        Stop
+      </Button>
+    </div>
+  )
+}
@@ -1,5 +1,6 @@
 import { useCallback } from 'react'

+import { formatRefValue } from '@/components/assistant-ui/directive-text'
 import { attachmentId, contextPath, pathLabel } from '@/lib/chat-runtime'
 import {
  addComposerAttachment,
@@ -57,7 +58,7 @@ export function useComposerActions({ activeSessionId, currentCwd, requestGateway
          kind,
          label: pathLabel(path),
          detail: rel,
-          refText: `@${kind}:${rel}`,
+          refText: `@${kind}:${formatRefValue(rel)}`,
          path
        })
      }
@@ -8,13 +8,14 @@ import { useStore } from '@nanostores/react'
 import { useQuery } from '@tanstack/react-query'
 import { ChevronDown } from 'lucide-react'
 import type * as React from 'react'
-import { Suspense, useMemo } from 'react'
+import { Suspense, useMemo, useRef } from 'react'
 import { useLocation } from 'react-router-dom'

 import { Thread } from '@/components/assistant-ui/thread'
 import { NotificationStack } from '@/components/notifications'
 import { Button } from '@/components/ui/button'
 import { getGlobalModelOptions, type HermesGateway } from '@/hermes'
+import type { ChatMessage } from '@/lib/chat-messages'
 import { quickModelOptions, sessionTitle, toRuntimeMessage } from '@/lib/chat-runtime'
 import { cn } from '@/lib/utils'
 import { $pinnedSessionIds } from '@/store/layout'
@@ -57,7 +58,7 @@ interface ChatViewProps extends Omit<React.ComponentProps<'div'>, 'onSubmit'> {
  onPickFolders: () => void
  onPickImages: () => void
  onRemoveAttachment: (id: string) => void
-  onSubmit: (text: string) => void
+  onSubmit: (text: string) => Promise<void> | void
  onChangeCwd: (cwd: string) => void
  onBrowseCwd: () => void
  onOpenModelPicker: () => void
@@ -118,6 +119,7 @@ export function ChatView({
  const pinnedSessionIds = useStore($pinnedSessionIds)
  const selectedSessionId = useStore($selectedStoredSessionId)
  const sessions = useStore($sessions)
+  const runtimeMessageCacheRef = useRef(new WeakMap<ChatMessage, ThreadMessage>())
  const activeStoredSession = sessions.find(session => session.id === selectedSessionId) || null
  const isRoutedSessionView = Boolean(routeSessionId(location.pathname))
  const selectedIsPinned = selectedSessionId ? pinnedSessionIds.includes(selectedSessionId) : false
@@ -128,6 +130,7 @@ export function ChatView({
  const loadingSession = isRoutedSessionView && messages.length === 0
  const threadLoading = threadLoadingState(loadingSession, busy, awaitingResponse)
  const showChatBar = !loadingSession
+  const threadKey = selectedSessionId || activeSessionId || (isRoutedSessionView ? location.pathname : 'new')
  const title = activeStoredSession ? sessionTitle(activeStoredSession) : ''

  const modelOptionsQuery = useQuery<ModelOptionsResponse>({
@@ -190,7 +193,14 @@ export function ChatView({
        parentId = branchParentByGroup.get(message.branchGroupId) ?? null
      }

-      items.push({ message: toRuntimeMessage(message), parentId })
+      const cachedMessage = runtimeMessageCacheRef.current.get(message)
+      const runtimeMessage = cachedMessage ?? toRuntimeMessage(message)
+
+      if (!cachedMessage) {
+        runtimeMessageCacheRef.current.set(message, runtimeMessage)
+      }
+
+      items.push({ message: runtimeMessage, parentId })

      if (!message.hidden) {
        visibleParentId = message.id
@@ -248,6 +258,7 @@ export function ChatView({
              intro={showIntro ? { personality: introPersonality, seed: introSeed } : undefined}
              loading={threadLoading}
              onBranchInNewChat={onBranchInNewChat}
+              sessionKey={threadKey}
            />
            {showChatBar && (
              <Suspense fallback={<ChatBarFallback />}>
@@ -14,6 +14,7 @@ import {
  listSessions,
  setGlobalModel
 } from '../hermes'
+import { formatRefValue } from '../components/assistant-ui/directive-text'
 import { toChatMessages } from '../lib/chat-messages'
 import { BUILTIN_PERSONALITIES, normalizePersonalityValue, personalityNamesFromConfig } from '../lib/chat-runtime'
 import { $pinnedSessionIds, pinSession, unpinSession } from '../store/layout'
@@ -571,7 +572,7 @@ export function DesktopController() {
      gateway={gatewayRef.current}
      maxVoiceRecordingSeconds={voiceMaxRecordingSeconds}
      onAddContextRef={addContextRefAttachment}
-      onAddUrl={url => addContextRefAttachment(`@url:${url}`, url)}
+      onAddUrl={url => addContextRefAttachment(`@url:${formatRefValue(url)}`, url)}
      onBranchInNewChat={messageId => void branchInNewChat(messageId)}
      onBrowseCwd={() => void browseSessionCwd()}
      onCancel={() => void cancelRun()}
@@ -589,7 +590,7 @@ export function DesktopController() {
      onReload={reloadFromMessage}
      onRemoveAttachment={id => void removeAttachment(id)}
      onSelectPersonality={name => void selectPersonality(name)}
-      onSubmit={text => void submitText(text)}
+      onSubmit={submitText}
      onThreadMessagesChange={handleThreadMessagesChange}
      onToggleSelectedPin={toggleSelectedPin}
      onTranscribeAudio={transcribeVoiceAudio}
@@ -1,6 +1,5 @@
 import type { QueryClient } from '@tanstack/react-query'
 import { type MutableRefObject, useCallback } from 'react'
-import { flushSync } from 'react-dom'

 import {
  appendReasoningPart,
@@ -60,7 +59,6 @@ export function useMessageStream({
      transform: (parts: ChatMessagePart[], message: ChatMessage) => ChatMessagePart[],
      seed: () => ChatMessagePart[],
      opts: {
-        sync?: boolean
        pending?: (message: ChatMessage) => boolean
      } = {}
    ) => {
@@ -112,7 +110,7 @@ export function useMessageStream({
        })
      }

-      opts.sync ? flushSync(apply) : apply()
+      apply()
    },
    [updateSessionState]
  )
@@ -126,8 +124,7 @@ export function useMessageStream({
      mutateStream(
        sessionId,
        parts => appendTextPart(parts, delta),
-        () => [textPart(delta)],
-        { sync: true }
+        () => [textPart(delta)]
      )
    },
    [mutateStream]
@@ -152,8 +149,7 @@ export function useMessageStream({

          return appendReasoningPart(parts, delta)
        },
-        () => [reasoningPart(delta)],
-        { sync: true }
+        () => [reasoningPart(delta)]
      )
    },
    [mutateStream]
@@ -299,6 +295,7 @@ export function useMessageStream({
        const apply = explicitSid ? isActiveEvent : !activeSessionIdRef.current
        const modelChanged = typeof payload?.model === 'string'
        const providerChanged = typeof payload?.provider === 'string'
+        const runningChanged = typeof payload?.running === 'boolean'

        if (apply) {
          if (modelChanged) {
@@ -320,6 +317,35 @@ export function useMessageStream({
          if (typeof payload?.personality === 'string') {
            setCurrentPersonality(normalizePersonalityValue(payload.personality))
          }
+
+          if (runningChanged && sessionId) {
+            updateSessionState(sessionId, state => {
+              const busy = Boolean(payload!.running)
+
+              if (state.busy === busy && (busy || !state.awaitingResponse)) {
+                return state
+              }
+
+              if (busy) {
+                return {
+                  ...state,
+                  busy
+                }
+              }
+
+              if (state.awaitingResponse && !state.sawAssistantPayload) {
+                return state
+              }
+
+              return {
+                ...state,
+                awaitingResponse: false,
+                busy,
+                pendingBranchGroup: null,
+                streamId: null
+              }
+            })
+          }
        }

        void refreshHermesConfig()
@@ -355,11 +381,11 @@ export function useMessageStream({
        }
      } else if (event.type === 'reasoning.delta') {
        if (sessionId) {
-          appendReasoningDelta(sessionId, coerceGatewayText(payload?.text))
+          appendReasoningDelta(sessionId, coerceThinkingText(payload?.text))
        }
      } else if (event.type === 'reasoning.available') {
        if (sessionId) {
-          appendReasoningDelta(sessionId, coerceGatewayText(payload?.text), true)
+          appendReasoningDelta(sessionId, coerceThinkingText(payload?.text), true)
        }
      } else if (event.type === 'message.complete') {
        if (!sessionId) {
@@ -13,7 +13,7 @@ import {
 import { triggerHaptic } from '@/lib/haptics'
 import { $composerAttachments, clearComposerAttachments } from '@/store/composer'
 import { clearNotifications, notify, notifyError } from '@/store/notifications'
-import { $busy, $messages, setAwaitingResponse, setBusy } from '@/store/session'
+import { $busy, $messages, setAwaitingResponse, setBusy, setMessages } from '@/store/session'

 import type { ClientSessionState, SlashExecResponse } from '../../types'

@@ -296,12 +296,34 @@ export function usePromptActions({
  )

  const cancelRun = useCallback(async () => {
-    if (!activeSessionId) {
+    const sessionId = activeSessionId || activeSessionIdRef.current
+
+    busyRef.current = false
+    setBusy(false)
+    setAwaitingResponse(false)
+
+    const finalizeMessages = (messages: ChatMessage[]) =>
+      messages.map(message =>
+        message.pending
+          ? {
+              ...message,
+              parts: chatMessageText(message).trim()
+                ? appendTextPart(message.parts, INTERRUPTED_MARKER)
+                : [...message.parts, textPart(INTERRUPTED_MARKER.trim())],
+              pending: false
+            }
+          : message
+      )
+
+    if (!sessionId) {
+      setMessages(finalizeMessages($messages.get()))
+
      return
    }

-    updateSessionState(activeSessionId, state => {
+    updateSessionState(sessionId, state => {
      const streamId = state.streamId
+
      const messages = streamId
        ? state.messages.map(message =>
            message.id === streamId
@@ -314,7 +336,7 @@ export function usePromptActions({
                }
              : message
          )
-        : state.messages
+        : finalizeMessages(state.messages)

      return {
        ...state,
@@ -328,11 +350,11 @@ export function usePromptActions({
    })

    try {
-      await requestGateway('session.interrupt', { session_id: activeSessionId })
+      await requestGateway('session.interrupt', { session_id: sessionId })
    } catch (err) {
      notifyError(err, 'Stop failed')
    }
-  }, [activeSessionId, requestGateway, updateSessionState])
+  }, [activeSessionId, activeSessionIdRef, busyRef, requestGateway, updateSessionState])

  const reloadFromMessage = useCallback(
    async (parentId: string | null) => {
@@ -87,6 +87,11 @@ export function useSessionActions({

  const createBackendSessionForSend = useCallback(async (): Promise<string | null> => {
    const created = await requestGateway<SessionCreateResponse>('session.create', { cols: 96 })
+
+    if (created.stored_session_id) {
+      navigate(sessionRoute(created.stored_session_id), { replace: true })
+    }
+
    setActiveSessionId(created.session_id)
    activeSessionIdRef.current = created.session_id
    ensureSessionState(created.session_id, created.stored_session_id ?? null)
@@ -94,7 +99,6 @@ export function useSessionActions({
    if (created.stored_session_id) {
      setSelectedStoredSessionId(created.stored_session_id)
      selectedStoredSessionIdRef.current = created.stored_session_id
-      navigate(sessionRoute(created.stored_session_id), { replace: true })
    }

    if (created.info?.model) {
@@ -60,6 +60,7 @@ export const ENUM_OPTIONS: Record<string, string[]> = {
  'context.engine': ['compressor', 'default', 'custom'],
  'delegation.reasoning_effort': ['', 'minimal', 'low', 'medium', 'high', 'xhigh'],
  'memory.provider': ['', 'builtin', 'honcho'],
+  'stt.elevenlabs.model_id': ['scribe_v2', 'scribe_v1'],
  'stt.local.model': ['tiny', 'base', 'small', 'medium', 'large-v3'],
  'tts.openai.voice': ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer']
 }
@@ -101,6 +102,10 @@ export const FIELD_LABELS: Record<string, string> = {
  'stt.provider': 'Speech-To-Text Provider',
  'stt.local.model': 'Local Transcription Model',
  'stt.local.language': 'Transcription Language',
+  'stt.elevenlabs.model_id': 'ElevenLabs STT Model',
+  'stt.elevenlabs.language_code': 'ElevenLabs Language',
+  'stt.elevenlabs.tag_audio_events': 'Tag Audio Events',
+  'stt.elevenlabs.diarize': 'Speaker Diarization',
  'tts.provider': 'Text-To-Speech Provider',
  'tts.edge.voice': 'Edge Voice',
  'tts.openai.model': 'OpenAI TTS Model',
@@ -157,6 +162,7 @@ export const FIELD_DESCRIPTIONS: Record<string, string> = {
  'compression.enabled': 'Summarize older context when conversations get large.',
  'voice.auto_tts': 'Automatically speak assistant responses.',
  'stt.enabled': 'Enable local or provider-backed speech transcription.',
+  'stt.elevenlabs.language_code': 'Optional ISO-639-3 language code. Blank lets ElevenLabs auto-detect.',
  'agent.max_turns': 'Upper bound for tool-calling turns before Hermes stops a run.'
 }

@@ -241,6 +247,10 @@ export const SECTIONS: DesktopConfigSection[] = [
      'tts.elevenlabs.model_id',
      'stt.local.model',
      'stt.local.language',
+      'stt.elevenlabs.model_id',
+      'stt.elevenlabs.language_code',
+      'stt.elevenlabs.tag_audio_events',
+      'stt.elevenlabs.diarize',
      'voice.record_key',
      'voice.max_recording_seconds'
    ]