feat: lots of speech stuff

2026-05-01 19:28:02 -05:00
parent 9f3d393a4d
commit d5d7b5c6dc
41 changed files with 1405 additions and 361 deletions
@@ -0,0 +1,39 @@
+import { describe, expect, it } from 'vitest'
+
+import { formatRefValue, hermesDirectiveFormatter } from './directive-text'
+
+describe('formatRefValue', () => {
+  it('leaves simple paths untouched', () => {
+    expect(formatRefValue('src/index.ts')).toBe('src/index.ts')
+    expect(formatRefValue('https://example.com/post')).toBe('https://example.com/post')
+  })
+
+  it('wraps paths with whitespace in backticks', () => {
+    expect(formatRefValue('apple-touch-icon (1).png')).toBe('`apple-touch-icon (1).png`')
+  })
+
+  it('falls back to double quotes when value contains backticks', () => {
+    expect(formatRefValue('weird `name` (1).md')).toBe('"weird `name` (1).md"')
+  })
+})
+
+describe('hermesDirectiveFormatter.parse', () => {
+  it('keeps quoted file paths whole when parsing', () => {
+    const segments = hermesDirectiveFormatter.parse('see @image:`apple-touch-icon (1).png` for the icon')
+
+    expect(segments).toEqual([
+      { kind: 'text', text: 'see ' },
+      { kind: 'mention', type: 'image', label: 'apple-touch-icon (1).png', id: 'apple-touch-icon (1).png' },
+      { kind: 'text', text: ' for the icon' }
+    ])
+  })
+
+  it('still parses unquoted paths', () => {
+    const segments = hermesDirectiveFormatter.parse('@file:src/main.tsx the entry point')
+
+    expect(segments).toEqual([
+      { kind: 'mention', type: 'file', label: 'main.tsx', id: 'src/main.tsx' },
+      { kind: 'text', text: ' the entry point' }
+    ])
+  })
+})
@@ -24,10 +24,63 @@ const ICONS: Record<HermesRefType, ComponentType<{ className?: string }>> = {
 * so they render as inline chips in user messages instead of raw text.
 *
 * Supported types: file, folder, url, image. Anything else stays plain text.
+ *
+ * Mirrors the Python `agent/context_references.REFERENCE_PATTERN` syntax:
+ * the value may be wrapped in backticks, single quotes, or double quotes so
+ * paths with spaces/parens/etc. survive parsing intact.
 */
-const CANONICAL_DIRECTIVE_RE = /:([\w-]{1,64})\[([^\]\n]{1,1024})\](?:\{name=([^}\n]{1,1024})\})?/gu
+const CANONICAL_DIRECTIVE_RE = /:([\w-]{1,64})\[([^\]\n]{1,1024})\](?:\{name=([^}\n]{1,1024})\})?/g

-const HERMES_DIRECTIVE_RE = /@(file|folder|url|image|tool):(\S+)/gu
+const HERMES_DIRECTIVE_RE = new RegExp(
+  '@(file|folder|url|image|tool):(' +
+    '`[^`\\n]+`' +
+    '|"[^"\\n]+"' +
+    "|'[^'\\n]+'" +
+    '|\\S+' +
+    ')',
+  'g'
+)
+
+const TRAILING_PUNCTUATION_RE = /[,.;!?]+$/
+
+function unwrapRefValue(raw: string): string {
+  if (raw.length < 2) {
+    return raw
+  }
+
+  const head = raw[0]
+  const tail = raw[raw.length - 1]
+
+  if ((head === '`' && tail === '`') || (head === '"' && tail === '"') || (head === "'" && tail === "'")) {
+    return raw.slice(1, -1)
+  }
+
+  return raw.replace(TRAILING_PUNCTUATION_RE, '')
+}
+
+function needsQuoting(value: string): boolean {
+  return /[\s()\[\]{}<>"'`]/.test(value)
+}
+
+export function formatRefValue(value: string): string {
+  if (!needsQuoting(value)) {
+    return value
+  }
+
+  if (!value.includes('`')) {
+    return `\`${value}\``
+  }
+
+  if (!value.includes('"')) {
+    return `"${value}"`
+  }
+
+  if (!value.includes("'")) {
+    return `'${value}'`
+  }
+
+  return value
+}

 export const hermesDirectiveFormatter: Unstable_DirectiveFormatter = {
  serialize(item: Unstable_TriggerItem): string {
@@ -35,7 +88,7 @@ export const hermesDirectiveFormatter: Unstable_DirectiveFormatter = {
      return `@${item.id}`
    }

-    return `@${item.type}:${item.id}`
+    return `@${item.type}:${formatRefValue(item.id)}`
  },
  parse(text: string): readonly Unstable_DirectiveSegment[] {
    return parseDirectiveText(text)
@@ -51,13 +104,17 @@ function parseDirectiveText(text: string): Unstable_DirectiveSegment[] {
      label: match[2] || match[3] || '',
      id: match[3] || match[2] || ''
    })),
-    ...Array.from(text.matchAll(HERMES_DIRECTIVE_RE)).map(match => ({
-      start: match.index ?? 0,
-      end: (match.index ?? 0) + match[0].length,
-      type: match[1] || 'file',
-      label: shortLabel(match[1] as HermesRefType, match[2] || ''),
-      id: match[2] || ''
-    }))
+    ...Array.from(text.matchAll(HERMES_DIRECTIVE_RE)).map(match => {
+      const id = unwrapRefValue(match[2] || '')
+
+      return {
+        start: match.index ?? 0,
+        end: (match.index ?? 0) + match[0].length,
+        type: match[1] || 'file',
+        label: shortLabel(match[1] as HermesRefType, id),
+        id
+      }
+    })
  ]
    .filter(match => match.id)
    .sort((a, b) => a.start - b.start)
@@ -136,14 +193,14 @@ const DirectiveChip: FC<{
  return (
    <span
      className={cn(
-        'mx-0.5 inline-flex max-w-56 items-center gap-1 rounded-full border border-border/80 bg-background/95 px-1.5 py-0.5 align-[0.05em] text-[0.82em] font-medium leading-none text-foreground shadow-sm ring-1 ring-black/3'
+        'mx-0.5 inline-flex max-w-64 items-center gap-1 rounded-full bg-[color-mix(in_srgb,var(--dt-primary)_16%,transparent)] px-2 py-0.5 align-[0.02em] text-[0.92em] font-semibold leading-tight text-primary ring-1 ring-inset ring-primary/10'
      )}
      data-directive-id={id}
      data-directive-type={type}
      data-slot="aui_directive-chip"
      title={id}
    >
-      {Icon && <Icon className="size-3 shrink-0 text-muted-foreground" />}
+      {Icon && <Icon className="size-3.5 shrink-0 text-primary" />}
      <span className="truncate">{label}</span>
    </span>
  )
@@ -19,6 +19,7 @@ export type IntroProps = {
 const NEUTRAL_PERSONALITIES = new Set(['', 'default', 'none', 'neutral'])

 const HERMES_FRAME_COUNT = 8
+const ASSET_BASE_URL = import.meta.env.BASE_URL || '/'

 const FALLBACK_COPY: IntroCopy[] = [
  {
@@ -154,6 +155,10 @@ function resolveCopy(personality?: string, seed?: number): IntroCopy {
  return pickCopy(copies, seed)
 }

+function publicAssetPath(path: string): string {
+  return `${ASSET_BASE_URL}${path}`.replace(/([^:]\/)\/+/g, '$1')
+}
+
 export const Intro: FC<IntroProps> = ({ personality, seed }) => {
  const [mountSeed] = useState(() => Math.floor(Math.random() * 100000))
  const [frameOffset, setFrameOffset] = useState(0)
@@ -184,7 +189,7 @@ export const Intro: FC<IntroProps> = ({ personality, seed }) => {
          aria-hidden="true"
          className="h-full w-full scale-110 object-contain select-none"
          draggable={false}
-          src={`/hermes-frames/hermes-frame-${frameIndex}.png?v=matte-clean-6`}
+          src={publicAssetPath(`hermes-frames/hermes-frame-${frameIndex}.png?v=matte-clean-6`)}
        />
      </button>
      <p className="mb-3 text-xs font-medium uppercase tracking-[0.18em] text-muted-foreground/75">Hermes Agent</p>
@@ -1,19 +1,53 @@
 import { AssistantRuntimeProvider, type ThreadMessage, useExternalStoreRuntime } from '@assistant-ui/react'
-import { act, render, screen, waitFor } from '@testing-library/react'
+import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'
 import { useEffect, useState } from 'react'
-import { describe, expect, it, vi } from 'vitest'
+import { beforeEach, describe, expect, it, vi } from 'vitest'

 import { Thread } from './thread'

 const createdAt = new Date('2026-05-01T00:00:00.000Z')

+const resizeObservers = new Set<TestResizeObserver>()
+
 class TestResizeObserver {
-  observe() {}
+  private target: Element | null = null
+
+  constructor(private readonly callback: ResizeObserverCallback) {
+    resizeObservers.add(this)
+  }
+
+  observe(target: Element) {
+    this.target = target
+  }
+
  unobserve() {}
-  disconnect() {}
+
+  disconnect() {
+    resizeObservers.delete(this)
+  }
+
+  trigger(height: number) {
+    if (!this.target) {
+      return
+    }
+
+    this.callback(
+      [
+        {
+          contentRect: { height } as DOMRectReadOnly,
+          target: this.target
+        } as ResizeObserverEntry
+      ],
+      this as unknown as ResizeObserver
+    )
+  }
 }

 vi.stubGlobal('ResizeObserver', TestResizeObserver)
+vi.stubGlobal('requestAnimationFrame', (callback: FrameRequestCallback) =>
+  window.setTimeout(() => callback(performance.now()), 0)
+)
+vi.stubGlobal('cancelAnimationFrame', (id: number) => window.clearTimeout(id))

 Element.prototype.scrollTo = function scrollTo() {}

@@ -90,6 +124,10 @@ function StreamingHarness() {
 }

 describe('assistant-ui streaming renderer', () => {
+  beforeEach(() => {
+    resizeObservers.clear()
+  })
+
  it('renders assistant text incrementally before completion', async () => {
    const { container } = render(<StreamingHarness />)

@@ -115,4 +153,42 @@ describe('assistant-ui streaming renderer', () => {
      expect(container.textContent).toContain('first chunk second chunk')
    })
  })
+
+  it('does not pull the viewport back down after the user scrolls up during streaming', async () => {
+    const { container } = render(<StreamingHarness />)
+
+    const viewport = container.querySelector('[data-slot="aui_thread-viewport"]') as HTMLDivElement
+    let scrollHeight = 1_000
+
+    Object.defineProperty(viewport, 'clientHeight', { configurable: true, value: 200 })
+    Object.defineProperty(viewport, 'scrollHeight', {
+      configurable: true,
+      get: () => scrollHeight
+    })
+
+    await wait(80)
+
+    await act(async () => {
+      viewport.scrollTop = 800
+      fireEvent.scroll(viewport)
+    })
+    await wait(0)
+
+    await act(async () => {
+      fireEvent.wheel(viewport, { deltaY: -120 })
+      viewport.scrollTop = 420
+      fireEvent.scroll(viewport)
+    })
+
+    scrollHeight = 1_200
+
+    await act(async () => {
+      for (const observer of resizeObservers) {
+        observer.trigger(1_200)
+      }
+    })
+    await wait(0)
+
+    expect(viewport.scrollTop).toBe(420)
+  })
 })
@@ -8,18 +8,28 @@ import {
  type ToolCallMessagePartProps,
  useAuiState
 } from '@assistant-ui/react'
+import { useStore } from '@nanostores/react'
 import {
  CheckIcon,
  ChevronLeftIcon,
  ChevronRightIcon,
  CopyIcon,
  GitBranchIcon,
+  Loader2Icon,
  MoreHorizontalIcon,
  RefreshCwIcon,
  Volume2Icon,
  VolumeXIcon
 } from 'lucide-react'
-import { type FC, type ReactNode, useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
+import {
+  type FC,
+  type ReactNode,
+  useCallback,
+  useEffect,
+  useLayoutEffect,
+  useRef,
+  useState
+} from 'react'

 import { useElapsedSeconds } from '@/components/assistant-ui/activity-timer'
 import { ActivityTimerText } from '@/components/assistant-ui/activity-timer-text'
@@ -38,11 +48,12 @@ import {
  DropdownMenuTrigger
 } from '@/components/ui/dropdown-menu'
 import { Loader } from '@/components/ui/loader'
-import { speakText } from '@/hermes'
 import { triggerHaptic } from '@/lib/haptics'
 import { cn } from '@/lib/utils'
+import { playSpeechText, stopVoicePlayback } from '@/lib/voice-playback'
 import { notifyError } from '@/store/notifications'
 import { setThreadScrolledUp } from '@/store/thread-scroll'
+import { $voicePlayback } from '@/store/voice-playback'

 const THINKING_FACES = [
  '(｡•́︿•̀｡)',
@@ -119,12 +130,16 @@ export const Thread: FC<{
  intro?: IntroProps
  loading?: ThreadLoadingState
  onBranchInNewChat?: (messageId: string) => void
-}> = ({ intro, loading, onBranchInNewChat }) => {
+  sessionKey?: string | null
+}> = ({ intro, loading, onBranchInNewChat, sessionKey }) => {
  const viewportRef = useRef<HTMLDivElement | null>(null)
+  const contentRef = useRef<HTMLDivElement | null>(null)
  const messageCount = useAuiState(s => s.thread.messages.length)
  const isRunning = useAuiState(s => s.thread.isRunning)
  const lastMessageId = useAuiState(s => s.thread.messages.at(-1)?.id ?? '')
  const shouldStickToBottomRef = useRef(true)
+  const scrollFrameRef = useRef<number | null>(null)
+  const sessionKeyRef = useRef<string | null>(sessionKey ?? null)

  const handleScroll = useCallback((event: React.UIEvent<HTMLDivElement>) => {
    const nearBottom = isNearBottom(event.currentTarget)
@@ -132,8 +147,44 @@ export const Thread: FC<{
    setThreadScrolledUp(!nearBottom)
  }, [])

+  const handleWheel = useCallback((event: React.WheelEvent<HTMLDivElement>) => {
+    if (event.deltaY < 0) {
+      shouldStickToBottomRef.current = false
+      setThreadScrolledUp(true)
+    }
+  }, [])
+
+  const scrollToBottom = useCallback(() => {
+    const viewport = viewportRef.current
+
+    if (!viewport) {
+      return
+    }
+
+    viewport.scrollTop = viewport.scrollHeight
+    shouldStickToBottomRef.current = true
+    setThreadScrolledUp(false)
+  }, [])
+
+  const scheduleScrollToBottom = useCallback(() => {
+    if (scrollFrameRef.current !== null) {
+      window.cancelAnimationFrame(scrollFrameRef.current)
+    }
+
+    scrollFrameRef.current = window.requestAnimationFrame(() => {
+      scrollFrameRef.current = null
+      scrollToBottom()
+    })
+  }, [scrollToBottom])
+
  useEffect(() => {
-    return () => setThreadScrolledUp(false)
+    return () => {
+      if (scrollFrameRef.current !== null) {
+        window.cancelAnimationFrame(scrollFrameRef.current)
+      }
+
+      setThreadScrolledUp(false)
+    }
  }, [])

  useLayoutEffect(() => {
@@ -143,16 +194,48 @@ export const Thread: FC<{
      return
    }

-    const force = loading === 'session'
+    const nextSessionKey = sessionKey ?? null
+    const sessionChanged = sessionKeyRef.current !== nextSessionKey
+    sessionKeyRef.current = nextSessionKey
+    const force = loading === 'session' || sessionChanged

    if (!force && !shouldStickToBottomRef.current) {
      return
    }

-    viewport.scrollTop = viewport.scrollHeight
-    shouldStickToBottomRef.current = true
-    setThreadScrolledUp(false)
-  }, [isRunning, lastMessageId, loading, messageCount])
+    scheduleScrollToBottom()
+  }, [isRunning, lastMessageId, loading, messageCount, scheduleScrollToBottom, sessionKey])
+
+  useLayoutEffect(() => {
+    const content = contentRef.current
+    const viewport = viewportRef.current
+
+    if (!content || !viewport) {
+      return
+    }
+
+    let previousHeight = content.getBoundingClientRect().height
+
+    const observer = new ResizeObserver(entries => {
+      const height = entries[0]?.contentRect.height ?? content.getBoundingClientRect().height
+
+      if (height === previousHeight) {
+        return
+      }
+
+      previousHeight = height
+
+      if (!shouldStickToBottomRef.current && !isNearBottom(viewport)) {
+        return
+      }
+
+      scheduleScrollToBottom()
+    })
+
+    observer.observe(content)
+
+    return () => observer.disconnect()
+  }, [scheduleScrollToBottom])

  return (
    <GeneratedImageProvider>
@@ -160,15 +243,17 @@ export const Thread: FC<{
        <AuiIf condition={s => Boolean(intro) && s.thread.isEmpty}>{intro && <Intro {...intro} />}</AuiIf>

        <ThreadPrimitive.Viewport
-          className="h-full min-h-0 overflow-y-auto overscroll-contain px-[clamp(1rem,10%,12rem)] pt-[calc(var(--vsq)*19)] scroll-smooth"
+          autoScroll={false}
+          className="h-full min-h-0 overflow-y-auto overscroll-contain px-[clamp(1rem,10%,12rem)] pt-[calc(var(--vsq)*19)]"
          data-slot="aui_thread-viewport"
          onScroll={handleScroll}
+          onWheel={handleWheel}
          ref={viewportRef}
          scrollToBottomOnInitialize
          scrollToBottomOnRunStart
          scrollToBottomOnThreadSwitch
        >
-          <div className="flex w-full flex-col gap-3">
+          <div className="flex w-full flex-col gap-3" ref={contentRef}>
            <ThreadPrimitive.Messages>{() => <ThreadMessage onBranchInNewChat={onBranchInNewChat} />}</ThreadPrimitive.Messages>
            {loading === 'response' && <ResponseLoadingIndicator />}
            {loading === 'working' && <WorkingIndicator />}
@@ -446,7 +531,7 @@ const AssistantActionBar: FC<MessageActionProps> = ({ messageId, messageText, on
              <GitBranchIcon />
              Branch in new chat
            </DropdownMenuItem>
-            <ReadAloudItem text={messageText} />
+            <ReadAloudItem messageId={messageId} text={messageText} />
          </DropdownMenuContent>
        </DropdownMenu>
      </ActionBarPrimitive.Root>
@@ -479,80 +564,39 @@ const CopyMessageButton: FC<{ text: string }> = ({ text }) => {
  )
 }

-let currentAudio: HTMLAudioElement | null = null
+const ReadAloudItem: FC<{ messageId: string; text: string }> = ({ messageId, text }) => {
+  const voicePlayback = useStore($voicePlayback)

-function stopCurrentAudio() {
-  if (!currentAudio) {
-    return
-  }
+  const readAloudStatus =
+    voicePlayback.source === 'read-aloud' && voicePlayback.messageId === messageId ? voicePlayback.status : 'idle'

-  currentAudio.pause()
-  currentAudio.src = ''
-  currentAudio = null
-}
-
-const ReadAloudItem: FC<{ text: string }> = ({ text }) => {
-  const [reading, setReading] = useState(false)
-  const seqRef = useRef(0)
-
-  const stop = useCallback(() => {
-    seqRef.current += 1
-    stopCurrentAudio()
-    setReading(false)
-  }, [])
+  const isPreparing = readAloudStatus === 'preparing'
+  const isSpeaking = readAloudStatus === 'speaking'
+  const anyPlaybackActive = voicePlayback.status !== 'idle'
+  const Icon = isPreparing ? Loader2Icon : isSpeaking ? VolumeXIcon : Volume2Icon

  const read = useCallback(async () => {
-    if (!text) {
+    if (!text || $voicePlayback.get().status !== 'idle') {
      return
    }

-    stopCurrentAudio()
-    const seq = ++seqRef.current
-    const isCurrent = () => seq === seqRef.current
-
-    const finish = () => {
-      if (!isCurrent()) {
-        return
-      }
-
-      currentAudio = null
-      setReading(false)
-    }
-
-    setReading(true)
-
    try {
-      const { data_url } = await speakText(text)
-
-      if (!isCurrent()) {
-        return
-      }
-
-      const audio = new Audio(data_url)
-      currentAudio = audio
-      audio.addEventListener('ended', finish, { once: true })
-      audio.addEventListener('error', finish, { once: true })
-      await audio.play()
+      await playSpeechText(text, { messageId, source: 'read-aloud' })
    } catch (error) {
-      if (isCurrent()) {
-        notifyError(error, 'Read aloud failed')
-        finish()
-      }
+      notifyError(error, 'Read aloud failed')
    }
-  }, [text])
-
-  const Icon = reading ? VolumeXIcon : Volume2Icon
+  }, [messageId, text])

  return (
    <DropdownMenuItem
-      disabled={!reading && !text}
+      disabled={isPreparing || (!isSpeaking && (anyPlaybackActive || !text))}
      onSelect={e => {
        e.preventDefault()
-        void (reading ? stop() : read())
+        void (isSpeaking ? stopVoicePlayback() : read())
      }}
    >
-      <Icon />
-      {reading ? 'Stop reading' : 'Read aloud'}
+      <Icon className={isPreparing ? 'animate-spin' : undefined} />
+      {isPreparing ? 'Preparing audio...' : isSpeaking ? 'Stop reading' : 'Read aloud'}
    </DropdownMenuItem>
  )
 }