feat: lots of speech stuff
This commit is contained in:
Brooklyn Nicholson
@@ -0,0 +1,18 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { chatMessageText, toChatMessages } from './chat-messages'
|
||||
|
||||
describe('toChatMessages', () => {
|
||||
it('hides attached context payloads from user message display', () => {
|
||||
const [message] = toChatMessages([
|
||||
{
|
||||
role: 'user',
|
||||
content:
|
||||
'what is this file\n\n--- Attached Context ---\n\n📄 @file:tsconfig.tsbuildinfo (981 tokens)\n```json\n{"root":["./src/main.tsx"]}\n```',
|
||||
timestamp: 1
|
||||
}
|
||||
])
|
||||
|
||||
expect(chatMessageText(message)).toBe('@file:tsconfig.tsbuildinfo\n\nwhat is this file')
|
||||
})
|
||||
})
|
||||
@@ -29,6 +29,7 @@ export type GatewayEventPayload = {
|
||||
todos?: unknown
|
||||
model?: string
|
||||
provider?: string
|
||||
running?: boolean
|
||||
cwd?: string
|
||||
branch?: string
|
||||
personality?: string
|
||||
@@ -49,6 +50,28 @@ export function chatMessageText(message: ChatMessage): string {
|
||||
.join('')
|
||||
}
|
||||
|
||||
const ATTACHED_CONTEXT_MARKER_RE = /(?:^|\n)--- Attached Context ---\s*\n/
|
||||
const CONTEXT_WARNINGS_MARKER_RE = /(?:^|\n)--- Context Warnings ---[\s\S]*$/
|
||||
const CONTEXT_REF_RE = /@(file|folder|url|image|tool):(?:"[^"\n]+"|'[^'\n]+'|`[^`\n]+`|\S+)/g
|
||||
|
||||
function displayContentForMessage(role: SessionMessage['role'], content: string): string {
|
||||
if (role !== 'user') {
|
||||
return content
|
||||
}
|
||||
|
||||
const marker = content.match(ATTACHED_CONTEXT_MARKER_RE)
|
||||
|
||||
if (!marker || marker.index === undefined) {
|
||||
return content.replace(CONTEXT_WARNINGS_MARKER_RE, '').trim()
|
||||
}
|
||||
|
||||
const visibleText = content.slice(0, marker.index).replace(CONTEXT_WARNINGS_MARKER_RE, '').trim()
|
||||
const attachedContext = content.slice(marker.index + marker[0].length)
|
||||
const refs = [...new Set(Array.from(attachedContext.matchAll(CONTEXT_REF_RE)).map(match => match[0]))]
|
||||
|
||||
return [refs.join('\n'), visibleText].filter(Boolean).join('\n\n') || visibleText
|
||||
}
|
||||
|
||||
export function appendTextPart(parts: ChatMessagePart[], delta: string): ChatMessagePart[] {
|
||||
const next = [...parts]
|
||||
const last = next.at(-1)
|
||||
@@ -363,6 +386,7 @@ export function toChatMessages(messages: SessionMessage[]): ChatMessage[] {
|
||||
}
|
||||
|
||||
const content = message.content || message.text || message.context || message.name || ''
|
||||
const displayContent = displayContentForMessage(message.role, content)
|
||||
const parts: ChatMessagePart[] = []
|
||||
|
||||
const reasoning =
|
||||
@@ -374,8 +398,8 @@ export function toChatMessages(messages: SessionMessage[]): ChatMessage[] {
|
||||
parts.push(reasoningPart(reasoning))
|
||||
}
|
||||
|
||||
if (content) {
|
||||
parts.push(textPart(content))
|
||||
if (displayContent) {
|
||||
parts.push(textPart(displayContent))
|
||||
}
|
||||
|
||||
if (message.role === 'assistant' && Array.isArray(message.tool_calls)) {
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { coerceThinkingText } from './chat-runtime'
|
||||
|
||||
describe('coerceThinkingText', () => {
|
||||
it('strips streaming status prefixes from thinking deltas', () => {
|
||||
expect(coerceThinkingText("◉_◉ processing... checking the user's request")).toBe("checking the user's request")
|
||||
expect(coerceThinkingText('(¬‿¬) analyzing... reading the file')).toBe('reading the file')
|
||||
})
|
||||
|
||||
it('drops empty thinking rewrite placeholder text', () => {
|
||||
expect(
|
||||
coerceThinkingText(
|
||||
"◉_◉ processing... I don't see any current rewritten thinking or next thinking to process. Could you provide the thinking content you'd like me to rewrite?"
|
||||
)
|
||||
).toBe('')
|
||||
})
|
||||
})
|
||||
@@ -2,6 +2,7 @@ import type { ThreadMessage } from '@assistant-ui/react'
|
||||
|
||||
import type { QuickModelOption } from '@/app/chat/composer/types'
|
||||
import type { ClientSessionState, CommandDispatchResponse } from '@/app/types'
|
||||
import { formatRefValue } from '@/components/assistant-ui/directive-text'
|
||||
import { type ChatMessage, type ChatMessagePart, chatMessageText, textPart } from '@/lib/chat-messages'
|
||||
import type { ComposerAttachment } from '@/store/composer'
|
||||
import type { ModelOptionsResponse, SessionInfo } from '@/types/hermes'
|
||||
@@ -25,7 +26,11 @@ export const BUILTIN_PERSONALITIES = [
|
||||
'hype'
|
||||
]
|
||||
|
||||
const SPINNER_STATUS_RE = /^\s*[（(][^\s)）]{1,8}[)）]\s+[^.\n]{2,48}\.\.\.\s*/
|
||||
const THINKING_STATUS_PREFIX_RE =
|
||||
/^\s*(?:(?:[^\s.]{1,16})\s+)?(?:processing|thinking|reasoning|analyzing|pondering|contemplating|musing|cogitating|ruminating|deliberating|mulling|reflecting|computing|synthesizing|formulating|brainstorming)\.\.\.\s*/i
|
||||
|
||||
const EMPTY_THINKING_PLACEHOLDER_RE =
|
||||
/\b(?:current rewritten thinking|next thinking to process|provide the thinking content|don't see any .*thinking)\b/i
|
||||
|
||||
export function createClientSessionState(
|
||||
storedSessionId: string | null = null,
|
||||
@@ -102,7 +107,9 @@ export function coerceGatewayText(value: unknown): string {
|
||||
}
|
||||
|
||||
export function coerceThinkingText(value: unknown): string {
|
||||
return coerceGatewayText(value).replace(SPINNER_STATUS_RE, '').trim()
|
||||
const text = coerceGatewayText(value).replace(THINKING_STATUS_PREFIX_RE, '').trim()
|
||||
|
||||
return EMPTY_THINKING_PLACEHOLDER_RE.test(text) ? '' : text
|
||||
}
|
||||
|
||||
export function isImageGenerationTool(name?: string): boolean {
|
||||
@@ -135,7 +142,7 @@ export function attachmentDisplayText(attachment: ComposerAttachment): string |
|
||||
if (attachment.kind === 'image') {
|
||||
const id = attachment.detail || attachment.path || attachment.label
|
||||
|
||||
return id ? `@image:${id}` : null
|
||||
return id ? `@image:${formatRefValue(id)}` : null
|
||||
}
|
||||
|
||||
return null
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
const EMOJI_RE = /[\p{Extended_Pictographic}\uFE0F\u200D]+/gu
|
||||
const FENCED_CODE_RE = /```[\s\S]*?(?:```|$)/g
|
||||
const INLINE_CODE_RE = /`([^`]+)`/g
|
||||
const MARKDOWN_LINK_RE = /\[([^\]]+)\]\(([^)]+)\)/g
|
||||
const URL_RE = /\bhttps?:\/\/\S+/gi
|
||||
|
||||
export function sanitizeTextForSpeech(text: string): string {
|
||||
return text
|
||||
.replace(FENCED_CODE_RE, ' ')
|
||||
.replace(MARKDOWN_LINK_RE, '$1')
|
||||
.replace(INLINE_CODE_RE, '$1')
|
||||
.replace(URL_RE, ' link ')
|
||||
.replace(EMOJI_RE, ' ')
|
||||
.replace(/^#{1,6}\s+/gm, '')
|
||||
.replace(/[*_~>#]/g, '')
|
||||
.replace(/^\s*[-+*]\s+/gm, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import { speakText } from '@/hermes'
|
||||
import {
|
||||
$voicePlayback,
|
||||
setVoicePlaybackState,
|
||||
type VoicePlaybackSource,
|
||||
type VoicePlaybackState
|
||||
} from '@/store/voice-playback'
|
||||
|
||||
import { sanitizeTextForSpeech } from './speech-text'
|
||||
|
||||
let currentAudio: HTMLAudioElement | null = null
|
||||
let sequence = 0
|
||||
|
||||
function currentState(status: VoicePlaybackState['status'], options?: VoicePlaybackOptions): VoicePlaybackState {
|
||||
return {
|
||||
messageId: options?.messageId ?? null,
|
||||
sequence,
|
||||
source: options?.source ?? null,
|
||||
status
|
||||
}
|
||||
}
|
||||
|
||||
export interface VoicePlaybackOptions {
|
||||
messageId?: string | null
|
||||
source: VoicePlaybackSource
|
||||
}
|
||||
|
||||
export function stopVoicePlayback() {
|
||||
sequence += 1
|
||||
|
||||
if (currentAudio) {
|
||||
currentAudio.pause()
|
||||
currentAudio.src = ''
|
||||
currentAudio = null
|
||||
}
|
||||
|
||||
setVoicePlaybackState({
|
||||
messageId: null,
|
||||
sequence,
|
||||
source: null,
|
||||
status: 'idle'
|
||||
})
|
||||
}
|
||||
|
||||
export async function playSpeechText(text: string, options: VoicePlaybackOptions): Promise<boolean> {
|
||||
stopVoicePlayback()
|
||||
|
||||
const speakableText = sanitizeTextForSpeech(text)
|
||||
|
||||
if (!speakableText) {
|
||||
return false
|
||||
}
|
||||
|
||||
const ownSequence = sequence
|
||||
const isCurrent = () => ownSequence === sequence
|
||||
|
||||
setVoicePlaybackState(currentState('preparing', options))
|
||||
|
||||
try {
|
||||
const response = await speakText(speakableText)
|
||||
|
||||
if (!isCurrent()) {
|
||||
return false
|
||||
}
|
||||
|
||||
const audio = new Audio(response.data_url)
|
||||
currentAudio = audio
|
||||
setVoicePlaybackState(currentState('speaking', options))
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
audio.addEventListener('ended', () => resolve(), { once: true })
|
||||
audio.addEventListener('error', () => reject(new Error('Playback failed')), { once: true })
|
||||
void audio.play().catch(reject)
|
||||
})
|
||||
|
||||
if (!isCurrent()) {
|
||||
return false
|
||||
}
|
||||
|
||||
currentAudio = null
|
||||
setVoicePlaybackState(currentState('idle'))
|
||||
|
||||
return true
|
||||
} catch (error) {
|
||||
if (isCurrent()) {
|
||||
currentAudio = null
|
||||
setVoicePlaybackState(currentState('idle'))
|
||||
}
|
||||
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export function isVoicePlaybackActive() {
|
||||
return $voicePlayback.get().status !== 'idle'
|
||||
}
|
||||
Reference in New Issue
Block a user