chore: uptick

This commit is contained in:
Brooklyn Nicholson
2026-05-02 03:19:39 -05:00
parent 420f68e4e2
commit db884f4646
240 changed files with 25206 additions and 3155 deletions
+84 -1
View File
@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest'
import { chatMessageText, toChatMessages } from './chat-messages'
import { appendAssistantTextPart, chatMessageText, renderMediaTags, toChatMessages, upsertToolPart } from './chat-messages'
describe('toChatMessages', () => {
it('hides attached context payloads from user message display', () => {
@@ -15,4 +15,87 @@ describe('toChatMessages', () => {
expect(chatMessageText(message)).toBe('@file:tsconfig.tsbuildinfo\n\nwhat is this file')
})
it('renders MEDIA tags as assistant attachment links', () => {
const [message] = toChatMessages([
{
role: 'assistant',
content: "MEDIA:/Users/brooklyn/.hermes/cache/audio/tts_20260501_222725.mp3\n\nhow's that sound?",
timestamp: 1
}
])
expect(chatMessageText(message)).toBe(
"[Audio: tts_20260501_222725.mp3](#media:%2FUsers%2Fbrooklyn%2F.hermes%2Fcache%2Faudio%2Ftts_20260501_222725.mp3)\n\nhow's that sound?"
)
})
it('coerces non-string message content without throwing', () => {
const [message] = toChatMessages([
{
content: {
text: 'hello from object content'
},
role: 'assistant',
timestamp: 1
}
])
expect(chatMessageText(message)).toBe('hello from object content')
})
it('applies attached-context filtering when user content is object-shaped', () => {
const [message] = toChatMessages([
{
content: {
text:
'look\n\n--- Attached Context ---\n\n📄 @file:foo.ts (10 tokens)\n```ts\nconst x = 1\n```'
},
role: 'user',
timestamp: 1
}
])
expect(chatMessageText(message)).toBe('@file:foo.ts\n\nlook')
})
})
describe('renderMediaTags', () => {
it('renders standalone and inline MEDIA tags as links', () => {
expect(renderMediaTags('here\nMEDIA:/tmp/voice.mp3\nthere')).toBe(
'here\n[Audio: voice.mp3](#media:%2Ftmp%2Fvoice.mp3)\nthere'
)
expect(renderMediaTags('audio: MEDIA:/tmp/voice.mp3 done')).toBe(
'audio: [Audio: voice.mp3](#media:%2Ftmp%2Fvoice.mp3) done'
)
expect(renderMediaTags('MEDIA:/tmp/demo.mp4')).toBe('[Video: demo.mp4](#media:%2Ftmp%2Fdemo.mp4)')
})
it('renders streamed assistant media once the tag is complete', () => {
const parts = appendAssistantTextPart(appendAssistantTextPart([], 'ok\nMEDIA:'), '/tmp/voice.mp3')
const text = chatMessageText({ id: 'a', role: 'assistant', parts })
expect(text).toBe('ok\n[Audio: voice.mp3](#media:%2Ftmp%2Fvoice.mp3)')
})
})
describe('upsertToolPart', () => {
it('preserves inline diffs from tool completion events', () => {
const parts = upsertToolPart(
[],
{
inline_diff: '--- a/foo.ts\n+++ b/foo.ts\n@@\n-old\n+new',
name: 'patch',
tool_id: 'tool-1'
},
'complete'
)
const [part] = parts
expect(part?.type).toBe('tool-call')
expect(part && 'result' in part ? part.result : undefined).toMatchObject({
inline_diff: '--- a/foo.ts\n+++ b/foo.ts\n@@\n-old\n+new'
})
})
})
+109 -16
View File
@@ -1,5 +1,6 @@
import type { ThreadMessageLike } from '@assistant-ui/react'
import { mediaDisplayLabel, mediaMarkdownHref } from '@/lib/media'
import type { SessionMessage } from '@/types/hermes'
export type ChatMessagePart = Exclude<ThreadMessageLike['content'], string>[number]
@@ -25,6 +26,7 @@ export type GatewayEventPayload = {
preview?: string
summary?: string
error?: string | boolean
inline_diff?: string
duration_s?: number
todos?: unknown
model?: string
@@ -33,6 +35,10 @@ export type GatewayEventPayload = {
cwd?: string
branch?: string
personality?: string
// clarify.request
request_id?: string
question?: string
choices?: string[] | null
}
export function textPart(text: string): ChatMessagePart {
@@ -43,6 +49,37 @@ export function reasoningPart(text: string): ChatMessagePart {
return { type: 'reasoning', text }
}
const MEDIA_LINE_RE =
/(^|\n)[\t ]*[`"']?MEDIA:\s*(?<line>`[^`\n]+`|"[^"\n]+"|'[^'\n]+'|\S+)[`"']?[\t ]*(?:\n|$)/g
const MEDIA_TAG_RE = /[`"']?MEDIA:\s*(?<inline>`[^`\n]+`|"[^"\n]+"|'[^'\n]+'|\S+)[`"']?/g
function unquoteMediaPath(value: string): string {
const trimmed = value.trim()
const quote = trimmed[0]
return quote && quote === trimmed.at(-1) && ['"', "'", '`'].includes(quote) ? trimmed.slice(1, -1) : trimmed
}
function mediaLink(value: string): string {
const path = unquoteMediaPath(value)
return `[${mediaDisplayLabel(path)}](${mediaMarkdownHref(path)})`
}
export function renderMediaTags(text: string): string {
return text
.replace(MEDIA_LINE_RE, (_match, lead: string, value: string) => `${lead}${mediaLink(value)}\n`)
.replace(MEDIA_TAG_RE, (_match, value: string) => mediaLink(value))
.replace(/[ \t]+\n/g, '\n')
.replace(/\n{3,}/g, '\n\n')
.trim()
}
export function assistantTextPart(text: string): ChatMessagePart {
return textPart(renderMediaTags(text))
}
export function chatMessageText(message: ChatMessage): string {
return message.parts
.filter((part): part is Extract<ChatMessagePart, { type: 'text' }> => part.type === 'text')
@@ -54,19 +91,57 @@ const ATTACHED_CONTEXT_MARKER_RE = /(?:^|\n)--- Attached Context ---\s*\n/
const CONTEXT_WARNINGS_MARKER_RE = /(?:^|\n)--- Context Warnings ---[\s\S]*$/
const CONTEXT_REF_RE = /@(file|folder|url|image|tool):(?:"[^"\n]+"|'[^'\n]+'|`[^`\n]+`|\S+)/g
function displayContentForMessage(role: SessionMessage['role'], content: string): string {
if (role !== 'user') {
return content
function textFromUnknown(value: unknown, depth = 0): string {
if (typeof value === 'string') {
return value
}
const marker = content.match(ATTACHED_CONTEXT_MARKER_RE)
if (value === null || value === undefined) {
return ''
}
if (depth > 2) {
return ''
}
if (Array.isArray(value)) {
return value.map(item => textFromUnknown(item, depth + 1)).join('')
}
if (typeof value === 'object') {
const row = value as Record<string, unknown>
const textValue = row.text ?? row.output_text ?? row.content ?? row.message
const nestedText = textFromUnknown(textValue, depth + 1)
if (nestedText) {
return nestedText
}
try {
return JSON.stringify(value)
} catch {
return ''
}
}
return String(value)
}
function displayContentForMessage(role: SessionMessage['role'], content: unknown): string {
const textContent = textFromUnknown(content)
if (role !== 'user') {
return textContent
}
const marker = textContent.match(ATTACHED_CONTEXT_MARKER_RE)
if (!marker || marker.index === undefined) {
return content.replace(CONTEXT_WARNINGS_MARKER_RE, '').trim()
return textContent.replace(CONTEXT_WARNINGS_MARKER_RE, '').trim()
}
const visibleText = content.slice(0, marker.index).replace(CONTEXT_WARNINGS_MARKER_RE, '').trim()
const attachedContext = content.slice(marker.index + marker[0].length)
const visibleText = textContent.slice(0, marker.index).replace(CONTEXT_WARNINGS_MARKER_RE, '').trim()
const attachedContext = textContent.slice(marker.index + marker[0].length)
const refs = [...new Set(Array.from(attachedContext.matchAll(CONTEXT_REF_RE)).map(match => match[0]))]
return [refs.join('\n'), visibleText].filter(Boolean).join('\n\n') || visibleText
@@ -87,6 +162,17 @@ export function appendTextPart(parts: ChatMessagePart[], delta: string): ChatMes
return next
}
export function appendAssistantTextPart(parts: ChatMessagePart[], delta: string): ChatMessagePart[] {
const next = appendTextPart(parts, delta)
const last = next.at(-1)
if (last?.type === 'text') {
next[next.length - 1] = { ...last, text: renderMediaTags(last.text) }
}
return next
}
export function appendReasoningPart(parts: ChatMessagePart[], delta: string): ChatMessagePart[] {
const next = [...parts]
const last = next.at(-1)
@@ -119,6 +205,7 @@ function toolArgs(payload: GatewayEventPayload | undefined): Record<string, unkn
function toolResult(payload: GatewayEventPayload | undefined): Record<string, unknown> {
return {
...(payload?.inline_diff ? { inline_diff: payload.inline_diff } : {}),
...(payload?.summary ? { summary: payload.summary } : {}),
...(payload?.message ? { message: payload.message } : {}),
...(payload?.preview ? { preview: payload.preview } : {}),
@@ -198,15 +285,21 @@ function firstNonEmptyObject(...values: unknown[]): Record<string, unknown> {
return {}
}
function parseStoredToolResult(content: string): unknown {
if (!content.trim()) {
function parseStoredToolResult(content: unknown): unknown {
if (content && typeof content === 'object') {
return content
}
const textContent = textFromUnknown(content)
if (!textContent.trim()) {
return ''
}
try {
return JSON.parse(content)
return JSON.parse(textContent)
} catch {
return content
return textContent
}
}
@@ -233,7 +326,7 @@ function toolPartFromStoredCall(call: unknown, fallbackIndex: number): ChatMessa
function applyStoredToolResult(messages: ChatMessage[], toolMessage: SessionMessage): boolean {
const toolCallId = toolMessage.tool_call_id || undefined
const toolName = toolMessage.tool_name || toolMessage.name || 'tool'
const content = toolMessage.content || toolMessage.text || toolMessage.context || toolMessage.name || ''
const content = toolMessage.content || toolMessage.text || toolMessage.context || toolMessage.name
for (let i = messages.length - 1; i >= 0; i -= 1) {
const message = messages[i]
@@ -270,7 +363,7 @@ function applyStoredToolResult(messages: ChatMessage[], toolMessage: SessionMess
function applyStoredToolResultToParts(parts: ChatMessagePart[], toolMessage: SessionMessage): ChatMessagePart[] | null {
const toolCallId = toolMessage.tool_call_id || undefined
const toolName = toolMessage.tool_name || toolMessage.name || 'tool'
const content = toolMessage.content || toolMessage.text || toolMessage.context || toolMessage.name || ''
const content = toolMessage.content || toolMessage.text || toolMessage.context || toolMessage.name
const partIndex = parts.findIndex(
part =>
@@ -295,7 +388,7 @@ function applyStoredToolResultToParts(parts: ChatMessagePart[], toolMessage: Ses
function storedToolMessagePart(toolMessage: SessionMessage, fallbackIndex: number): ChatMessagePart {
const name = toolMessage.tool_name || toolMessage.name || 'tool'
const context = toolMessage.context || toolMessage.text || toolMessage.content || ''
const context = textFromUnknown(toolMessage.context || toolMessage.text || toolMessage.content || '')
const args = context ? { context } : {}
return {
@@ -385,7 +478,7 @@ export function toChatMessages(messages: SessionMessage[]): ChatMessage[] {
return
}
const content = message.content || message.text || message.context || message.name || ''
const content = message.content || message.text || message.context || message.name
const displayContent = displayContentForMessage(message.role, content)
const parts: ChatMessagePart[] = []
@@ -399,7 +492,7 @@ export function toChatMessages(messages: SessionMessage[]): ChatMessage[] {
}
if (displayContent) {
parts.push(textPart(displayContent))
parts.push(message.role === 'assistant' ? assistantTextPart(displayContent) : textPart(displayContent))
}
if (message.role === 'assistant' && Array.isArray(message.tool_calls)) {
@@ -0,0 +1,108 @@
import { describe, expect, it } from 'vitest'
import {
desktopSlashDescription,
desktopSlashUnavailableMessage,
desktopSkinSlashCompletions,
filterDesktopCommandsCatalog,
isDesktopSlashCommand,
isDesktopSlashSuggestion
} from './desktop-slash-commands'
describe('desktop slash command curation', () => {
it('keeps core desktop chat commands in suggestions', () => {
expect(isDesktopSlashSuggestion('/new')).toBe(true)
expect(isDesktopSlashSuggestion('/branch')).toBe(true)
expect(isDesktopSlashSuggestion('/skin')).toBe(true)
expect(isDesktopSlashSuggestion('/usage')).toBe(true)
})
it('lets explicitly typed extension commands run without suggesting them', () => {
expect(isDesktopSlashSuggestion('/my-skill')).toBe(false)
expect(isDesktopSlashCommand('/my-skill')).toBe(true)
})
it('hides terminal, messaging, and dedicated-UI commands from suggestions', () => {
expect(isDesktopSlashSuggestion('/clear')).toBe(false)
expect(isDesktopSlashSuggestion('/compact')).toBe(false)
expect(isDesktopSlashSuggestion('/redraw')).toBe(false)
expect(isDesktopSlashSuggestion('/approve')).toBe(false)
expect(isDesktopSlashSuggestion('/model')).toBe(false)
expect(isDesktopSlashSuggestion('/skills')).toBe(false)
expect(isDesktopSlashSuggestion('/voice')).toBe(false)
expect(isDesktopSlashSuggestion('/curator')).toBe(false)
})
it('allows aliases to execute without cluttering the popover', () => {
expect(isDesktopSlashSuggestion('/reset')).toBe(false)
expect(isDesktopSlashCommand('/reset')).toBe(true)
})
it('filters command catalogs down to core desktop commands', () => {
const filtered = filterDesktopCommandsCatalog({
categories: [
{
name: 'Session',
pairs: [
['/new', 'Start a new session'],
['/clear', 'Clear terminal screen']
]
},
{
name: 'User commands',
pairs: [['/ship-it', 'Run release checklist']]
}
],
pairs: [
['/new', 'Start a new session'],
['/model', 'Switch model'],
['/ship-it', 'Run release checklist']
],
skill_count: 2
})
expect(filtered.categories).toEqual([{ name: 'Session', pairs: [['/new', 'Start a new desktop chat']] }])
expect(filtered.pairs).toEqual([['/new', 'Start a new desktop chat']])
expect(filtered.skill_count).toBe(2)
})
it('uses desktop-specific labels for commands with different UI behavior', () => {
expect(desktopSlashDescription('/branch', 'Branch the current session')).toBe(
'Branch the latest message into a new chat'
)
expect(desktopSlashDescription('/skin', 'Show or change the display skin/theme')).toBe(
'Switch desktop theme or cycle to the next one'
)
})
it('builds /skin completions from desktop themes', () => {
const completions = desktopSkinSlashCompletions(
[
{ name: 'mono', label: 'Mono', description: 'Clean grayscale' },
{ name: 'midnight', label: 'Midnight', description: 'Deep blue' },
{ name: 'slate', label: 'Slate', description: 'Cool slate blue' }
],
'mono',
'm'
)
expect(completions).toEqual([
{
text: '/skin mono',
display: '/skin mono',
meta: 'Mono (current) - Clean grayscale'
},
{
text: '/skin midnight',
display: '/skin midnight',
meta: 'Midnight - Deep blue'
}
])
})
it('explains known commands that desktop owns elsewhere', () => {
expect(desktopSlashUnavailableMessage('/model sonnet')).toContain('model picker')
expect(desktopSlashUnavailableMessage('/skills')).toContain('desktop sidebar')
expect(desktopSlashUnavailableMessage('/clear')).toContain('terminal interface')
})
})
@@ -0,0 +1,251 @@
export interface CommandsCatalogSection {
name: string
pairs: [string, string][]
}
export interface CommandsCatalogLike {
categories?: CommandsCatalogSection[]
pairs?: [string, string][]
skill_count?: number
warning?: string
}
export interface DesktopSlashCompletion {
display: string
meta: string
text: string
}
export interface DesktopThemeCommandOption {
description: string
label: string
name: string
}
const DESKTOP_COMMAND_META = [
['/agents', 'Show active desktop sessions and running tasks'],
['/background', 'Run a prompt in the background'],
['/branch', 'Branch the latest message into a new chat'],
['/compress', 'Compress this conversation context'],
['/debug', 'Create a debug report'],
['/goal', 'Manage the standing goal for this session'],
['/help', 'Show desktop slash commands'],
['/new', 'Start a new desktop chat'],
['/queue', 'Queue a prompt for the next turn'],
['/resume', 'Resume a saved session'],
['/retry', 'Retry the last user message'],
['/rollback', 'List or restore filesystem checkpoints'],
['/skin', 'Switch desktop theme or cycle to the next one'],
['/status', 'Show current session status'],
['/steer', 'Steer the current run after the next tool call'],
['/stop', 'Stop running background processes'],
['/title', 'Rename the current session'],
['/undo', 'Remove the last user/assistant exchange'],
['/usage', 'Show token usage for this session']
] as const
const DESKTOP_COMMANDS: ReadonlySet<string> = new Set(DESKTOP_COMMAND_META.map(([command]) => command))
const DESKTOP_ALIASES = new Map([
['/bg', '/background'],
['/btw', '/background'],
['/fork', '/branch'],
['/q', '/queue'],
['/reload_mcp', '/reload-mcp'],
['/reload_skills', '/reload-skills'],
['/reset', '/new'],
['/tasks', '/agents']
])
const DESKTOP_COMMAND_DESCRIPTIONS: ReadonlyMap<string, string> = new Map(DESKTOP_COMMAND_META)
const PICKER_OWNED_COMMANDS = new Set(['/model', '/provider'])
const TERMINAL_ONLY_COMMANDS = new Set([
'/browser',
'/busy',
'/clear',
'/commands',
'/compact',
'/config',
'/copy',
'/cron',
'/details',
'/exit',
'/footer',
'/gateway',
'/gquota',
'/history',
'/image',
'/indicator',
'/logs',
'/mouse',
'/paste',
'/platforms',
'/plugins',
'/quit',
'/redraw',
'/reload',
'/restart',
'/save',
'/sb',
'/set-home',
'/sethome',
'/snap',
'/snapshot',
'/statusbar',
'/toolsets',
'/tools',
'/update',
'/verbose'
])
const MESSAGING_ONLY_COMMANDS = new Set(['/approve', '/deny'])
const SETTINGS_OWNED_COMMANDS = new Set(['/skills'])
const ADVANCED_COMMANDS = new Set([
'/curator',
'/fast',
'/insights',
'/kanban',
'/personality',
'/profile',
'/reasoning',
'/reload-mcp',
'/reload-skills',
'/voice',
'/yolo'
])
const BLOCKED_COMMANDS = new Set([
...PICKER_OWNED_COMMANDS,
...TERMINAL_ONLY_COMMANDS,
...MESSAGING_ONLY_COMMANDS,
...SETTINGS_OWNED_COMMANDS,
...ADVANCED_COMMANDS
])
function normalizeCommand(command: string): string {
const trimmed = command.trim()
const base = (trimmed.startsWith('/') ? trimmed : `/${trimmed}`).split(/\s+/, 1)[0]?.toLowerCase() || ''
return base
}
export function canonicalDesktopSlashCommand(command: string): string {
const normalized = normalizeCommand(command)
return DESKTOP_ALIASES.get(normalized) || normalized
}
export function isDesktopSlashCommand(command: string): boolean {
const normalized = normalizeCommand(command)
const canonical = canonicalDesktopSlashCommand(normalized)
if (BLOCKED_COMMANDS.has(normalized) || BLOCKED_COMMANDS.has(canonical)) {
return false
}
return DESKTOP_COMMANDS.has(canonical) || !isKnownHermesSlashCommand(normalized)
}
export function isDesktopSlashSuggestion(command: string): boolean {
const normalized = normalizeCommand(command)
const canonical = canonicalDesktopSlashCommand(normalized)
return DESKTOP_COMMANDS.has(canonical) && !DESKTOP_ALIASES.has(normalized)
}
export function desktopSlashUnavailableMessage(command: string): string | null {
const normalized = normalizeCommand(command)
const canonical = canonicalDesktopSlashCommand(normalized)
if (PICKER_OWNED_COMMANDS.has(canonical)) {
return `/${canonical.slice(1)} uses the desktop model picker instead of a slash command.`
}
if (SETTINGS_OWNED_COMMANDS.has(canonical)) {
return `/${canonical.slice(1)} is managed from the desktop sidebar.`
}
if (MESSAGING_ONLY_COMMANDS.has(canonical)) {
return `/${canonical.slice(1)} is only used from messaging platforms.`
}
if (ADVANCED_COMMANDS.has(canonical)) {
return `/${canonical.slice(1)} is not shown in the desktop slash palette. Use the relevant desktop control or terminal interface instead.`
}
if (TERMINAL_ONLY_COMMANDS.has(normalized) || TERMINAL_ONLY_COMMANDS.has(canonical)) {
return `/${canonical.slice(1)} is only available in the terminal interface.`
}
return null
}
export function desktopSlashDescription(command: string, fallback = ''): string {
const canonical = canonicalDesktopSlashCommand(command)
return DESKTOP_COMMAND_DESCRIPTIONS.get(canonical) || fallback
}
export function desktopSkinSlashCompletions(
themes: DesktopThemeCommandOption[],
activeThemeName: string,
argPrefix: string
): DesktopSlashCompletion[] {
const prefix = argPrefix.trim().toLowerCase()
const commands: DesktopSlashCompletion[] = [
{
text: '/skin list',
display: '/skin list',
meta: 'Show available desktop themes'
},
{
text: '/skin next',
display: '/skin next',
meta: 'Cycle to the next desktop theme'
},
...themes.map(theme => ({
text: `/skin ${theme.name}`,
display: `/skin ${theme.name}`,
meta: `${theme.label}${theme.name === activeThemeName ? ' (current)' : ''} - ${theme.description}`
}))
]
if (!prefix) {
return commands
}
return commands.filter(item => item.text.slice('/skin '.length).toLowerCase().startsWith(prefix))
}
export function filterDesktopCommandsCatalog(catalog: CommandsCatalogLike): CommandsCatalogLike {
const categories = catalog.categories
?.map(section => ({
...section,
pairs: section.pairs
.filter(([command]) => isDesktopSlashSuggestion(command))
.map(([command, description]) => [command, desktopSlashDescription(command, description)] as [string, string])
}))
.filter(section => section.pairs.length > 0)
const pairs = catalog.pairs
?.filter(([command]) => isDesktopSlashSuggestion(command))
.map(([command, description]) => [command, desktopSlashDescription(command, description)] as [string, string])
return {
...catalog,
...(categories ? { categories } : {}),
...(pairs ? { pairs } : {})
}
}
function isKnownHermesSlashCommand(command: string): boolean {
return (
DESKTOP_COMMANDS.has(command) ||
DESKTOP_ALIASES.has(command) ||
BLOCKED_COMMANDS.has(command)
)
}
@@ -0,0 +1,35 @@
import { describe, expect, it } from 'vitest'
import { extractEmbeddedImages } from './embedded-images'
const SAMPLE_PNG_DATA_URL = 'data:image/png;base64,' + 'A'.repeat(120)
describe('extractEmbeddedImages', () => {
it('returns text untouched when no data URL is present', () => {
expect(extractEmbeddedImages('describe this')).toEqual({ cleanedText: 'describe this', images: [] })
})
it('lifts a bare data:image URL out of prose', () => {
const result = extractEmbeddedImages(`describe this ${SAMPLE_PNG_DATA_URL}`)
expect(result.cleanedText).toBe('describe this')
expect(result.images).toEqual([SAMPLE_PNG_DATA_URL])
})
it('lifts a JSON-wrapped image_url envelope out of prose', () => {
const result = extractEmbeddedImages(
`describe this{"type":"image_url","image_url":{"url":"${SAMPLE_PNG_DATA_URL}"}}`
)
expect(result.cleanedText).toBe('describe this')
expect(result.images).toEqual([SAMPLE_PNG_DATA_URL])
})
it('extracts multiple embedded images', () => {
const second = 'data:image/jpeg;base64,' + 'B'.repeat(96)
const result = extractEmbeddedImages(`first ${SAMPLE_PNG_DATA_URL} mid ${second} tail`)
expect(result.cleanedText).toBe('first mid tail')
expect(result.images).toEqual([SAMPLE_PNG_DATA_URL, second])
})
})
+59
View File
@@ -0,0 +1,59 @@
const EMBEDDED_IMAGE_RE =
/(\{\s*"type"\s*:\s*"image_url"\s*,\s*"image_url"\s*:\s*\{\s*"url"\s*:\s*")?(data:image\/[\w.+-]+;base64,[A-Za-z0-9+/=]{64,})("\s*\}\s*\})?/g
const DATA_URL_RE = /^data:([\w./+-]+);base64,(.*)$/i
export const DATA_IMAGE_URL_RE = /^data:image\/[\w.+-]+;base64,/i
export interface EmbeddedImageExtraction {
cleanedText: string
images: string[]
}
export function dataUrlToBlob(dataUrl: string): Blob | null {
const match = DATA_URL_RE.exec(dataUrl.trim())
if (!match) {
return null
}
try {
const bytes = atob(match[2])
const buffer = new Uint8Array(bytes.length)
for (let i = 0; i < bytes.length; i += 1) {
buffer[i] = bytes.charCodeAt(i)
}
return new Blob([buffer], { type: match[1] })
} catch {
return null
}
}
export function extractEmbeddedImages(text: string): EmbeddedImageExtraction {
if (!text || !text.includes('data:image/')) {
return { cleanedText: text, images: [] }
}
const images: string[] = []
const cleanedText = text
.replace(EMBEDDED_IMAGE_RE, (_match, _open, dataUrl: string) => {
images.push(dataUrl)
return ''
})
.replace(/[ \t]+\n/g, '\n')
.replace(/\n{3,}/g, '\n\n')
.trim()
return { cleanedText, images }
}
export function embeddedImageUrls(text: string): string[] {
return extractEmbeddedImages(text).images
}
export function textWithoutEmbeddedImages(text: string): string {
return extractEmbeddedImages(text).cleanedText
}
@@ -0,0 +1,23 @@
import { describe, expect, it } from 'vitest'
import { isLikelyProseCodeBlock } from './markdown-code'
describe('isLikelyProseCodeBlock', () => {
it('detects prose that Streamdown mislabels as an unknown language', () => {
expect(
isLikelyProseCodeBlock(
'heads',
[
'- Pure white (`#ffffff`), roughness 0.55, no emissive',
'- Black wireframe edges at 35% opacity',
'',
'Want the bunny gone, or want me to keep riffing on it?'
].join('\n')
)
).toBe(true)
})
it('keeps real code blocks', () => {
expect(isLikelyProseCodeBlock('ts', 'const value = { bunny: true };\nreturn value')).toBe(false)
})
})
+132
View File
@@ -0,0 +1,132 @@
const VALID_LANGUAGE_RE = /^[a-z0-9][a-z0-9+#-]*$/i
const NON_CODE_FENCE_LANGUAGES = new Set(['', 'text', 'plain', 'plaintext', 'md', 'markdown'])
const COMMON_CODE_LANGUAGES = new Set([
'bash',
'c',
'cpp',
'css',
'diff',
'go',
'html',
'java',
'javascript',
'js',
'json',
'jsx',
'markdown',
'md',
'php',
'python',
'py',
'ruby',
'rust',
'rs',
'sh',
'sql',
'swift',
'tsx',
'ts',
'typescript',
'xml',
'yaml',
'yml'
])
interface CodeSignals {
bulletLines: number
codeSignals: number
hasMarkdown: boolean
proseLines: number
trimmed: string
}
export function sanitizeLanguageTag(tag: string): string {
const trimmed = tag.trim()
const first = trimmed.split(/\s/, 1)[0] || ''
return VALID_LANGUAGE_RE.test(first) && first.length <= 16 ? first.toLowerCase() : ''
}
function proseLineCount(body: string): number {
return body
.split('\n')
.filter(line => {
const trimmed = line.trim()
return Boolean(trimmed) && /^[A-Za-z0-9"'`*-]/.test(trimmed)
})
.length
}
const CODE_SIGNAL_RE = [
/(^|\s)(const|let|var|function|class|import|export|return|if|for|while|switch)\b/gim,
/=>|==|===|!=|!==|\{|\}|;|<\/?[a-z][^>]*>/gi,
/^\s*(#include|SELECT|INSERT|UPDATE|DELETE|CREATE|DROP)\b/gim
]
function codeSignalCount(body: string): number {
return CODE_SIGNAL_RE.reduce((total, pattern) => total + (body.match(pattern)?.length ?? 0), 0)
}
function codeSignals(body: string): CodeSignals {
const trimmed = body.trim()
const markdownSignals = (trimmed.match(/\*\*[^*]+\*\*/g) || []).length + (trimmed.match(/`[^`\n]+`/g) || []).length
return {
bulletLines: (trimmed.match(/^\s*[-*]\s+\S+/gm) || []).length,
codeSignals: codeSignalCount(trimmed),
hasMarkdown: markdownSignals > 0,
proseLines: proseLineCount(trimmed),
trimmed
}
}
export function isLikelyProseFence(info: string, body: string): boolean {
const trimmedInfo = info.trim()
const rawInfo = trimmedInfo.toLowerCase()
const language = sanitizeLanguageTag(info)
const infoToken = trimmedInfo.split(/\s+/, 1)[0] || ''
const hasInfoTail = Boolean(trimmedInfo) && trimmedInfo !== infoToken
if (/^[-*+]\s/.test(rawInfo) || /^https?:\/\//.test(rawInfo)) {
return true
}
const signals = codeSignals(body)
if (!signals.trimmed) {
return false
}
if (hasInfoTail && signals.codeSignals <= 2 && (signals.proseLines >= 2 || signals.bulletLines >= 1)) {
return true
}
if (!NON_CODE_FENCE_LANGUAGES.has(language)) {
return false
}
return (
(signals.bulletLines >= 2 && signals.hasMarkdown && signals.codeSignals <= 2) ||
(signals.proseLines >= 3 && signals.codeSignals === 0)
)
}
export function isLikelyProseCodeBlock(language: string | undefined, code: string | undefined): boolean {
const cleanLanguage = sanitizeLanguageTag(language || '')
const signals = codeSignals(code || '')
if (!signals.trimmed || signals.codeSignals >= 3) {
return false
}
if (signals.bulletLines >= 1 && (signals.hasMarkdown || signals.proseLines >= 2)) {
return true
}
if (NON_CODE_FENCE_LANGUAGES.has(cleanLanguage)) {
return signals.proseLines >= 3 && signals.codeSignals === 0
}
return !COMMON_CODE_LANGUAGES.has(cleanLanguage) && signals.proseLines >= 2 && signals.codeSignals <= 1
}
+90
View File
@@ -0,0 +1,90 @@
export type MediaKind = 'audio' | 'image' | 'video' | 'file'
interface MediaInfo {
kind: MediaKind
mime: string
}
const MEDIA_BY_EXT: Record<string, MediaInfo> = {
avi: { kind: 'video', mime: 'video/x-msvideo' },
bmp: { kind: 'image', mime: 'image/bmp' },
flac: { kind: 'audio', mime: 'audio/flac' },
gif: { kind: 'image', mime: 'image/gif' },
jpeg: { kind: 'image', mime: 'image/jpeg' },
jpg: { kind: 'image', mime: 'image/jpeg' },
m4a: { kind: 'audio', mime: 'audio/mp4' },
mkv: { kind: 'video', mime: 'video/x-matroska' },
mov: { kind: 'video', mime: 'video/quicktime' },
mp3: { kind: 'audio', mime: 'audio/mpeg' },
mp4: { kind: 'video', mime: 'video/mp4' },
ogg: { kind: 'audio', mime: 'audio/ogg' },
opus: { kind: 'audio', mime: 'audio/ogg; codecs=opus' },
png: { kind: 'image', mime: 'image/png' },
svg: { kind: 'image', mime: 'image/svg+xml' },
wav: { kind: 'audio', mime: 'audio/wav' },
webm: { kind: 'video', mime: 'video/webm' },
webp: { kind: 'image', mime: 'image/webp' }
}
function mediaInfo(path: string): MediaInfo | undefined {
const ext = path.split(/[?#]/, 1)[0]?.split('.').pop()?.toLowerCase()
return ext ? MEDIA_BY_EXT[ext] : undefined
}
export function mediaKind(path: string): MediaKind {
return mediaInfo(path)?.kind ?? 'file'
}
export function mediaMime(path: string): string {
return mediaInfo(path)?.mime ?? 'application/octet-stream'
}
export function mediaName(path: string): string {
try {
const url = new URL(path)
return url.pathname.split('/').filter(Boolean).pop() || path
} catch {
return path.split(/[\\/]/).filter(Boolean).pop() || path
}
}
export function mediaMarkdownHref(path: string): string {
return `#media:${encodeURIComponent(path)}`
}
export function mediaExternalUrl(path: string): string {
return /^(?:https?|file):/i.test(path) ? path : `file://${path}`
}
export function mediaPathFromMarkdownHref(href?: string): string | null {
if (!href?.startsWith('#media:')) {
return null
}
try {
return decodeURIComponent(href.slice('#media:'.length))
} catch {
return null
}
}
export function filePathFromMediaPath(path: string): string {
if (!path.startsWith('file:')) {
return path
}
try {
return decodeURIComponent(new URL(path).pathname)
} catch {
return path.replace(/^file:\/\//, '')
}
}
export function mediaDisplayLabel(path: string): string {
const escaped = mediaName(path).replace(/[[\]\\]/g, '\\$&')
const kind = mediaKind(path)
return `${kind[0].toUpperCase()}${kind.slice(1)}: ${escaped}`
}
@@ -0,0 +1,53 @@
import { describe, expect, it } from 'vitest'
import {
extractPreviewCandidates,
extractPreviewTargets,
isLikelyPreviewCandidate,
previewTargetFromMarkdownHref,
renderPreviewTargets,
stripPreviewTargets
} from './preview-targets'
describe('preview target detection', () => {
it('extracts local server URLs and html files', () => {
expect(
extractPreviewCandidates(
'Open http://localhost:5173/ and /tmp/mycelium-bunnies/index.html, not https://example.com/app.'
)
).toEqual(['http://localhost:5173/', '/tmp/mycelium-bunnies/index.html'])
})
it('accepts relative html files and file URLs', () => {
expect(extractPreviewCandidates('Wrote ./dist/index.html and file:///tmp/demo.html.')).toEqual([
'./dist/index.html',
'file:///tmp/demo.html'
])
})
it('ignores remote web URLs', () => {
expect(isLikelyPreviewCandidate('https://example.com/demo')).toBe(false)
expect(isLikelyPreviewCandidate('http://127.0.0.1:3000')).toBe(true)
})
it('renders previewable paths as markdown links', () => {
expect(renderPreviewTargets('ready\n/tmp/mycelium-bunnies.html\nopen it')).toBe(
'ready\n[Preview: mycelium-bunnies.html](#preview/%2Ftmp%2Fmycelium-bunnies.html)\nopen it'
)
})
it('decodes preview markdown hrefs', () => {
expect(previewTargetFromMarkdownHref('#preview/%2Ftmp%2Fdemo.html')).toBe('/tmp/demo.html')
expect(previewTargetFromMarkdownHref('#preview:%2Ftmp%2Fdemo.html')).toBe('/tmp/demo.html')
expect(previewTargetFromMarkdownHref('#media:%2Ftmp%2Fdemo.mp4')).toBeNull()
})
it('extracts preview targets from already-rendered preview markers', () => {
expect(extractPreviewTargets('[Preview: demo.html](#preview:%2Ftmp%2Fdemo.html)')).toEqual(['/tmp/demo.html'])
})
it('strips preview targets from visible assistant text', () => {
expect(stripPreviewTargets('ready\n/tmp/mycelium-bunnies.html\nopen it')).toBe('ready\nopen it')
expect(stripPreviewTargets('[Preview: demo.html](#preview:%2Ftmp%2Fdemo.html)\nopen it')).toBe('open it')
})
})
+216
View File
@@ -0,0 +1,216 @@
const LOCAL_HOSTS = new Set(['0.0.0.0', '127.0.0.1', '::1', '[::1]', 'localhost'])
const HTML_EXT_RE = /\.html?(?:[?#].*)?$/i
const URL_RE = /\bhttps?:\/\/[^\s<>"'`)\]]+/gi
const FILE_URL_RE = /\bfile:\/\/[^\s<>"'`)\]]+/gi
const POSIX_HTML_PATH_RE = /(?:^|[\s("'`])(?<path>\/[^\s<>"'`]*?\.html?)(?:[),.;:!?]*)(?=$|[\s)"'`])/gi
const RELATIVE_HTML_PATH_RE = /(?:^|[\s("'`])(?<path>\.{1,2}\/[^\s<>"'`]*?\.html?)(?:[),.;:!?]*)(?=$|[\s)"'`])/gi
const PREVIEW_MARKDOWN_RE = /\[Preview:[^\]]+\]\((?<href>#preview[:/][^)]+)\)/gi
interface PreviewCandidateMatch {
end: number
index: number
value: string
}
function stripTrailingPunctuation(value: string): string {
return value.replace(/[),.;:!?]+$/, '')
}
function isLocalPreviewUrl(value: string): boolean {
try {
const url = new URL(value)
if (!['http:', 'https:'].includes(url.protocol)) {
return false
}
return LOCAL_HOSTS.has(url.hostname.toLowerCase())
} catch {
return false
}
}
export function isLikelyPreviewCandidate(value: string): boolean {
const trimmed = stripTrailingPunctuation(value.trim())
return trimmed.startsWith('file://') || HTML_EXT_RE.test(trimmed) || isLocalPreviewUrl(trimmed)
}
function collectPreviewMatches(text: string): PreviewCandidateMatch[] {
const matches: PreviewCandidateMatch[] = []
const collect = (index: number | undefined, raw: string, value = raw) => {
if (index === undefined) {
return
}
const candidate = stripTrailingPunctuation(value.trim())
if (!candidate || !isLikelyPreviewCandidate(candidate)) {
return
}
const offset = raw.indexOf(value)
const start = index + Math.max(0, offset)
matches.push({
end: start + candidate.length,
index: start,
value: candidate
})
}
for (const match of text.matchAll(URL_RE)) {
collect(match.index, match[0])
}
for (const match of text.matchAll(FILE_URL_RE)) {
collect(match.index, match[0])
}
for (const match of text.matchAll(POSIX_HTML_PATH_RE)) {
collect(match.index, match[0], match.groups?.path || '')
}
for (const match of text.matchAll(RELATIVE_HTML_PATH_RE)) {
collect(match.index, match[0], match.groups?.path || '')
}
return matches.sort((a, b) => a.index - b.index)
}
export function extractPreviewCandidates(text: string): string[] {
const candidates: string[] = []
const seen = new Set<string>()
const push = (value: string) => {
const candidate = stripTrailingPunctuation(value.trim())
if (!candidate || seen.has(candidate) || !isLikelyPreviewCandidate(candidate)) {
return
}
seen.add(candidate)
candidates.push(candidate)
}
for (const match of collectPreviewMatches(text)) {
push(match.value)
}
return candidates
}
export function stripPreviewTargets(text: string): string {
const matches = collectPreviewMatches(text)
let cursor = 0
let stripped = ''
for (const match of matches) {
if (match.index < cursor) {
continue
}
const lineStart = text.lastIndexOf('\n', Math.max(0, match.index - 1)) + 1
const nextLineBreak = text.indexOf('\n', match.end)
const lineEnd = nextLineBreak === -1 ? text.length : nextLineBreak + 1
const beforeOnLine = text.slice(lineStart, match.index)
const afterOnLine = text.slice(match.end, nextLineBreak === -1 ? text.length : nextLineBreak)
if (lineStart >= cursor && !beforeOnLine.trim() && !afterOnLine.trim()) {
stripped += text.slice(cursor, lineStart)
cursor = lineEnd
continue
}
stripped += text.slice(cursor, match.index)
cursor = match.end
}
stripped += text.slice(cursor)
return stripped
.replace(PREVIEW_MARKDOWN_RE, '')
.replace(/[ \t]+\n/g, '\n')
.replace(/\n{3,}/g, '\n\n')
.trim()
}
export function extractPreviewTargets(text: string): string[] {
const targets = extractPreviewCandidates(text)
const seen = new Set(targets)
for (const match of text.matchAll(PREVIEW_MARKDOWN_RE)) {
const target = previewTargetFromMarkdownHref(match.groups?.href)
if (target && !seen.has(target)) {
seen.add(target)
targets.push(target)
}
}
return targets
}
export function previewMarkdownHref(target: string): string {
return `#preview/${encodeURIComponent(target)}`
}
export function previewTargetFromMarkdownHref(href?: string): string | null {
if (!href?.startsWith('#preview:') && !href?.startsWith('#preview/')) {
return null
}
try {
return decodeURIComponent(href.slice('#preview'.length + 1))
} catch {
return null
}
}
export function previewName(target: string): string {
try {
const url = new URL(target)
if (url.protocol === 'file:') {
return decodeURIComponent(url.pathname).split(/[\\/]/).filter(Boolean).pop() || target
}
const file = url.pathname.split('/').filter(Boolean).pop()
return file || url.host
} catch {
return target.split(/[\\/]/).filter(Boolean).pop() || target
}
}
export function previewDisplayLabel(target: string): string {
const escaped = previewName(target).replace(/[[\]\\]/g, '\\$&')
return `Preview: ${escaped}`
}
function previewLink(value: string): string {
return `[${previewDisplayLabel(value)}](${previewMarkdownHref(value)})`
}
export function renderPreviewTargets(text: string): string {
const matches = collectPreviewMatches(text)
let cursor = 0
let rendered = ''
const seen = new Set<string>()
for (const match of matches) {
if (match.index < cursor || seen.has(match.value)) {
continue
}
rendered += text.slice(cursor, match.index)
rendered += previewLink(match.value)
cursor = match.end
seen.add(match.value)
}
return rendered + text.slice(cursor)
}