fix(desktop): keep generated images in the tool slot, not inline
The image-generate tool showed a placeholder, then the model echoed a (often different) image inline in its prose — a second, jarring copy in the wrong place, dimmed as tool scaffolding, with a misplaced download button. Now the generated image lives only in the tool slot: - Strip every embedded image/media link from the assistant prose of a message that produced an image (the model frequently restates the remote URL while the result holds the local path), preserving the agent's words. Applied on hydration, live deltas, and completion. - One stable frame sized from the aspect_ratio arg up front, so the diffusion placeholder and the decoded image share the same box and crossfade with no layout shift; the box derives its height from the true ratio on load (no letterboxing). - Exempt generated images from the tool-block dim-until-hover rule. - Extract a shared useImageDownload hook + ImageLightbox so the tool image and markdown images share one implementation.
This commit is contained in:
@@ -16,6 +16,11 @@ import {
|
|||||||
} from '@/lib/chat-messages'
|
} from '@/lib/chat-messages'
|
||||||
import { coerceGatewayText, coerceThinkingText, normalizePersonalityValue } from '@/lib/chat-runtime'
|
import { coerceGatewayText, coerceThinkingText, normalizePersonalityValue } from '@/lib/chat-runtime'
|
||||||
import { gatewayEventRequiresSessionId } from '@/lib/gateway-events'
|
import { gatewayEventRequiresSessionId } from '@/lib/gateway-events'
|
||||||
|
import {
|
||||||
|
dedupeGeneratedImageEchoesInParts,
|
||||||
|
generatedImageEchoSources,
|
||||||
|
stripGeneratedImageEchoes
|
||||||
|
} from '@/lib/generated-images'
|
||||||
import { triggerHaptic } from '@/lib/haptics'
|
import { triggerHaptic } from '@/lib/haptics'
|
||||||
import { isProviderSetupErrorMessage } from '@/lib/provider-setup-errors'
|
import { isProviderSetupErrorMessage } from '@/lib/provider-setup-errors'
|
||||||
import { parseTodos } from '@/lib/todos'
|
import { parseTodos } from '@/lib/todos'
|
||||||
@@ -343,7 +348,7 @@ export function useMessageStream({
|
|||||||
if (queued.assistant) {
|
if (queued.assistant) {
|
||||||
mutateStream(
|
mutateStream(
|
||||||
id,
|
id,
|
||||||
parts => appendAssistantTextPart(parts, queued.assistant),
|
parts => dedupeGeneratedImageEchoesInParts(appendAssistantTextPart(parts, queued.assistant)),
|
||||||
() => [assistantTextPart(queued.assistant)]
|
() => [assistantTextPart(queued.assistant)]
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -507,7 +512,7 @@ export function useMessageStream({
|
|||||||
|
|
||||||
mutateStream(
|
mutateStream(
|
||||||
sessionId,
|
sessionId,
|
||||||
parts => upsertToolPart(parts, payload, phase),
|
parts => dedupeGeneratedImageEchoesInParts(upsertToolPart(parts, payload, phase)),
|
||||||
() => upsertToolPart([], payload, phase),
|
() => upsertToolPart([], payload, phase),
|
||||||
{ pending: m => phase !== 'complete' || (m.pending ?? false) }
|
{ pending: m => phase !== 'complete' || (m.pending ?? false) }
|
||||||
)
|
)
|
||||||
@@ -540,9 +545,11 @@ export function useMessageStream({
|
|||||||
const finalText = renderMediaTags(text).trim()
|
const finalText = renderMediaTags(text).trim()
|
||||||
const completionError = completionErrorText(finalText)
|
const completionError = completionErrorText(finalText)
|
||||||
const normalize = (value: string) => value.replace(/\s+/g, ' ').trim()
|
const normalize = (value: string) => value.replace(/\s+/g, ' ').trim()
|
||||||
const dedupeReference = normalize(finalText)
|
|
||||||
|
|
||||||
const replaceTextPart = (parts: ChatMessagePart[]) => {
|
const replaceTextPart = (parts: ChatMessagePart[]) => {
|
||||||
|
const visibleFinalText = stripGeneratedImageEchoes(finalText, generatedImageEchoSources(parts)).trim()
|
||||||
|
const dedupeReference = normalize(visibleFinalText)
|
||||||
|
|
||||||
const kept = parts.filter(part => {
|
const kept = parts.filter(part => {
|
||||||
if (part.type === 'text') {
|
if (part.type === 'text') {
|
||||||
return false
|
return false
|
||||||
@@ -557,7 +564,7 @@ export function useMessageStream({
|
|||||||
return !(r && (dedupeReference.startsWith(r) || r.startsWith(dedupeReference)))
|
return !(r && (dedupeReference.startsWith(r) || r.startsWith(dedupeReference)))
|
||||||
})
|
})
|
||||||
|
|
||||||
return finalText ? [...kept, assistantTextPart(finalText)] : kept
|
return visibleFinalText ? [...kept, assistantTextPart(visibleFinalText)] : kept
|
||||||
}
|
}
|
||||||
|
|
||||||
const completeMessage = (message: ChatMessage): ChatMessage =>
|
const completeMessage = (message: ChatMessage): ChatMessage =>
|
||||||
|
|||||||
@@ -216,6 +216,32 @@ function assistantTodoMessage(
|
|||||||
} as ThreadMessage
|
} as ThreadMessage
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function assistantImageMessage(running = false): ThreadMessage {
|
||||||
|
return {
|
||||||
|
id: `assistant-image-${running ? 'running' : 'done'}`,
|
||||||
|
role: 'assistant',
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: 'tool-call',
|
||||||
|
toolCallId: 'image-1',
|
||||||
|
toolName: 'image_generate',
|
||||||
|
args: { prompt: 'draw a cat' },
|
||||||
|
argsText: JSON.stringify({ prompt: 'draw a cat' }),
|
||||||
|
...(running ? {} : { result: { image: 'https://cdn.example/cat.png', success: true } })
|
||||||
|
}
|
||||||
|
],
|
||||||
|
status: running ? { type: 'running' } : { type: 'complete', reason: 'stop' },
|
||||||
|
createdAt,
|
||||||
|
metadata: {
|
||||||
|
unstable_state: null,
|
||||||
|
unstable_annotations: [],
|
||||||
|
unstable_data: [],
|
||||||
|
steps: [],
|
||||||
|
custom: {}
|
||||||
|
}
|
||||||
|
} as ThreadMessage
|
||||||
|
}
|
||||||
|
|
||||||
function StreamingHarness() {
|
function StreamingHarness() {
|
||||||
const [messages, setMessages] = useState<ThreadMessage[]>([userMessage()])
|
const [messages, setMessages] = useState<ThreadMessage[]>([userMessage()])
|
||||||
const [isRunning, setIsRunning] = useState(true)
|
const [isRunning, setIsRunning] = useState(true)
|
||||||
@@ -640,14 +666,19 @@ describe('assistant-ui streaming renderer', () => {
|
|||||||
it('renders an incomplete streaming reasoning fenced code block as a code card', async () => {
|
it('renders an incomplete streaming reasoning fenced code block as a code card', async () => {
|
||||||
const { container } = render(<RunningReasoningHarness />)
|
const { container } = render(<RunningReasoningHarness />)
|
||||||
const ui = within(container)
|
const ui = within(container)
|
||||||
|
const thinkingToggle = ui.getByRole('button', { name: /thinking/i })
|
||||||
|
|
||||||
fireEvent.click(ui.getByRole('button', { name: /thinking/i }))
|
if (thinkingToggle.getAttribute('aria-expanded') !== 'true') {
|
||||||
|
fireEvent.click(thinkingToggle)
|
||||||
|
}
|
||||||
|
|
||||||
await waitFor(() => {
|
await waitFor(() => {
|
||||||
expect(container.querySelector('[data-slot="code-card"]')).toBeTruthy()
|
expect(container.querySelector('[data-slot="code-card"]')).toBeTruthy()
|
||||||
})
|
})
|
||||||
|
|
||||||
expect(container.querySelector('[data-slot="aui_reasoning-text"]')?.textContent).toContain('const answer = 42')
|
await waitFor(() => {
|
||||||
|
expect(container.querySelector('[data-slot="aui_reasoning-text"]')?.textContent).toContain('const answer = 42')
|
||||||
|
})
|
||||||
expect(container.textContent).not.toContain('```ts')
|
expect(container.textContent).not.toContain('```ts')
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -700,4 +731,16 @@ describe('assistant-ui streaming renderer', () => {
|
|||||||
|
|
||||||
expect(container.querySelector('[data-slot="aui_todo-hoisted"]')).toBeNull()
|
expect(container.querySelector('[data-slot="aui_todo-hoisted"]')).toBeNull()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('renders completed image generation results in the tool slot', async () => {
|
||||||
|
const { container } = render(<MessageHarness message={assistantImageMessage()} />)
|
||||||
|
|
||||||
|
await waitFor(() => {
|
||||||
|
expect(screen.getByRole('img', { name: 'Generated image' }).getAttribute('src')).toBe(
|
||||||
|
'https://cdn.example/cat.png'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
expect(container.querySelector('[data-slot="aui_generated-image"]')).toBeTruthy()
|
||||||
|
expect(screen.queryByRole('status', { name: /rendering image/i })).toBeNull()
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -70,8 +70,7 @@ import { UserMessageText } from '@/components/assistant-ui/user-message-text'
|
|||||||
import { useElapsedSeconds } from '@/components/chat/activity-timer'
|
import { useElapsedSeconds } from '@/components/chat/activity-timer'
|
||||||
import { ActivityTimerText } from '@/components/chat/activity-timer-text'
|
import { ActivityTimerText } from '@/components/chat/activity-timer-text'
|
||||||
import { DisclosureRow } from '@/components/chat/disclosure-row'
|
import { DisclosureRow } from '@/components/chat/disclosure-row'
|
||||||
import { GeneratedImageProvider, useGeneratedImageContext } from '@/components/chat/generated-image-context'
|
import { GeneratedImage } from '@/components/chat/generated-image-result'
|
||||||
import { ImageGenerationPlaceholder } from '@/components/chat/image-generation-placeholder'
|
|
||||||
import { Intro, type IntroProps } from '@/components/chat/intro'
|
import { Intro, type IntroProps } from '@/components/chat/intro'
|
||||||
import { PreviewAttachment } from '@/components/chat/preview-attachment'
|
import { PreviewAttachment } from '@/components/chat/preview-attachment'
|
||||||
import { Codicon } from '@/components/ui/codicon'
|
import { Codicon } from '@/components/ui/codicon'
|
||||||
@@ -200,18 +199,16 @@ export const Thread: FC<{
|
|||||||
) : undefined
|
) : undefined
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<GeneratedImageProvider>
|
<div className="relative grid h-full min-h-0 max-w-full grid-rows-[minmax(0,1fr)] overflow-hidden bg-transparent contain-[layout_paint]">
|
||||||
<div className="relative grid h-full min-h-0 max-w-full grid-rows-[minmax(0,1fr)] overflow-hidden bg-transparent contain-[layout_paint]">
|
<VirtualizedThread
|
||||||
<VirtualizedThread
|
clampToComposer={clampToComposer}
|
||||||
clampToComposer={clampToComposer}
|
components={messageComponents}
|
||||||
components={messageComponents}
|
emptyPlaceholder={emptyPlaceholder}
|
||||||
emptyPlaceholder={emptyPlaceholder}
|
loadingIndicator={loading === 'response' ? <ResponseLoadingIndicator /> : null}
|
||||||
loadingIndicator={loading === 'response' ? <ResponseLoadingIndicator /> : null}
|
sessionKey={sessionKey}
|
||||||
sessionKey={sessionKey}
|
/>
|
||||||
/>
|
{loading === 'session' && <CenteredThreadSpinner />}
|
||||||
{loading === 'session' && <CenteredThreadSpinner />}
|
</div>
|
||||||
</div>
|
|
||||||
</GeneratedImageProvider>
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -404,21 +401,12 @@ const StreamStallIndicator: FC = () => {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
const ImageGenerateTool: FC<ToolCallMessagePartProps> = ({ result }) => {
|
const ImageGenerateTool: FC<ToolCallMessagePartProps> = ({ args, result }) => {
|
||||||
const generatedImage = useGeneratedImageContext()
|
const aspectRatio = typeof args?.aspect_ratio === 'string' ? args.aspect_ratio : undefined
|
||||||
const running = result === undefined
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
generatedImage?.setPending(running)
|
|
||||||
}, [generatedImage, running])
|
|
||||||
|
|
||||||
if (!running) {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="mt-1.5">
|
<div className="mt-1.5">
|
||||||
<ImageGenerationPlaceholder />
|
<GeneratedImage aspectRatio={aspectRatio} result={result} />
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,19 +0,0 @@
|
|||||||
'use client'
|
|
||||||
|
|
||||||
import { createContext, type ReactNode, useContext, useMemo, useState } from 'react'
|
|
||||||
|
|
||||||
type Value = {
|
|
||||||
isPending: boolean
|
|
||||||
setPending: (pending: boolean) => void
|
|
||||||
}
|
|
||||||
|
|
||||||
const Ctx = createContext<Value | null>(null)
|
|
||||||
|
|
||||||
export function GeneratedImageProvider({ children }: { children: ReactNode }) {
|
|
||||||
const [isPending, setPending] = useState(false)
|
|
||||||
const value = useMemo(() => ({ isPending, setPending }), [isPending])
|
|
||||||
|
|
||||||
return <Ctx.Provider value={value}>{children}</Ctx.Provider>
|
|
||||||
}
|
|
||||||
|
|
||||||
export const useGeneratedImageContext = () => useContext(Ctx)
|
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
'use client'
|
||||||
|
|
||||||
|
import { type FC, useEffect, useState } from 'react'
|
||||||
|
|
||||||
|
import { DiffusionCanvas } from '@/components/chat/image-generation-placeholder'
|
||||||
|
import { ImageActionButton, ImageLightbox } from '@/components/chat/zoomable-image'
|
||||||
|
import { useImageDownload } from '@/hooks/use-image-download'
|
||||||
|
import { useI18n } from '@/i18n'
|
||||||
|
import { generatedImageFromResult } from '@/lib/generated-images'
|
||||||
|
import { filePathFromMediaPath, gatewayMediaDataUrl, isRemoteGateway, mediaExternalUrl, mediaName } from '@/lib/media'
|
||||||
|
import { cn } from '@/lib/utils'
|
||||||
|
|
||||||
|
// Aspect hint from the tool args sizes the frame *before* the image loads, so
|
||||||
|
// the placeholder and the resolved image occupy the same box — no layout shift.
|
||||||
|
const ASPECT_HINTS: Record<string, number> = {
|
||||||
|
landscape: 16 / 9,
|
||||||
|
square: 1,
|
||||||
|
portrait: 9 / 16
|
||||||
|
}
|
||||||
|
|
||||||
|
function hintedRatio(aspectRatio?: string): number {
|
||||||
|
return ASPECT_HINTS[String(aspectRatio ?? '').toLowerCase().trim()] ?? ASPECT_HINTS.landscape
|
||||||
|
}
|
||||||
|
|
||||||
|
function isInlineSrc(path: string): boolean {
|
||||||
|
return /^(?:https?|data):/i.test(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveImageSrc(path: string): Promise<string> {
|
||||||
|
if (isInlineSrc(path)) {
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
if (window.hermesDesktop && isRemoteGateway()) {
|
||||||
|
return gatewayMediaDataUrl(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!window.hermesDesktop?.readFileDataUrl) {
|
||||||
|
return mediaExternalUrl(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
return window.hermesDesktop.readFileDataUrl(filePathFromMediaPath(path))
|
||||||
|
}
|
||||||
|
|
||||||
|
export const GeneratedImage: FC<{ aspectRatio?: string; result?: unknown }> = ({ aspectRatio, result }) => {
|
||||||
|
const { t } = useI18n()
|
||||||
|
const copy = t.desktop
|
||||||
|
const image = result === undefined ? null : generatedImageFromResult(result)
|
||||||
|
const pending = result === undefined
|
||||||
|
|
||||||
|
const [ratio, setRatio] = useState(() => hintedRatio(aspectRatio))
|
||||||
|
const [src, setSrc] = useState(() => (image && isInlineSrc(image) ? image : ''))
|
||||||
|
const [loaded, setLoaded] = useState(false)
|
||||||
|
const [canvasGone, setCanvasGone] = useState(false)
|
||||||
|
const [failed, setFailed] = useState(false)
|
||||||
|
const [lightboxOpen, setLightboxOpen] = useState(false)
|
||||||
|
const { download, saving } = useImageDownload(src)
|
||||||
|
|
||||||
|
useEffect(() => setRatio(hintedRatio(aspectRatio)), [aspectRatio])
|
||||||
|
|
||||||
|
// Resolve the deliverable path (local read / gateway proxy / remote URL). The
|
||||||
|
// <img> stays mounted under the placeholder and only fades in once it decodes,
|
||||||
|
// so the frame keeps its hinted size and never jumps.
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false
|
||||||
|
setFailed(false)
|
||||||
|
setLoaded(false)
|
||||||
|
setCanvasGone(false)
|
||||||
|
setSrc(image && isInlineSrc(image) ? image : '')
|
||||||
|
|
||||||
|
if (!image || isInlineSrc(image)) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
void resolveImageSrc(image)
|
||||||
|
.then(resolved => !cancelled && setSrc(resolved))
|
||||||
|
.catch(() => !cancelled && setFailed(true))
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
cancelled = true
|
||||||
|
}
|
||||||
|
}, [image])
|
||||||
|
|
||||||
|
// Completed but no usable image (generation failed): the agent's prose carries
|
||||||
|
// the explanation, so render nothing here.
|
||||||
|
if (!pending && !image) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
if (failed && image) {
|
||||||
|
return (
|
||||||
|
<a
|
||||||
|
className="mt-2 inline-block font-semibold text-foreground underline underline-offset-4 decoration-current/20 wrap-anywhere"
|
||||||
|
href="#"
|
||||||
|
onClick={event => {
|
||||||
|
event.preventDefault()
|
||||||
|
void window.hermesDesktop?.openExternal(mediaExternalUrl(image))
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{copy.openImage}: {mediaName(image)}
|
||||||
|
</a>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<span
|
||||||
|
aria-label={pending ? t.assistant.tool.renderingImage : undefined}
|
||||||
|
aria-live={pending ? 'polite' : undefined}
|
||||||
|
className="group/image relative block max-w-full overflow-hidden rounded-2xl transition-[width,height] duration-300 ease-out"
|
||||||
|
data-slot="aui_generated-image"
|
||||||
|
role={pending ? 'status' : undefined}
|
||||||
|
style={{
|
||||||
|
aspectRatio: ratio,
|
||||||
|
// Width is capped so the derived height (width / ratio) never exceeds
|
||||||
|
// --image-preview-height; the box then matches the image exactly with
|
||||||
|
// no letterboxing.
|
||||||
|
width: `min(calc(var(--image-preview-height) * ${ratio}), var(--image-preview-max-width), 100%)`
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{!canvasGone && (
|
||||||
|
<div
|
||||||
|
className={cn('absolute inset-0 transition-opacity duration-500 ease-out', loaded && 'opacity-0')}
|
||||||
|
onTransitionEnd={() => loaded && setCanvasGone(true)}
|
||||||
|
>
|
||||||
|
<DiffusionCanvas />
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{src && (
|
||||||
|
<button
|
||||||
|
className="absolute inset-0 block size-full cursor-zoom-in"
|
||||||
|
onClick={() => setLightboxOpen(true)}
|
||||||
|
title={copy.openImage}
|
||||||
|
type="button"
|
||||||
|
>
|
||||||
|
<img
|
||||||
|
alt="Generated image"
|
||||||
|
className={cn(
|
||||||
|
'absolute inset-0 size-full object-contain opacity-0 transition-opacity duration-500 ease-out',
|
||||||
|
loaded && 'opacity-100'
|
||||||
|
)}
|
||||||
|
draggable={false}
|
||||||
|
onError={() => setFailed(true)}
|
||||||
|
onLoad={event => {
|
||||||
|
const { naturalHeight, naturalWidth } = event.currentTarget
|
||||||
|
|
||||||
|
if (naturalWidth && naturalHeight) {
|
||||||
|
setRatio(naturalWidth / naturalHeight)
|
||||||
|
}
|
||||||
|
|
||||||
|
setLoaded(true)
|
||||||
|
}}
|
||||||
|
src={src}
|
||||||
|
/>
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
{loaded && src && (
|
||||||
|
<ImageActionButton className="group-hover/image:opacity-100" copy={copy} onClick={download} saving={saving} />
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
|
{src && (
|
||||||
|
<ImageLightbox
|
||||||
|
alt="Generated image"
|
||||||
|
copy={copy}
|
||||||
|
onClick={download}
|
||||||
|
onOpenChange={setLightboxOpen}
|
||||||
|
open={lightboxOpen}
|
||||||
|
saving={saving}
|
||||||
|
src={src}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
import { type FC, useCallback, useEffect, useRef } from 'react'
|
import { type FC, useCallback, useEffect, useRef } from 'react'
|
||||||
|
|
||||||
import { useResizeObserver } from '@/hooks/use-resize-observer'
|
import { useResizeObserver } from '@/hooks/use-resize-observer'
|
||||||
import { useI18n } from '@/i18n'
|
|
||||||
|
|
||||||
type Rgb = { r: number; g: number; b: number }
|
type Rgb = { r: number; g: number; b: number }
|
||||||
|
|
||||||
@@ -241,7 +240,7 @@ const drawAsciiDiffusion = (
|
|||||||
ctx.fillRect(0, 0, width, height)
|
ctx.fillRect(0, 0, width, height)
|
||||||
}
|
}
|
||||||
|
|
||||||
const DiffusionCanvas: FC = () => {
|
export const DiffusionCanvas: FC = () => {
|
||||||
const canvasRef = useRef<HTMLCanvasElement | null>(null)
|
const canvasRef = useRef<HTMLCanvasElement | null>(null)
|
||||||
const sizeRef = useRef({ width: 0, height: 0 })
|
const sizeRef = useRef({ width: 0, height: 0 })
|
||||||
const themeRef = useRef<Theme>(FALLBACKS)
|
const themeRef = useRef<Theme>(FALLBACKS)
|
||||||
@@ -305,15 +304,3 @@ const DiffusionCanvas: FC = () => {
|
|||||||
|
|
||||||
return <canvas className="absolute inset-0 h-full w-full" ref={canvasRef} />
|
return <canvas className="absolute inset-0 h-full w-full" ref={canvasRef} />
|
||||||
}
|
}
|
||||||
|
|
||||||
export const ImageGenerationPlaceholder: FC = () => {
|
|
||||||
const { t } = useI18n()
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div aria-label={t.assistant.tool.renderingImage} aria-live="polite" className="w-full max-w-136 self-start" role="status">
|
|
||||||
<div className="relative h-(--image-preview-height) overflow-hidden rounded-4xl border border-border/55 shadow-[inset_0_0.0625rem_0_color-mix(in_srgb,white_45%,transparent),inset_0_0_0_0.0625rem_color-mix(in_srgb,var(--dt-border)_34%,transparent),inset_0_-0.75rem_1.75rem_color-mix(in_srgb,var(--dt-primary)_5%,transparent)]">
|
|
||||||
<DiffusionCanvas />
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -3,55 +3,17 @@
|
|||||||
import { type ComponentProps, useState } from 'react'
|
import { type ComponentProps, useState } from 'react'
|
||||||
|
|
||||||
import { Dialog, DialogContent } from '@/components/ui/dialog'
|
import { Dialog, DialogContent } from '@/components/ui/dialog'
|
||||||
|
import { useImageDownload } from '@/hooks/use-image-download'
|
||||||
import { useI18n } from '@/i18n'
|
import { useI18n } from '@/i18n'
|
||||||
import { Download } from '@/lib/icons'
|
import { Download } from '@/lib/icons'
|
||||||
import { cn } from '@/lib/utils'
|
import { cn } from '@/lib/utils'
|
||||||
import { notify, notifyError } from '@/store/notifications'
|
|
||||||
|
|
||||||
function imageFilename(src?: string): string {
|
|
||||||
if (!src) {
|
|
||||||
return 'image'
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const { pathname } = new URL(src, window.location.href)
|
|
||||||
|
|
||||||
return pathname.split('/').filter(Boolean).pop() || 'image'
|
|
||||||
} catch {
|
|
||||||
return src.split(/[\\/]/).filter(Boolean).pop() || 'image'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function isMissingIpcHandler(error: unknown): boolean {
|
|
||||||
const message = error instanceof Error ? error.message : typeof error === 'string' ? error : ''
|
|
||||||
|
|
||||||
return message.includes("No handler registered for 'hermes:saveImageFromUrl'")
|
|
||||||
}
|
|
||||||
|
|
||||||
async function startBrowserDownload(src: string) {
|
|
||||||
const response = await fetch(src)
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Could not fetch image: ${response.status}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
const blobUrl = URL.createObjectURL(await response.blob())
|
|
||||||
const link = document.createElement('a')
|
|
||||||
link.href = blobUrl
|
|
||||||
link.download = imageFilename(src)
|
|
||||||
link.rel = 'noopener noreferrer'
|
|
||||||
document.body.appendChild(link)
|
|
||||||
link.click()
|
|
||||||
link.remove()
|
|
||||||
window.setTimeout(() => URL.revokeObjectURL(blobUrl), 30_000)
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ZoomableImageProps extends ComponentProps<'img'> {
|
export interface ZoomableImageProps extends ComponentProps<'img'> {
|
||||||
containerClassName?: string
|
containerClassName?: string
|
||||||
slot?: string
|
slot?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
interface ImageActionCopy {
|
export interface ImageActionCopy {
|
||||||
downloadImage: string
|
downloadImage: string
|
||||||
savingImage: string
|
savingImage: string
|
||||||
}
|
}
|
||||||
@@ -59,70 +21,10 @@ interface ImageActionCopy {
|
|||||||
export function ZoomableImage({ className, containerClassName, src, alt, slot, ...props }: ZoomableImageProps) {
|
export function ZoomableImage({ className, containerClassName, src, alt, slot, ...props }: ZoomableImageProps) {
|
||||||
const { t } = useI18n()
|
const { t } = useI18n()
|
||||||
const copy = t.desktop
|
const copy = t.desktop
|
||||||
const [saving, setSaving] = useState(false)
|
const { download, saving } = useImageDownload(src)
|
||||||
const [lightboxOpen, setLightboxOpen] = useState(false)
|
const [lightboxOpen, setLightboxOpen] = useState(false)
|
||||||
const canOpen = Boolean(src)
|
const canOpen = Boolean(src)
|
||||||
|
|
||||||
async function handleDownload() {
|
|
||||||
if (!src || saving) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
setSaving(true)
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (window.hermesDesktop?.saveImageFromUrl) {
|
|
||||||
const saved = await window.hermesDesktop.saveImageFromUrl(src)
|
|
||||||
|
|
||||||
if (saved) {
|
|
||||||
notify({ kind: 'success', title: copy.imageSaved, message: imageFilename(src) })
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
await startBrowserDownload(src)
|
|
||||||
} catch (error) {
|
|
||||||
if (isMissingIpcHandler(error)) {
|
|
||||||
try {
|
|
||||||
await startBrowserDownload(src)
|
|
||||||
notify({
|
|
||||||
kind: 'info',
|
|
||||||
title: copy.downloadStarted,
|
|
||||||
message: copy.restartToUseSaveImage
|
|
||||||
})
|
|
||||||
} catch (fallbackError) {
|
|
||||||
notifyError(fallbackError, copy.restartToSaveImages)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
notifyError(error, copy.imageDownloadFailed)
|
|
||||||
} finally {
|
|
||||||
setSaving(false)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const lightbox = src ? (
|
|
||||||
<Dialog onOpenChange={setLightboxOpen} open={lightboxOpen}>
|
|
||||||
<DialogContent
|
|
||||||
className="block w-auto max-h-[calc(100vh-12rem)] max-w-[calc(100vw-12rem)] overflow-visible border-0 bg-transparent p-0 shadow-none"
|
|
||||||
showCloseButton={false}
|
|
||||||
>
|
|
||||||
<div className="group/lightbox relative inline-block">
|
|
||||||
<img
|
|
||||||
alt={alt ?? ''}
|
|
||||||
className="block max-h-[calc(100vh-12rem)] max-w-[calc(100vw-12rem)] cursor-zoom-out select-auto rounded-lg object-contain shadow-2xl"
|
|
||||||
onClick={() => setLightboxOpen(false)}
|
|
||||||
src={src}
|
|
||||||
/>
|
|
||||||
<ImageActionButton copy={copy} onClick={handleDownload} saving={saving} variant="lightbox" />
|
|
||||||
</div>
|
|
||||||
</DialogContent>
|
|
||||||
</Dialog>
|
|
||||||
) : null
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
<span
|
<span
|
||||||
@@ -138,30 +40,79 @@ export function ZoomableImage({ className, containerClassName, src, alt, slot, .
|
|||||||
>
|
>
|
||||||
<img alt={alt ?? ''} className={className} src={src} {...props} />
|
<img alt={alt ?? ''} className={className} src={src} {...props} />
|
||||||
</button>
|
</button>
|
||||||
{src && <ImageActionButton copy={copy} onClick={handleDownload} saving={saving} variant="inline" />}
|
{src && (
|
||||||
|
<ImageActionButton className="group-hover/image:opacity-100" copy={copy} onClick={download} saving={saving} />
|
||||||
|
)}
|
||||||
</span>
|
</span>
|
||||||
{lightbox}
|
{src && (
|
||||||
|
<ImageLightbox
|
||||||
|
alt={alt}
|
||||||
|
copy={copy}
|
||||||
|
onClick={download}
|
||||||
|
onOpenChange={setLightboxOpen}
|
||||||
|
open={lightboxOpen}
|
||||||
|
saving={saving}
|
||||||
|
src={src}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
</>
|
</>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
function ImageActionButton({
|
export function ImageLightbox({
|
||||||
|
alt,
|
||||||
copy,
|
copy,
|
||||||
onClick,
|
onClick,
|
||||||
|
onOpenChange,
|
||||||
|
open,
|
||||||
saving,
|
saving,
|
||||||
variant
|
src
|
||||||
}: {
|
}: {
|
||||||
|
alt?: string
|
||||||
|
copy: ImageActionCopy
|
||||||
|
onClick: () => void
|
||||||
|
onOpenChange: (open: boolean) => void
|
||||||
|
open: boolean
|
||||||
|
saving: boolean
|
||||||
|
src: string
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<Dialog onOpenChange={onOpenChange} open={open}>
|
||||||
|
<DialogContent
|
||||||
|
className="block w-auto max-h-[calc(100vh-12rem)] max-w-[calc(100vw-12rem)] overflow-visible border-0 bg-transparent p-0 shadow-none"
|
||||||
|
showCloseButton={false}
|
||||||
|
>
|
||||||
|
<div className="group/lightbox relative inline-block">
|
||||||
|
<img
|
||||||
|
alt={alt ?? ''}
|
||||||
|
className="block max-h-[calc(100vh-12rem)] max-w-[calc(100vw-12rem)] cursor-zoom-out select-auto rounded-lg object-contain shadow-2xl"
|
||||||
|
onClick={() => onOpenChange(false)}
|
||||||
|
src={src}
|
||||||
|
/>
|
||||||
|
<ImageActionButton className="group-hover/lightbox:opacity-100" copy={copy} onClick={onClick} saving={saving} />
|
||||||
|
</div>
|
||||||
|
</DialogContent>
|
||||||
|
</Dialog>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ImageActionButton({
|
||||||
|
className,
|
||||||
|
copy,
|
||||||
|
onClick,
|
||||||
|
saving
|
||||||
|
}: {
|
||||||
|
className?: string
|
||||||
copy: ImageActionCopy
|
copy: ImageActionCopy
|
||||||
onClick: () => void
|
onClick: () => void
|
||||||
saving: boolean
|
saving: boolean
|
||||||
variant: 'inline' | 'lightbox'
|
|
||||||
}) {
|
}) {
|
||||||
return (
|
return (
|
||||||
<button
|
<button
|
||||||
aria-label={saving ? copy.savingImage : copy.downloadImage}
|
aria-label={saving ? copy.savingImage : copy.downloadImage}
|
||||||
className={cn(
|
className={cn(
|
||||||
'absolute right-2 top-2 grid size-8 place-items-center rounded-full border border-border/70 bg-background/80 text-muted-foreground opacity-0 shadow-sm backdrop-blur transition-opacity hover:bg-accent hover:text-foreground focus-visible:opacity-100 disabled:opacity-50',
|
'absolute right-2 top-2 grid size-8 place-items-center rounded-full border border-border/70 bg-background/80 text-muted-foreground opacity-0 shadow-sm backdrop-blur transition-opacity hover:bg-accent hover:text-foreground focus-visible:opacity-100 disabled:opacity-50',
|
||||||
variant === 'inline' ? 'group-hover/image:opacity-100' : 'group-hover/lightbox:opacity-100'
|
className
|
||||||
)}
|
)}
|
||||||
disabled={saving}
|
disabled={saving}
|
||||||
onClick={event => {
|
onClick={event => {
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
import { useCallback, useState } from 'react'
|
||||||
|
|
||||||
|
import { useI18n } from '@/i18n'
|
||||||
|
import { notify, notifyError } from '@/store/notifications'
|
||||||
|
|
||||||
|
export function imageFilename(src?: string): string {
|
||||||
|
if (!src) {
|
||||||
|
return 'image'
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return new URL(src, window.location.href).pathname.split('/').filter(Boolean).pop() || 'image'
|
||||||
|
} catch {
|
||||||
|
return src.split(/[\\/]/).filter(Boolean).pop() || 'image'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isMissingIpcHandler(error: unknown): boolean {
|
||||||
|
const message = error instanceof Error ? error.message : typeof error === 'string' ? error : ''
|
||||||
|
|
||||||
|
return message.includes("No handler registered for 'hermes:saveImageFromUrl'")
|
||||||
|
}
|
||||||
|
|
||||||
|
async function startBrowserDownload(src: string) {
|
||||||
|
const response = await fetch(src)
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`Could not fetch image: ${response.status}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const blobUrl = URL.createObjectURL(await response.blob())
|
||||||
|
const link = document.createElement('a')
|
||||||
|
link.href = blobUrl
|
||||||
|
link.download = imageFilename(src)
|
||||||
|
link.rel = 'noopener noreferrer'
|
||||||
|
document.body.appendChild(link)
|
||||||
|
link.click()
|
||||||
|
link.remove()
|
||||||
|
window.setTimeout(() => URL.revokeObjectURL(blobUrl), 30_000)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Save an image to disk via the desktop IPC bridge, falling back to a browser
|
||||||
|
* download when the handler is unavailable (older shell / web preview). */
|
||||||
|
export function useImageDownload(src?: string) {
|
||||||
|
const { t } = useI18n()
|
||||||
|
const copy = t.desktop
|
||||||
|
const [saving, setSaving] = useState(false)
|
||||||
|
|
||||||
|
const download = useCallback(async () => {
|
||||||
|
if (!src || saving) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
setSaving(true)
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (window.hermesDesktop?.saveImageFromUrl) {
|
||||||
|
if (await window.hermesDesktop.saveImageFromUrl(src)) {
|
||||||
|
notify({ kind: 'success', title: copy.imageSaved, message: imageFilename(src) })
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
await startBrowserDownload(src)
|
||||||
|
} catch (error) {
|
||||||
|
if (isMissingIpcHandler(error)) {
|
||||||
|
try {
|
||||||
|
await startBrowserDownload(src)
|
||||||
|
notify({ kind: 'info', title: copy.downloadStarted, message: copy.restartToUseSaveImage })
|
||||||
|
} catch (fallbackError) {
|
||||||
|
notifyError(fallbackError, copy.restartToSaveImages)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
notifyError(error, copy.imageDownloadFailed)
|
||||||
|
} finally {
|
||||||
|
setSaving(false)
|
||||||
|
}
|
||||||
|
}, [copy, saving, src])
|
||||||
|
|
||||||
|
return { download, saving }
|
||||||
|
}
|
||||||
@@ -95,6 +95,38 @@ describe('toChatMessages', () => {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('keeps the generated image on the tool row while preserving agent prose', () => {
|
||||||
|
const [message] = toChatMessages([
|
||||||
|
{
|
||||||
|
content: '',
|
||||||
|
role: 'assistant',
|
||||||
|
timestamp: 1,
|
||||||
|
tool_calls: [{ id: 'img-1', function: { name: 'image_generate', arguments: '{"prompt":"draw a cat"}' } }]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
content: '{"success":true,"image":"https://cdn.example/cat.png"}',
|
||||||
|
role: 'tool',
|
||||||
|
timestamp: 2,
|
||||||
|
tool_call_id: 'img-1',
|
||||||
|
tool_name: 'image_generate'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
content: 'Here you go.\n\n',
|
||||||
|
role: 'assistant',
|
||||||
|
timestamp: 3
|
||||||
|
}
|
||||||
|
])
|
||||||
|
|
||||||
|
const toolPart = message.parts.find(
|
||||||
|
(part): part is Extract<ChatMessagePart, { type: 'tool-call' }> =>
|
||||||
|
part.type === 'tool-call' && part.toolName === 'image_generate'
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(toolPart?.result).toMatchObject({ image: 'https://cdn.example/cat.png', success: true })
|
||||||
|
// The duplicated image is stripped, but the agent's words survive.
|
||||||
|
expect(chatMessageText(message)).toBe('Here you go.')
|
||||||
|
})
|
||||||
|
|
||||||
it('coerces non-string message content without throwing', () => {
|
it('coerces non-string message content without throwing', () => {
|
||||||
const [message] = toChatMessages([
|
const [message] = toChatMessages([
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import type { ThreadMessageLike } from '@assistant-ui/react'
|
import type { ThreadMessageLike } from '@assistant-ui/react'
|
||||||
|
|
||||||
|
import { dedupeGeneratedImageEchoesInParts } from '@/lib/generated-images'
|
||||||
import { mediaDisplayLabel, mediaMarkdownHref } from '@/lib/media'
|
import { mediaDisplayLabel, mediaMarkdownHref } from '@/lib/media'
|
||||||
import { parseTodos } from '@/lib/todos'
|
import { parseTodos } from '@/lib/todos'
|
||||||
import type { SessionMessage, UsageStats } from '@/types/hermes'
|
import type { SessionMessage, UsageStats } from '@/types/hermes'
|
||||||
@@ -811,8 +812,12 @@ export function toChatMessages(messages: SessionMessage[]): ChatMessage[] {
|
|||||||
})
|
})
|
||||||
flushPendingTools(messages.length)
|
flushPendingTools(messages.length)
|
||||||
|
|
||||||
|
const withoutGeneratedImageEchoes = result.map(message =>
|
||||||
|
message.role === 'assistant' ? { ...message, parts: dedupeGeneratedImageEchoesInParts(message.parts) } : message
|
||||||
|
)
|
||||||
|
|
||||||
return withUniqueToolCallIds(
|
return withUniqueToolCallIds(
|
||||||
result.filter(m => chatMessageText(m).trim() || m.parts.some(part => part.type !== 'text'))
|
withoutGeneratedImageEchoes.filter(m => chatMessageText(m).trim() || m.parts.some(part => part.type !== 'text'))
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
import { describe, expect, it } from 'vitest'
|
||||||
|
|
||||||
|
import {
|
||||||
|
dedupeGeneratedImageEchoesInParts,
|
||||||
|
generatedImageEchoSources,
|
||||||
|
generatedImageFromResult,
|
||||||
|
stripGeneratedImageEchoes
|
||||||
|
} from './generated-images'
|
||||||
|
|
||||||
|
describe('generatedImageFromResult', () => {
|
||||||
|
it('prefers the host-visible image path', () => {
|
||||||
|
expect(
|
||||||
|
generatedImageFromResult({
|
||||||
|
agent_visible_image: '/container/cache/cat.png',
|
||||||
|
host_image: '/Users/me/.hermes/cache/images/cat.png',
|
||||||
|
image: '/Users/me/.hermes/cache/images/cat.png',
|
||||||
|
success: true
|
||||||
|
})
|
||||||
|
).toBe('/Users/me/.hermes/cache/images/cat.png')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('ignores failed image generation results', () => {
|
||||||
|
expect(generatedImageFromResult({ image: 'https://cdn.example/cat.png', success: false })).toBeNull()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('stripGeneratedImageEchoes', () => {
|
||||||
|
it('removes repeated generated image markdown without removing prose', () => {
|
||||||
|
expect(
|
||||||
|
stripGeneratedImageEchoes('Here you go.\n\n', [
|
||||||
|
'https://cdn.example/cat.png'
|
||||||
|
])
|
||||||
|
).toBe('Here you go.')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('removes media links for generated local image paths', () => {
|
||||||
|
expect(
|
||||||
|
stripGeneratedImageEchoes('Saved image: [Image: cat.png](#media:%2Ftmp%2Fcat.png)', ['/tmp/cat.png'])
|
||||||
|
).toBe('Saved image:')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('generatedImageEchoSources', () => {
|
||||||
|
it('collects every path variant the model might restate', () => {
|
||||||
|
expect(
|
||||||
|
generatedImageEchoSources([
|
||||||
|
{
|
||||||
|
result: { agent_visible_image: '/sandbox/cat.png', host_image: '/host/cat.png', image: '/host/cat.png', success: true },
|
||||||
|
toolName: 'image_generate',
|
||||||
|
type: 'tool-call'
|
||||||
|
}
|
||||||
|
])
|
||||||
|
).toEqual(['/host/cat.png', '/sandbox/cat.png'])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('dedupeGeneratedImageEchoesInParts', () => {
|
||||||
|
it('keeps the agent prose while removing the duplicated image', () => {
|
||||||
|
expect(
|
||||||
|
dedupeGeneratedImageEchoesInParts([
|
||||||
|
{ text: 'Here is your peacock!  Enjoy.', type: 'text' },
|
||||||
|
{ result: { host_image: '/host/p.png', image: '/host/p.png', success: true }, toolName: 'image_generate', type: 'tool-call' }
|
||||||
|
])
|
||||||
|
).toEqual([
|
||||||
|
{ text: 'Here is your peacock! Enjoy.', type: 'text' },
|
||||||
|
{ result: { host_image: '/host/p.png', image: '/host/p.png', success: true }, toolName: 'image_generate', type: 'tool-call' }
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('strips a sandbox path the model restated instead of the host path', () => {
|
||||||
|
expect(
|
||||||
|
dedupeGeneratedImageEchoesInParts([
|
||||||
|
{ text: '', type: 'text' },
|
||||||
|
{
|
||||||
|
result: { agent_visible_image: '/sandbox/cat.png', host_image: '/host/cat.png', image: '/host/cat.png', success: true },
|
||||||
|
toolName: 'image_generate',
|
||||||
|
type: 'tool-call'
|
||||||
|
}
|
||||||
|
])
|
||||||
|
).toEqual([
|
||||||
|
{
|
||||||
|
result: { agent_visible_image: '/sandbox/cat.png', host_image: '/host/cat.png', image: '/host/cat.png', success: true },
|
||||||
|
toolName: 'image_generate',
|
||||||
|
type: 'tool-call'
|
||||||
|
}
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('leaves pending generations untouched so the agent prose survives', () => {
|
||||||
|
const parts = [
|
||||||
|
{ text: 'Another peacock, coming up!', type: 'text' },
|
||||||
|
{ result: undefined, toolName: 'image_generate', type: 'tool-call' }
|
||||||
|
]
|
||||||
|
|
||||||
|
expect(dedupeGeneratedImageEchoesInParts(parts)).toEqual(parts)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
type ToolLike = {
|
||||||
|
result?: unknown
|
||||||
|
toolName?: unknown
|
||||||
|
type?: unknown
|
||||||
|
}
|
||||||
|
|
||||||
|
type TextLike = {
|
||||||
|
text?: unknown
|
||||||
|
type?: unknown
|
||||||
|
}
|
||||||
|
|
||||||
|
// Path-ish result fields the model may echo into its prose. Display prefers the
|
||||||
|
// host path (gateway-deliverable); stripping must catch every variant so a
|
||||||
|
// sandbox path the model restated doesn't slip through as a duplicate image.
|
||||||
|
const DISPLAY_KEYS = ['host_image', 'image'] as const
|
||||||
|
const ECHO_KEYS = ['host_image', 'image', 'agent_visible_image'] as const
|
||||||
|
|
||||||
|
function recordFromUnknown(value: unknown): Record<string, unknown> | null {
|
||||||
|
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
||||||
|
return value as Record<string, unknown>
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof value !== 'string' || !value.trim()) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(value)
|
||||||
|
|
||||||
|
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : null
|
||||||
|
} catch {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function stringFields(record: Record<string, unknown>, keys: readonly string[]): string[] {
|
||||||
|
return keys.map(key => record[key]).filter((v): v is string => typeof v === 'string' && v.trim().length > 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
function regexEscape(value: string): string {
|
||||||
|
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||||
|
}
|
||||||
|
|
||||||
|
function unique(values: string[]): string[] {
|
||||||
|
return [...new Set(values.filter(Boolean))]
|
||||||
|
}
|
||||||
|
|
||||||
|
function imageResult(part: ToolLike): Record<string, unknown> | null {
|
||||||
|
if (part.type !== 'tool-call' || part.toolName !== 'image_generate') {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
const record = recordFromUnknown(part.result)
|
||||||
|
|
||||||
|
return record && record.success !== false ? record : null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Display source for a completed `image_generate` result (host path wins). */
|
||||||
|
export function generatedImageFromResult(result: unknown): string | null {
|
||||||
|
const record = recordFromUnknown(result)
|
||||||
|
|
||||||
|
if (!record || record.success === false) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
return stringFields(record, DISPLAY_KEYS)[0] ?? null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Every path/URL a generated image might appear as in prose, for de-duping. */
|
||||||
|
export function generatedImageEchoSources(parts: readonly ToolLike[]): string[] {
|
||||||
|
return unique(parts.flatMap(part => stringFields(imageResult(part) ?? {}, ECHO_KEYS)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Strip a generated image out of prose so it only ever shows in the tool slot.
|
||||||
|
* Once a generation succeeded (`sources` is non-empty) we drop every embedded
|
||||||
|
* image and media link from that message — the model frequently restates the
|
||||||
|
* remote URL while the result holds the local path, so matching the exact
|
||||||
|
* source is not enough. Bare occurrences of the known paths/URLs are removed
|
||||||
|
* too. Surrounding prose is preserved. */
|
||||||
|
export function stripGeneratedImageEchoes(text: string, sources: readonly string[]): string {
|
||||||
|
if (!text || sources.length === 0) {
|
||||||
|
return text
|
||||||
|
}
|
||||||
|
|
||||||
|
let next = text
|
||||||
|
.replace(/!\[[^\]\n]*\]\([^)\n]*\)/g, '')
|
||||||
|
.replace(/\[[^\]\n]*\]\(\s*#media:[^)\n]*\)/g, '')
|
||||||
|
|
||||||
|
for (const source of unique([...sources])) {
|
||||||
|
next = next.replace(new RegExp(String.raw`(^|[\s([{])<?${regexEscape(source)}>?(?=$|[\s)\]},.!?])`, 'g'), '$1')
|
||||||
|
}
|
||||||
|
|
||||||
|
return next
|
||||||
|
.replace(/[ \t]+\n/g, '\n')
|
||||||
|
.replace(/\n{3,}/g, '\n\n')
|
||||||
|
.replace(/[ \t]{2,}/g, ' ')
|
||||||
|
.trim()
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Strip generated-image echoes from text parts, dropping any part left empty.
|
||||||
|
* The image lives in the tool slot; prose keeps the agent's actual words. */
|
||||||
|
export function dedupeGeneratedImageEchoesInParts<T extends TextLike & ToolLike>(parts: readonly T[]): T[] {
|
||||||
|
const sources = generatedImageEchoSources(parts)
|
||||||
|
|
||||||
|
if (!sources.length) {
|
||||||
|
return [...parts]
|
||||||
|
}
|
||||||
|
|
||||||
|
return parts
|
||||||
|
.map(part =>
|
||||||
|
part.type === 'text' && typeof part.text === 'string'
|
||||||
|
? { ...part, text: stripGeneratedImageEchoes(part.text, sources) }
|
||||||
|
: part
|
||||||
|
)
|
||||||
|
.filter(part => part.type !== 'text' || (typeof part.text === 'string' && part.text.trim().length > 0))
|
||||||
|
}
|
||||||
@@ -1180,6 +1180,12 @@ canvas {
|
|||||||
opacity: 1;
|
opacity: 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A generated image is the deliverable, not scaffolding — keep it at full
|
||||||
|
strength instead of dimming it until hover. */
|
||||||
|
[data-slot='aui_assistant-message-content'] > [data-slot='tool-block']:has([data-slot='aui_generated-image']) {
|
||||||
|
opacity: 1;
|
||||||
|
}
|
||||||
|
|
||||||
/* Conversation block rhythm. assistant-ui renders each range as a direct child
|
/* Conversation block rhythm. assistant-ui renders each range as a direct child
|
||||||
of the message content with no per-part wrapper, so adjacency rules cover
|
of the message content with no per-part wrapper, so adjacency rules cover
|
||||||
every pairing — first block needs no reset, nested tool rows are untouched.
|
every pairing — first block needs no reset, nested tool rows are untouched.
|
||||||
|
|||||||
Reference in New Issue
Block a user