feat(desktop+gateway): remote media relay — attach images/PDFs and display gateway images over the network
Desktop connected to a remote gateway can now attach images and PDFs and
display agent-written images. Previously the desktop passed a LOCAL file path
to image.attach; on a remote gateway that path doesn't exist, so the image was
silently dropped ("skipped unreadable path") and the vision model never saw it.
The reverse direction was also broken — images the agent wrote on the gateway
rendered as dead links in the remote client.
Gateway (tui_gateway/server.py):
- image.attach_bytes: base64 byte upload written into the gateway's own images
dir and queued via the existing native-image-attach pipeline. Magic-byte
extension sniffing, data-URL prefix + whitespace tolerance, 25 MB cap,
structured error codes. Accepts content_base64/filename (canonical) and
data/ext (older-desktop aliases).
- pdf.attach: renders each page to PNG via pdftoppm (poppler-utils) at 150 DPI
and queues the pages as images; 50 MB / 25-page caps. Accepts host path or
base64 upload.
- Shared helpers (_decode_attach_base64, _sniff_image_ext, _queue_attached_image)
so the two methods and the existing image.attach don't duplicate logic.
Gateway (hermes_cli/web_server.py):
- GET /api/media: returns a gateway-local image as a base64 data URL so remote
clients can display it. Auth-gated like every /api route, extension
allowlist + size cap, AND confined to the gateway's own media roots
(images/screenshots/cache, resolved symlink-safe) so an authed caller can't
read image-extension files anywhere on disk.
Desktop (apps/desktop):
- syncImageAttachmentsForSubmit uploads bytes via image.attach_bytes when the
connection mode is 'remote'; the local fast path is unchanged.
- media.ts gains isRemoteGateway() + gatewayMediaDataUrl(); directive-text and
markdown-text fetch images over /api/media in remote mode.
Consolidates the competing remote-media PRs (#38876, #40317, #21908, #39437)
into one coherent implementation, taking the strongest parts of each and adding
shared-helper cleanup plus the /api/media root-confinement hardening on top.
The per-profile gateway switching from #38876 is intentionally left out as a
separable feature. TUI file uploads (#40492) remain a separate surface.
Tested: 11 new tui_gateway tests + 5 /api/media endpoint tests + desktop
media.remote unit tests; full tui_gateway + web_server suites green (472
passed); tsc -b clean; E2E verified the full attach→disk→queue and
gateway-path→data-URL display round-trip plus the out-of-root security block.
Co-authored-by: Max Mitcham <maxmitcham@mac.home>
Co-authored-by: Justlrnal4 <Justlrnal4@users.noreply.github.com>
Co-authored-by: Chris Cook <ccook@nvms.com>
Co-authored-by: Thomas Paquette <thomas.paquette@gmail.com>
This commit is contained in:
committed by
Teknium
co-authored by
Max Mitcham
Justlrnal4
Chris Cook
Thomas Paquette
parent
20fd0bde5d
commit
16786f3bb3
@@ -7,6 +7,7 @@ import { Fragment, useEffect, useMemo, useState } from 'react'
|
||||
|
||||
import { ZoomableImage } from '@/components/chat/zoomable-image'
|
||||
import { extractEmbeddedImages } from '@/lib/embedded-images'
|
||||
import { gatewayMediaDataUrl, isRemoteGateway } from '@/lib/media'
|
||||
|
||||
const HERMES_REF_TYPES = ['file', 'folder', 'url', 'image', 'tool', 'line', 'terminal', 'session'] as const
|
||||
type HermesRefType = (typeof HERMES_REF_TYPES)[number]
|
||||
@@ -327,25 +328,32 @@ export const DirectiveText: TextMessagePartComponent = ({ text }: TextMessagePar
|
||||
* messages render after the backend embeds the data URL, so the UX is stable
|
||||
* across initial send and refresh. */
|
||||
const DirectiveImage: FC<{ id: string; label: string }> = ({ id, label }) => {
|
||||
const remote = /^(?:https?|data):/i.test(id)
|
||||
const [src, setSrc] = useState<string | null>(remote ? id : null)
|
||||
const isUrl = /^(?:https?|data):/i.test(id)
|
||||
const [src, setSrc] = useState<string | null>(isUrl ? id : null)
|
||||
const [failed, setFailed] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (remote || !id) {
|
||||
if (isUrl || !id) {
|
||||
return
|
||||
}
|
||||
|
||||
let alive = true
|
||||
void window.hermesDesktop
|
||||
?.readFileDataUrl(id)
|
||||
.then(url => alive && setSrc(url))
|
||||
|
||||
// Remote gateway: the image lives on the gateway's disk, not ours — fetch
|
||||
// it over the authenticated API. Local: read it straight off this disk.
|
||||
const load =
|
||||
window.hermesDesktop && isRemoteGateway()
|
||||
? gatewayMediaDataUrl(id)
|
||||
: window.hermesDesktop?.readFileDataUrl(id)
|
||||
|
||||
void Promise.resolve(load)
|
||||
.then(url => alive && url && setSrc(url))
|
||||
.catch(() => alive && setFailed(true))
|
||||
|
||||
return () => {
|
||||
alive = false
|
||||
}
|
||||
}, [id, remote])
|
||||
}, [id, isUrl])
|
||||
|
||||
if (failed) {
|
||||
return <DirectiveChip id={id} label={label} type="image" />
|
||||
|
||||
@@ -17,6 +17,8 @@ import { createMemoizedMathPlugin } from '@/lib/katex-memo'
|
||||
import { preprocessMarkdown } from '@/lib/markdown-preprocess'
|
||||
import {
|
||||
filePathFromMediaPath,
|
||||
gatewayMediaDataUrl,
|
||||
isRemoteGateway,
|
||||
mediaExternalUrl,
|
||||
mediaKind,
|
||||
mediaName,
|
||||
@@ -51,6 +53,12 @@ async function mediaSrc(path: string): Promise<string> {
|
||||
return mediaStreamUrl(path)
|
||||
}
|
||||
|
||||
// Remote gateway: the image lives on the gateway machine, so read it over the
|
||||
// authenticated API rather than this machine's disk.
|
||||
if (window.hermesDesktop && isRemoteGateway()) {
|
||||
return gatewayMediaDataUrl(path)
|
||||
}
|
||||
|
||||
if (!window.hermesDesktop?.readFileDataUrl) {
|
||||
return mediaExternalUrl(path)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user