feat(desktop+gateway): remote media relay — attach images/PDFs and display gateway images over the network
Desktop connected to a remote gateway can now attach images and PDFs and
display agent-written images. Previously the desktop passed a LOCAL file path
to image.attach; on a remote gateway that path doesn't exist, so the image was
silently dropped ("skipped unreadable path") and the vision model never saw it.
The reverse direction was also broken — images the agent wrote on the gateway
rendered as dead links in the remote client.
Gateway (tui_gateway/server.py):
- image.attach_bytes: base64 byte upload written into the gateway's own images
dir and queued via the existing native-image-attach pipeline. Magic-byte
extension sniffing, data-URL prefix + whitespace tolerance, 25 MB cap,
structured error codes. Accepts content_base64/filename (canonical) and
data/ext (older-desktop aliases).
- pdf.attach: renders each page to PNG via pdftoppm (poppler-utils) at 150 DPI
and queues the pages as images; 50 MB / 25-page caps. Accepts host path or
base64 upload.
- Shared helpers (_decode_attach_base64, _sniff_image_ext, _queue_attached_image)
so the two methods and the existing image.attach don't duplicate logic.
Gateway (hermes_cli/web_server.py):
- GET /api/media: returns a gateway-local image as a base64 data URL so remote
clients can display it. Auth-gated like every /api route, extension
allowlist + size cap, AND confined to the gateway's own media roots
(images/screenshots/cache, resolved symlink-safe) so an authed caller can't
read image-extension files anywhere on disk.
Desktop (apps/desktop):
- syncImageAttachmentsForSubmit uploads bytes via image.attach_bytes when the
connection mode is 'remote'; the local fast path is unchanged.
- media.ts gains isRemoteGateway() + gatewayMediaDataUrl(); directive-text and
markdown-text fetch images over /api/media in remote mode.
Consolidates the competing remote-media PRs (#38876, #40317, #21908, #39437)
into one coherent implementation, taking the strongest parts of each and adding
shared-helper cleanup plus the /api/media root-confinement hardening on top.
The per-profile gateway switching from #38876 is intentionally left out as a
separable feature. TUI file uploads (#40492) remain a separate surface.
Tested: 11 new tui_gateway tests + 5 /api/media endpoint tests + desktop
media.remote unit tests; full tui_gateway + web_server suites green (472
passed); tsc -b clean; E2E verified the full attach→disk→queue and
gateway-path→data-URL display round-trip plus the out-of-root security block.
Co-authored-by: Max Mitcham <maxmitcham@mac.home>
Co-authored-by: Justlrnal4 <Justlrnal4@users.noreply.github.com>
Co-authored-by: Chris Cook <ccook@nvms.com>
Co-authored-by: Thomas Paquette <thomas.paquette@gmail.com>
This commit is contained in:
committed by
Teknium
co-authored by
Max Mitcham
Justlrnal4
Chris Cook
Thomas Paquette
parent
20fd0bde5d
commit
16786f3bb3
@@ -34,6 +34,7 @@ import { requestDesktopOnboarding } from '@/store/onboarding'
|
||||
import { $activeGatewayProfile, $newChatProfile, ensureGatewayProfile, normalizeProfileKey } from '@/store/profile'
|
||||
import {
|
||||
$busy,
|
||||
$connection,
|
||||
$messages,
|
||||
$yoloActive,
|
||||
setAwaitingResponse,
|
||||
@@ -80,6 +81,28 @@ function inlineErrorMessage(error: unknown, fallback: string): string {
|
||||
return (raw.match(/Error invoking remote method '[^']+': Error: (.+)$/)?.[1] ?? raw).replace(/^Error:\s*/, '').trim()
|
||||
}
|
||||
|
||||
function base64FromDataUrl(dataUrl: string): string {
|
||||
const comma = dataUrl.indexOf(',')
|
||||
|
||||
return comma >= 0 ? dataUrl.slice(comma + 1) : ''
|
||||
}
|
||||
|
||||
function imageFilenameFromPath(filePath: string): string {
|
||||
return filePath.split(/[\\/]/).filter(Boolean).pop() || 'image.png'
|
||||
}
|
||||
|
||||
// Remote gateway: the local composer-image file lives on THIS machine's disk,
|
||||
// not the gateway's, so read the bytes here and upload them via
|
||||
// image.attach_bytes. Returns null when the file can't be read.
|
||||
async function readImageForRemoteAttach(
|
||||
filePath: string
|
||||
): Promise<{ contentBase64: string; filename: string } | null> {
|
||||
const dataUrl = await window.hermesDesktop?.readFileDataUrl(filePath)
|
||||
const contentBase64 = dataUrl ? base64FromDataUrl(dataUrl) : ''
|
||||
|
||||
return contentBase64 ? { contentBase64, filename: imageFilenameFromPath(filePath) } : null
|
||||
}
|
||||
|
||||
interface PromptActionsOptions {
|
||||
activeSessionId: string | null
|
||||
activeSessionIdRef: MutableRefObject<string | null>
|
||||
@@ -197,16 +220,36 @@ export function usePromptActions({
|
||||
) => {
|
||||
const updateComposerAttachments = options.updateComposerAttachments ?? true
|
||||
const images = attachments.filter(attachment => attachment.kind === 'image' && attachment.path)
|
||||
const remote = $connection.get()?.mode === 'remote'
|
||||
|
||||
for (const attachment of images) {
|
||||
if (attachment.attachedSessionId === sessionId) {
|
||||
continue
|
||||
}
|
||||
|
||||
const result = await requestGateway<ImageAttachResponse>('image.attach', {
|
||||
session_id: sessionId,
|
||||
path: attachment.path
|
||||
})
|
||||
let result: ImageAttachResponse
|
||||
|
||||
if (remote) {
|
||||
// The gateway is on another machine — it can't read attachment.path
|
||||
// (a path on THIS disk). Upload the bytes via image.attach_bytes.
|
||||
const payload = attachment.path ? await readImageForRemoteAttach(attachment.path) : null
|
||||
|
||||
if (!payload) {
|
||||
const label = attachment.label || (attachment.path ? pathLabel(attachment.path) : 'image')
|
||||
throw new Error(`Could not read ${label}`)
|
||||
}
|
||||
|
||||
result = await requestGateway<ImageAttachResponse>('image.attach_bytes', {
|
||||
session_id: sessionId,
|
||||
content_base64: payload.contentBase64,
|
||||
filename: payload.filename
|
||||
})
|
||||
} else {
|
||||
result = await requestGateway<ImageAttachResponse>('image.attach', {
|
||||
session_id: sessionId,
|
||||
path: attachment.path
|
||||
})
|
||||
}
|
||||
|
||||
if (!result.attached) {
|
||||
const label = attachment.label || (attachment.path ? pathLabel(attachment.path) : 'image')
|
||||
|
||||
@@ -13,6 +13,13 @@ export interface ImageAttachResponse {
|
||||
path?: string
|
||||
text?: string
|
||||
message?: string
|
||||
// Returned by the byte-upload variant (image.attach_bytes) used in remote mode.
|
||||
count?: number
|
||||
bytes?: number
|
||||
name?: string
|
||||
width?: number
|
||||
height?: number
|
||||
token_estimate?: number
|
||||
}
|
||||
|
||||
export interface ImageDetachResponse {
|
||||
|
||||
Reference in New Issue
Block a user