feat(desktop+gateway): remote media relay — attach images/PDFs and display gateway images over the network

Desktop connected to a remote gateway can now attach images and PDFs and
display agent-written images. Previously the desktop passed a LOCAL file path
to image.attach; on a remote gateway that path doesn't exist, so the image was
silently dropped ("skipped unreadable path") and the vision model never saw it.
The reverse direction was also broken — images the agent wrote on the gateway
rendered as dead links in the remote client.

Gateway (tui_gateway/server.py):
- image.attach_bytes: base64 byte upload written into the gateway's own images
  dir and queued via the existing native-image-attach pipeline. Magic-byte
  extension sniffing, data-URL prefix + whitespace tolerance, 25 MB cap,
  structured error codes. Accepts content_base64/filename (canonical) and
  data/ext (older-desktop aliases).
- pdf.attach: renders each page to PNG via pdftoppm (poppler-utils) at 150 DPI
  and queues the pages as images; 50 MB / 25-page caps. Accepts host path or
  base64 upload.
- Shared helpers (_decode_attach_base64, _sniff_image_ext, _queue_attached_image)
  so the two methods and the existing image.attach don't duplicate logic.

Gateway (hermes_cli/web_server.py):
- GET /api/media: returns a gateway-local image as a base64 data URL so remote
  clients can display it. Auth-gated like every /api route, extension
  allowlist + size cap, AND confined to the gateway's own media roots
  (images/screenshots/cache, resolved symlink-safe) so an authed caller can't
  read image-extension files anywhere on disk.

Desktop (apps/desktop):
- syncImageAttachmentsForSubmit uploads bytes via image.attach_bytes when the
  connection mode is 'remote'; the local fast path is unchanged.
- media.ts gains isRemoteGateway() + gatewayMediaDataUrl(); directive-text and
  markdown-text fetch images over /api/media in remote mode.

Consolidates the competing remote-media PRs (#38876, #40317, #21908, #39437)
into one coherent implementation, taking the strongest parts of each and adding
shared-helper cleanup plus the /api/media root-confinement hardening on top.
The per-profile gateway switching from #38876 is intentionally left out as a
separable feature. TUI file uploads (#40492) remain a separate surface.

Tested: 11 new tui_gateway tests + 5 /api/media endpoint tests + desktop
media.remote unit tests; full tui_gateway + web_server suites green (472
passed); tsc -b clean; E2E verified the full attach→disk→queue and
gateway-path→data-URL display round-trip plus the out-of-root security block.

Co-authored-by: Max Mitcham <maxmitcham@mac.home>
Co-authored-by: Justlrnal4 <Justlrnal4@users.noreply.github.com>
Co-authored-by: Chris Cook <ccook@nvms.com>
Co-authored-by: Thomas Paquette <thomas.paquette@gmail.com>
This commit is contained in:
teknium1
2026-06-07 10:05:53 -07:00
committed by Teknium
co-authored by Max Mitcham Justlrnal4 Chris Cook Thomas Paquette
parent 20fd0bde5d
commit 16786f3bb3
11 changed files with 759 additions and 11 deletions
+58
View File
@@ -0,0 +1,58 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { $connection } from '@/store/session'
import { filePathFromMediaPath, gatewayMediaDataUrl, isRemoteGateway } from './media'
describe('isRemoteGateway', () => {
afterEach(() => {
$connection.set(null)
})
it('is false with no connection', () => {
$connection.set(null)
expect(isRemoteGateway()).toBe(false)
})
it('is false in local mode', () => {
$connection.set({ mode: 'local' } as never)
expect(isRemoteGateway()).toBe(false)
})
it('is true in remote mode', () => {
$connection.set({ mode: 'remote' } as never)
expect(isRemoteGateway()).toBe(true)
})
})
describe('filePathFromMediaPath', () => {
it('passes through a plain path', () => {
expect(filePathFromMediaPath('/home/u/.hermes/images/a.png')).toBe('/home/u/.hermes/images/a.png')
})
it('decodes a file:// URL with encoded characters', () => {
expect(filePathFromMediaPath('file:///tmp/a%20b.png')).toBe('/tmp/a b.png')
})
})
describe('gatewayMediaDataUrl', () => {
const api = vi.fn(async () => ({ data_url: 'data:image/png;base64,ZHVtbXk=' }))
beforeEach(() => {
api.mockClear()
vi.stubGlobal('window', { hermesDesktop: { api } })
})
afterEach(() => {
vi.unstubAllGlobals()
})
it('requests the encoded gateway path and returns the data URL', async () => {
const url = await gatewayMediaDataUrl('/home/u/.hermes/images/a b.png')
expect(url).toBe('data:image/png;base64,ZHVtbXk=')
expect(api).toHaveBeenCalledWith({
path: '/api/media?path=%2Fhome%2Fu%2F.hermes%2Fimages%2Fa%20b.png'
})
})
})
+22
View File
@@ -1,3 +1,5 @@
import { $connection } from '@/store/session'
export type MediaKind = 'audio' | 'image' | 'video' | 'file'
interface MediaInfo {
@@ -89,6 +91,26 @@ export function filePathFromMediaPath(path: string): string {
}
}
// True when this desktop shell is wired to a remote gateway. Local media paths
// then live on the gateway machine, not this disk, so we fetch them over the API.
export function isRemoteGateway(): boolean {
return $connection.get()?.mode === 'remote'
}
// Fetch a gateway-local image as a data URL via the authenticated REST bridge.
// Used in remote mode where readFileDataUrl (which reads THIS machine's disk)
// can't see files the agent wrote on the gateway. Requires the gateway to
// expose GET /api/media (hermes_cli/web_server.py).
export async function gatewayMediaDataUrl(path: string): Promise<string> {
const file = filePathFromMediaPath(path)
const result = await window.hermesDesktop!.api<{ data_url: string }>({
path: `/api/media?path=${encodeURIComponent(file)}`
})
return result.data_url
}
export function mediaDisplayLabel(path: string): string {
const escaped = mediaName(path).replace(/[[\]\\]/g, '\\$&')
const kind = mediaKind(path)