Files
hermes-agent/ui-tui/src/entry.tsx
T
teknium1 e76d8bf5aa fix(tui): stop persisting full tool output in trail lines (silent OOM death)
A heavy --tui session (browser snapshots, large tool outputs) silently
OOM-killed the Node parent within minutes — closing the gateway child's
stdin, which the user saw only as a bare "gateway exited" / stdin EOF.
CLI was immune. Root cause: each completed tool's verbose trail line
embedded up to 16KB of result_text, persisted in transcript Msg.tools[]
for the whole session and rendered EXPANDED by default, so an Ink
render-node tree was built for every one of up to 800 messages at once.
That tree blew past Node's heap at a few hundred MB — far below the 2.5GB
memory-monitor exit threshold, so the death was never even attributed.

- text.ts: persisted verbose tool-trail blocks now cap to a small preview
  (VERBOSE_TRAIL_MAX_CHARS=800/12 lines), not the 16KB live-render budget.
  Retained trail strings drop ~17x (12.2MB -> 0.7MB at 800 msgs); the live
  streaming tail still uses the larger LIVE_RENDER budget.
- tui_gateway/server.py: lower the gateway-side verbose text cap to match
  (1KB/16 lines) so we stop shipping output the TUI no longer renders.
- memoryMonitor.ts: derive critical/high thresholds from the real V8 heap
  ceiling (~88%/70%) instead of the hardcoded 2.5GB that killed the process
  at 31% of an 8GB ceiling; add a one-shot onWarn early-warning on fast
  sub-threshold heap growth so the next such death is diagnosable, not silent.
- entry.tsx: wire onWarn to a crash-log breadcrumb + stderr line.

Full tool output is unchanged in the agent context and SQLite session — this
is display/transport only, no behavior or context change.

Fixes #34095. Related #27282.

Tests: ui-tui text + new memoryMonitor suites (33 pass), python verbose-cap
guard (5 pass); full ui-tui suite shows no new failures vs pristine main.
E2E repro confirms the retention drop.
2026-06-03 06:00:22 -07:00

125 lines
5.0 KiB
TypeScript

#!/usr/bin/env -S node --max-old-space-size=8192 --expose-gc
// Must be first import. If the user explicitly opts into truecolor, this
// nudges chalk / supports-color before either package is initialized.
import './lib/forceTruecolor.js'
import type { FrameEvent } from '@hermes/ink'
import { TERMUX_TUI_MODE } from './config/env.js'
import { GatewayClient } from './gatewayClient.js'
import { setupGracefulExit } from './lib/gracefulExit.js'
import { formatBytes, type HeapDumpResult, performHeapDump } from './lib/memory.js'
import { type MemorySnapshot, startMemoryMonitor } from './lib/memoryMonitor.js'
import { openExternalUrl } from './lib/openExternalUrl.js'
import { recordParentLifecycle } from './lib/parentLog.js'
import { resetTerminalModes } from './lib/terminalModes.js'
if (!process.stdin.isTTY) {
console.log('hermes-tui: no TTY')
process.exit(0)
}
// Start from a clean slate. If a previous TUI crashed or was kill -9'd, the
// terminal tab can still have mouse/focus/paste modes enabled.
resetTerminalModes()
// Desktop terminals benefit from a clean startup slate because the TUI usually
// runs in AlternateScreen. On Termux we keep prior output intact so users can
// review/copy earlier assistant replies after reopening the app.
if (TERMUX_TUI_MODE) {
process.stdout.write('\n')
} else {
process.stdout.write('\x1b[2J\x1b[H\x1b[3J')
}
const gw = new GatewayClient()
gw.start()
const dumpNotice = (snap: MemorySnapshot, dump: HeapDumpResult | null) =>
`hermes-tui: ${snap.level} memory (${formatBytes(snap.heapUsed)}) — auto heap dump → ${dump?.heapPath ?? '(failed)'}\n`
setupGracefulExit({
cleanups: [
() => {
resetTerminalModes()
return gw.kill('graceful-exit-cleanup')
}
],
onError: (scope, err) => {
const message = err instanceof Error ? `${err.name}: ${err.message}\n${err.stack ?? ''}` : String(err)
recordParentLifecycle(`${scope}: ${message.split('\n')[0]?.slice(0, 400) ?? ''}`)
process.stderr.write(`hermes-tui lifecycle ${scope}: ${message.slice(0, 2000)}\n`)
},
onSignal: signal => {
// The next line in the crash log is the child's `=== SIGTERM received ===`
// (gw.kill forwards SIGTERM regardless of which signal hit us) — this is
// what tells SIGHUP (terminal/SSH dropped) apart from a real SIGTERM.
recordParentLifecycle(`graceful-exit received signal=${signal} → killing gateway`)
resetTerminalModes()
process.stderr.write(`hermes-tui lifecycle: received ${signal}\n`)
}
})
const stopMemoryMonitor = startMemoryMonitor({
onCritical: (snap, dump) => {
// process.exit(137) closes the child's stdin → the gateway logs a clean
// EOF, NOT SIGTERM. Recording it here is the only way a crash report can
// attribute a death to Node OOM rather than a signal-driven kill.
recordParentLifecycle(`memory-critical process.exit(137) heap=${formatBytes(snap.heapUsed)} rss=${formatBytes(snap.rss)} dump=${dump?.heapPath ?? 'failed'}`)
resetTerminalModes()
process.stderr.write(`hermes-tui lifecycle: memory critical exit heap=${formatBytes(snap.heapUsed)} rss=${formatBytes(snap.rss)}\n`)
process.stderr.write(dumpNotice(snap, dump))
process.stderr.write('hermes-tui: exiting to avoid OOM; restart to recover\n')
process.exit(137)
},
onHigh: (snap, dump) => process.stderr.write(dumpNotice(snap, dump)),
// Sub-threshold abnormal heap growth (#34095). The TUI used to die silently
// here — Node OOMs from a render-tree blowup well below the exit threshold,
// so the only trace was a bare gateway `stdin EOF`. Persist a breadcrumb +
// stderr line so the next such death is attributable instead of silent.
onWarn: snap => {
recordParentLifecycle(`memory-warning fast heap growth heap=${formatBytes(snap.heapUsed)} rss=${formatBytes(snap.rss)}`)
process.stderr.write(
`hermes-tui: heap climbing fast (${formatBytes(snap.heapUsed)}) — a large tool output or long session may be straining memory\n`
)
}
})
if (process.env.HERMES_HEAPDUMP_ON_START === '1') {
void performHeapDump('manual')
}
process.on('beforeExit', () => stopMemoryMonitor())
const [ink, { App }, { logFrameEvent }, { trackFrame }] = await Promise.all([
import('@hermes/ink'),
import('./app.js'),
import('./lib/perfPane.js'),
import('./lib/fpsStore.js')
])
// Both consumers are undefined when their env flags are off; only attach
// onFrame when at least one is on so ink skips timing in the default case.
const onFrame =
logFrameEvent || trackFrame
? (event: FrameEvent) => {
logFrameEvent?.(event)
trackFrame?.(event.durationMs)
}
: undefined
ink.render(<App gw={gw} />, {
exitOnCtrlC: false,
onFrame,
// Open URLs in the user's default browser when a link cell is clicked.
// The TUI's mouse tracking captures click events before Terminal.app's
// own URL detection can fire, so without this hook clicks on `<Link>`
// do nothing in any terminal where mouseTracking is on.
onHyperlinkClick: url => {
openExternalUrl(url)
}
})