A heavy --tui session (browser snapshots, large tool outputs) silently OOM-killed the Node parent within minutes — closing the gateway child's stdin, which the user saw only as a bare "gateway exited" / stdin EOF. CLI was immune. Root cause: each completed tool's verbose trail line embedded up to 16KB of result_text, persisted in transcript Msg.tools[] for the whole session and rendered EXPANDED by default, so an Ink render-node tree was built for every one of up to 800 messages at once. That tree blew past Node's heap at a few hundred MB — far below the 2.5GB memory-monitor exit threshold, so the death was never even attributed. - text.ts: persisted verbose tool-trail blocks now cap to a small preview (VERBOSE_TRAIL_MAX_CHARS=800/12 lines), not the 16KB live-render budget. Retained trail strings drop ~17x (12.2MB -> 0.7MB at 800 msgs); the live streaming tail still uses the larger LIVE_RENDER budget. - tui_gateway/server.py: lower the gateway-side verbose text cap to match (1KB/16 lines) so we stop shipping output the TUI no longer renders. - memoryMonitor.ts: derive critical/high thresholds from the real V8 heap ceiling (~88%/70%) instead of the hardcoded 2.5GB that killed the process at 31% of an 8GB ceiling; add a one-shot onWarn early-warning on fast sub-threshold heap growth so the next such death is diagnosable, not silent. - entry.tsx: wire onWarn to a crash-log breadcrumb + stderr line. Full tool output is unchanged in the agent context and SQLite session — this is display/transport only, no behavior or context change. Fixes #34095. Related #27282. Tests: ui-tui text + new memoryMonitor suites (33 pass), python verbose-cap guard (5 pass); full ui-tui suite shows no new failures vs pristine main. E2E repro confirms the retention drop.
125 lines
5.0 KiB
TypeScript
125 lines
5.0 KiB
TypeScript
#!/usr/bin/env -S node --max-old-space-size=8192 --expose-gc
|
|
// Must be first import. If the user explicitly opts into truecolor, this
|
|
// nudges chalk / supports-color before either package is initialized.
|
|
import './lib/forceTruecolor.js'
|
|
|
|
import type { FrameEvent } from '@hermes/ink'
|
|
|
|
import { TERMUX_TUI_MODE } from './config/env.js'
|
|
import { GatewayClient } from './gatewayClient.js'
|
|
import { setupGracefulExit } from './lib/gracefulExit.js'
|
|
import { formatBytes, type HeapDumpResult, performHeapDump } from './lib/memory.js'
|
|
import { type MemorySnapshot, startMemoryMonitor } from './lib/memoryMonitor.js'
|
|
import { openExternalUrl } from './lib/openExternalUrl.js'
|
|
import { recordParentLifecycle } from './lib/parentLog.js'
|
|
import { resetTerminalModes } from './lib/terminalModes.js'
|
|
|
|
if (!process.stdin.isTTY) {
|
|
console.log('hermes-tui: no TTY')
|
|
process.exit(0)
|
|
}
|
|
|
|
// Start from a clean slate. If a previous TUI crashed or was kill -9'd, the
|
|
// terminal tab can still have mouse/focus/paste modes enabled.
|
|
resetTerminalModes()
|
|
|
|
// Desktop terminals benefit from a clean startup slate because the TUI usually
|
|
// runs in AlternateScreen. On Termux we keep prior output intact so users can
|
|
// review/copy earlier assistant replies after reopening the app.
|
|
if (TERMUX_TUI_MODE) {
|
|
process.stdout.write('\n')
|
|
} else {
|
|
process.stdout.write('\x1b[2J\x1b[H\x1b[3J')
|
|
}
|
|
|
|
const gw = new GatewayClient()
|
|
|
|
gw.start()
|
|
|
|
const dumpNotice = (snap: MemorySnapshot, dump: HeapDumpResult | null) =>
|
|
`hermes-tui: ${snap.level} memory (${formatBytes(snap.heapUsed)}) — auto heap dump → ${dump?.heapPath ?? '(failed)'}\n`
|
|
|
|
setupGracefulExit({
|
|
cleanups: [
|
|
() => {
|
|
resetTerminalModes()
|
|
|
|
return gw.kill('graceful-exit-cleanup')
|
|
}
|
|
],
|
|
onError: (scope, err) => {
|
|
const message = err instanceof Error ? `${err.name}: ${err.message}\n${err.stack ?? ''}` : String(err)
|
|
|
|
recordParentLifecycle(`${scope}: ${message.split('\n')[0]?.slice(0, 400) ?? ''}`)
|
|
process.stderr.write(`hermes-tui lifecycle ${scope}: ${message.slice(0, 2000)}\n`)
|
|
},
|
|
onSignal: signal => {
|
|
// The next line in the crash log is the child's `=== SIGTERM received ===`
|
|
// (gw.kill forwards SIGTERM regardless of which signal hit us) — this is
|
|
// what tells SIGHUP (terminal/SSH dropped) apart from a real SIGTERM.
|
|
recordParentLifecycle(`graceful-exit received signal=${signal} → killing gateway`)
|
|
resetTerminalModes()
|
|
process.stderr.write(`hermes-tui lifecycle: received ${signal}\n`)
|
|
}
|
|
})
|
|
|
|
const stopMemoryMonitor = startMemoryMonitor({
|
|
onCritical: (snap, dump) => {
|
|
// process.exit(137) closes the child's stdin → the gateway logs a clean
|
|
// EOF, NOT SIGTERM. Recording it here is the only way a crash report can
|
|
// attribute a death to Node OOM rather than a signal-driven kill.
|
|
recordParentLifecycle(`memory-critical process.exit(137) heap=${formatBytes(snap.heapUsed)} rss=${formatBytes(snap.rss)} dump=${dump?.heapPath ?? 'failed'}`)
|
|
resetTerminalModes()
|
|
process.stderr.write(`hermes-tui lifecycle: memory critical exit heap=${formatBytes(snap.heapUsed)} rss=${formatBytes(snap.rss)}\n`)
|
|
process.stderr.write(dumpNotice(snap, dump))
|
|
process.stderr.write('hermes-tui: exiting to avoid OOM; restart to recover\n')
|
|
process.exit(137)
|
|
},
|
|
onHigh: (snap, dump) => process.stderr.write(dumpNotice(snap, dump)),
|
|
// Sub-threshold abnormal heap growth (#34095). The TUI used to die silently
|
|
// here — Node OOMs from a render-tree blowup well below the exit threshold,
|
|
// so the only trace was a bare gateway `stdin EOF`. Persist a breadcrumb +
|
|
// stderr line so the next such death is attributable instead of silent.
|
|
onWarn: snap => {
|
|
recordParentLifecycle(`memory-warning fast heap growth heap=${formatBytes(snap.heapUsed)} rss=${formatBytes(snap.rss)}`)
|
|
process.stderr.write(
|
|
`hermes-tui: heap climbing fast (${formatBytes(snap.heapUsed)}) — a large tool output or long session may be straining memory\n`
|
|
)
|
|
}
|
|
})
|
|
|
|
if (process.env.HERMES_HEAPDUMP_ON_START === '1') {
|
|
void performHeapDump('manual')
|
|
}
|
|
|
|
process.on('beforeExit', () => stopMemoryMonitor())
|
|
|
|
const [ink, { App }, { logFrameEvent }, { trackFrame }] = await Promise.all([
|
|
import('@hermes/ink'),
|
|
import('./app.js'),
|
|
import('./lib/perfPane.js'),
|
|
import('./lib/fpsStore.js')
|
|
])
|
|
|
|
// Both consumers are undefined when their env flags are off; only attach
|
|
// onFrame when at least one is on so ink skips timing in the default case.
|
|
const onFrame =
|
|
logFrameEvent || trackFrame
|
|
? (event: FrameEvent) => {
|
|
logFrameEvent?.(event)
|
|
trackFrame?.(event.durationMs)
|
|
}
|
|
: undefined
|
|
|
|
ink.render(<App gw={gw} />, {
|
|
exitOnCtrlC: false,
|
|
onFrame,
|
|
// Open URLs in the user's default browser when a link cell is clicked.
|
|
// The TUI's mouse tracking captures click events before Terminal.app's
|
|
// own URL detection can fire, so without this hook clicks on `<Link>`
|
|
// do nothing in any terminal where mouseTracking is on.
|
|
onHyperlinkClick: url => {
|
|
openExternalUrl(url)
|
|
}
|
|
})
|