opentui(harden): auto-heal — restart + resume on gateway crash

ui-tui-opentui-v2/src/boundary/gateway/client.ts

@@ -74,6 +74,9 @@ export class RawGatewayClient {
       stdout: 'pipe',
       stderr: 'pipe',
       onExit: (_p, code, signal) => {
+        // Identity guard: a stale child's late exit must not act after a restart
+        // has already installed a new `this.proc` (else it'd null the live child).
+        if (this.proc !== proc) return
         const reason = `gateway exited (code=${code ?? 'null'} signal=${signal ?? 'null'})`
         this.log.warn('gateway', reason)
         this.rejectAll(reason)
@@ -171,7 +174,9 @@ export class RawGatewayClient {
 
   /** Send a JSON-RPC request; resolves with `result` (long handlers reply async). */
   request<A = unknown>(method: string, params: unknown): Promise<A> {
-    if (!this.proc) this.start()
+    // Do NOT auto-start here: during the recovery backoff window `this.proc` is
+    // null, and a respawn here would BYPASS the backoff (the first spawn always
+    // comes from subscribe() → client.start()). A null proc rejects below.
     const proc = this.proc
     const stdin = proc?.stdin
     if (!stdin || typeof stdin === 'number') return Promise.reject(new Error('gateway not running'))

ui-tui-opentui-v2/src/boundary/gateway/liveGateway.ts

@@ -15,6 +15,7 @@
 import { Effect, Layer, Option, Schema } from 'effect'
 import { batch } from 'solid-js'
 
+import { backoffMs, planGatewayRecovery } from '../../logic/gatewayRecovery.ts'
 import { GatewayError } from '../errors.ts'
 import { getLog } from '../log.ts'
 import { GatewayEventSchema, type GatewayEvent } from '../schema/GatewayEvent.ts'
@@ -30,6 +31,14 @@ function makeLiveGateway(): { service: GatewayServiceShape; stop: () => void } {
   const handlers = new Set<(event: GatewayEvent) => void>()
   let sessionId: string | undefined
 
+  // Auto-heal recovery state (driver below). `recoverSid` is the resume target
+  // carried across a respawn that died before gateway.ready; `recoveryAttempts`
+  // is the sliding crash-loop budget window; `restartTimer` is the pending
+  // backoff respawn (cleared on teardown so it can't fire post-stop).
+  let recoverSid: string | undefined
+  let recoveryAttempts: number[] = []
+  let restartTimer: ReturnType<typeof setTimeout> | undefined
+
   // 16ms event coalescing → one batched repaint (opencode sdk.tsx model).
   let queue: GatewayEvent[] = []
   let timer: ReturnType<typeof setTimeout> | undefined
@@ -69,10 +78,37 @@ function makeLiveGateway(): { service: GatewayServiceShape; stop: () => void } {
     enqueue(decoded.value)
   }
 
+  // Recovery driver: on a child exit, clear the frozen spinner (via the store's
+  // gateway.exited case), then — under the crash-loop budget — respawn the child
+  // on exponential backoff. The post-respawn gateway.ready triggers the re-resume
+  // (driven from entry's subscribe callback). Hoisted so it can be passed to
+  // `new RawGatewayClient` below while itself referencing the `client` const —
+  // `client` is assigned by the time onExit ever fires at runtime.
+  function onExit(reason: string): void {
+    log.warn('gateway', 'transport exited', { reason })
+    // Clears the frozen spinner + shows status (store handles gateway.exited).
+    enqueue({ type: 'gateway.exited', payload: { reason } })
+    const plan = planGatewayRecovery(sessionId ?? null, recoverSid ?? null, recoveryAttempts, Date.now())
+    recoveryAttempts = plan.attempts
+    if (!plan.recover || plan.sid === null) {
+      enqueue({ type: 'error', payload: { message: 'gateway exited repeatedly — type /resume to retry' } })
+      return
+    }
+    recoverSid = plan.sid
+    const attempt = recoveryAttempts.length
+    const delay = backoffMs(attempt)
+    enqueue({ type: 'gateway.recovering', payload: { attempt, delay_ms: delay } })
+    if (restartTimer) clearTimeout(restartTimer)
+    restartTimer = setTimeout(() => {
+      restartTimer = undefined
+      client.start()
+    }, delay)
+  }
+
   const client = new RawGatewayClient({
     log,
     onEvent: onRawEvent,
-    onExit: reason => log.warn('gateway', 'transport exited', { reason })
+    onExit
   })
 
   const service: GatewayServiceShape = {
@@ -118,6 +154,9 @@ function makeLiveGateway(): { service: GatewayServiceShape; stop: () => void } {
   const stop = () => {
     if (timer) clearTimeout(timer)
     timer = undefined
+    // Also kill any pending backoff respawn so it can't fire after teardown.
+    if (restartTimer) clearTimeout(restartTimer)
+    restartTimer = undefined
     client.stop()
   }
   return { service, stop }

ui-tui-opentui-v2/src/entry/main.tsx

@@ -194,8 +194,28 @@ export const run = Effect.fn('Tui.run')(function* (input: TuiInput) {
       const pasteStore = createPasteStore()
 
       // Contact point #2: boundary pushes decoded events into the Solid store.
+      // The callback ALSO drives auto-heal re-resume: a post-crash gateway.ready
+      // (i.e. one that follows a gateway.exited, so `recoverSid` is set) re-resumes
+      // the session so the transcript continues. The INITIAL gateway.ready has
+      // `recoverSid === undefined`, so the normal bootstrap path is untouched.
       const gateway = yield* GatewayService
-      yield* gateway.subscribe(event => store.apply(event))
+      let recoverSid: string | undefined
+      yield* gateway.subscribe(event => {
+        store.apply(event)
+        if (event.type === 'gateway.exited') {
+          recoverSid = gateway.sessionId() ?? recoverSid
+        } else if (event.type === 'gateway.ready' && recoverSid !== undefined) {
+          const sid = recoverSid
+          recoverSid = undefined
+          Effect.runFork(
+            resumeInto(gateway, store, sid, input.cols).pipe(
+              Effect.catchCause(cause =>
+                Effect.sync(() => getLog().warn('recover', 'resume failed', { cause: String(cause) }))
+              )
+            )
+          )
+        }
+      })
 
       // ── Ctrl+C state machine (item 11) ──────────────────────────────────
       // While a turn runs, the first Ctrl+C STOPS the agent (session.interrupt);

ui-tui-opentui-v2/src/logic/store.ts

@@ -685,7 +685,10 @@ export function createSessionStore() {
       // the user their in-flight reply was lost, and show a recovering status.
       case 'gateway.exited': {
         setState('info', prev => ({ ...prev, running: false }))
-        setState('status', 'gateway exited — recovering…')
+        // Neutral status: we don't ALWAYS recover (budget exhaustion). The
+        // "recovering…" wording now comes from the gateway.recovering case,
+        // which fires only when a respawn is actually scheduled.
+        setState('status', 'gateway exited')
         const reason = event.payload?.reason
         const base = 'gateway exited — recovering your session (any in-flight reply was lost)'
         pushSystem(reason ? `${base}: ${reason}` : base)

ui-tui-opentui-v2/src/test/store.test.ts

@@ -339,7 +339,8 @@ describe('session store — gateway lifecycle / transport errors (auto-heal foun
     store.apply({ type: 'gateway.exited' })
     // THE key bug fix: the spinner is cleared even though no message.complete arrived.
     expect(store.state.info.running).toBe(false)
-    expect(store.state.status).toBe('gateway exited — recovering…')
+    // Neutral status — "recovering…" now comes from gateway.recovering only.
+    expect(store.state.status).toBe('gateway exited')
     const sys = store.state.messages.filter(m => m.role === 'system')
     expect(sys).toHaveLength(1)
     expect(sys[0]!.text).toContain('in-flight reply was lost')