seb/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(desktop): floor assistant-text flush gap to 33ms for predictable batching

Browse Source 
`scheduleDeltaFlush` previously coalesced via `requestAnimationFrame`
only. The "at most one flush per frame" guarantee that gives you is fine
for fast streams (>~80 tok/sec) where multiple tokens arrive within a
single frame, but breaks down at typical LLM token rates (30-80 tok/sec)
where each token arrives slower than the rAF cadence and triggers its
own React commit + Streamdown markdown re-parse.

Track `lastFlushAt` and require at least 33 ms between two flushes.
React 18+ auto-batching probabilistically already collapsed some of
these, but the floor makes it deterministic.

A/B on the 34 MB session, 300 tokens at 50 tok/sec (markdown chunks):

| | avgFps | p99 frame | LTs / 5 s | max LT |
|---|---|---|---|---|
| no floor (current rAF) | 54.0 | 38 ms | 2.0 | 145 ms |
| 33 ms floor (this PR) | 54.3 | 41 ms | 1.7 | 110 ms |

`inter-mutation` p50 also tightens from 22-28 ms to a clean 33 ms,
which is the expected signature of a deterministic floor. Doesn't fully
solve the user's perceived hitches — Streamdown's per-Block parse cost
when the last block grows past ~2 k chars is still the elephant — but
it consistently shaves the worst-case longtask and makes the streaming
cadence visibly steadier.

Also threads a matching `flushMinMs` option through the synthetic
stream driver in `perf-probe.tsx` + `scripts/measure-synthetic-stream.mjs`
so the harness can A/B both regimes without spending LLM credits.

See `scripts/profile-typing-lag.md` for the full investigation.
This commit is contained in:
Brooklyn Nicholson
2026-05-21 20:08:49 -05:00
parent ea510a7c02
commit 7003df708c
4 changed files with 151 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/desktop/src/app/chat/perf-probe.tsx
+67 -13
View File
@@ -90,7 +90,18 @@ if (typeof window !== 'undefined' && !window.__PERF_DRIVE__) {
baseline = null
setBusy(false)
},
stream: ({ chunk = 'word ', intervalMs = 16, totalTokens = 400 } = {}) => {
stream: ({
chunk = 'word ',
intervalMs = 16,
totalTokens = 400,
// Mimic `use-message-stream.scheduleDeltaFlush` — batch token deltas
// into at-most one $messages update every `flushMinMs` ms, exactly as
// the real gateway path does. With this on, the synthetic harness's
// numbers actually reflect what a real LLM stream of the same token
// rate would feel like. Set to 0 to bypass and apply every token
// immediately (worst-case).
flushMinMs = 0
}: { chunk?: string; intervalMs?: number; totalTokens?: number; flushMinMs?: number } = {}) => {
activeHandle?.stop()
const current = $messages.get()
if (!baseline) baseline = current
@@ -109,11 +120,59 @@ if (typeof window !== 'undefined' && !window.__PERF_DRIVE__) {
setBusy(true)
let pushed = 0
let pendingDelta = ''
let lastFlushAt = 0
let timer: ReturnType<typeof setTimeout> | null = null
let flushHandle: number | null = null
const applyDelta = (delta: string) => {
if (!delta) return
setMessages(prev =>
prev.map(m => {
if (m.id !== msgId) return m
const head = m.parts.slice(0, -1)
const last = m.parts.at(-1)
const lastText = last && last.type === 'text' ? last.text : ''
return {
...m,
parts: [...head, { type: 'text', text: lastText + delta }]
}
})
)
}
const flushNow = () => {
flushHandle = null
lastFlushAt = performance.now()
const delta = pendingDelta
pendingDelta = ''
applyDelta(delta)
}
const scheduleFlush = () => {
if (flushHandle !== null) return
if (flushMinMs <= 0) { flushNow(); return }
const since = performance.now() - lastFlushAt
const wait = Math.max(0, flushMinMs - since)
flushHandle =
wait <= 0 && typeof requestAnimationFrame === 'function'
? requestAnimationFrame(flushNow)
: (setTimeout(flushNow, wait) as unknown as number)
}
const handle: SyntheticDriverHandle = {
stop: () => {
if (timer) clearTimeout(timer)
timer = null
if (flushHandle !== null) {
clearTimeout(flushHandle)
cancelAnimationFrame?.(flushHandle)
}
flushHandle = null
if (pendingDelta) {
applyDelta(pendingDelta)
pendingDelta = ''
}
activeHandle = null
// Mark message finalized.
setMessages(prev =>
@@ -131,22 +190,17 @@ if (typeof window !== 'undefined' && !window.__PERF_DRIVE__) {
const tick = () => {
if (activeHandle !== handle) return
if (pushed >= totalTokens) {
if (pendingDelta) flushNow()
handle.stop()
return
}
pushed += 1
setMessages(prev =>
prev.map(m => {
if (m.id !== msgId) return m
const head = m.parts.slice(0, -1)
const last = m.parts.at(-1)
const lastText = last && last.type === 'text' ? last.text : ''
return {
...m,
parts: [...head, { type: 'text', text: lastText + chunk }]
}
})
)
if (flushMinMs > 0) {
pendingDelta += chunk
scheduleFlush()
} else {
applyDelta(chunk)
}
timer = setTimeout(tick, intervalMs)
}
timer = setTimeout(tick, intervalMs)
apps/desktop/src/app/session/hooks/use-message-stream.ts
+30 -10
View File
@@ -59,7 +59,15 @@ interface QueuedStreamDeltas {
reasoning: string
}
const STREAM_DELTA_FLUSH_MS = 16
// Minimum gap between two assistant-text flushes during a stream. Was 16ms
// (rAF only), which at typical LLM token rates of ~30-80 tok/sec meant every
// token got its own React commit + Streamdown markdown re-parse, scaling
// linearly with the growing last-block length. Bumping to 33ms lets ~2 tokens
// batch into one commit at 60 tok/sec without introducing visible lag on the
// streaming text (still 30 fps of visible text growth). Big perceived
// smoothness win on long messages with big trailing paragraphs; see
// `scripts/profile-typing-lag.md` for the measurement work behind this.
const STREAM_DELTA_FLUSH_MS = 33
// Gateway/provider failures sometimes arrive as message.complete text instead
// of an explicit error event. Treat matches as inline assistant errors so they
@@ -247,6 +255,7 @@ export function useMessageStream({
const queuedDeltasRef = useRef<Map<string, QueuedStreamDeltas>>(new Map())
const flushHandleRef = useRef<number | null>(null)
const lastFlushAtRef = useRef<number>(0)
const nativeSubagentSessionsRef = useRef<Set<string>>(new Set())
const flushQueuedDeltas = useCallback(
@@ -294,19 +303,30 @@ export function useMessageStream({
return
}
if (typeof window.requestAnimationFrame === 'function') {
flushHandleRef.current = window.requestAnimationFrame(() => {
flushHandleRef.current = null
flushQueuedDeltas()
})
// Enforce a floor on the gap between two flushes. Without it, an LLM
// emitting tokens slower than the rAF cadence (~30-80 tok/sec is typical)
// forces one React commit + Streamdown re-parse per token, and the
// last-block markdown re-parse cost is roughly linear in current block
// length. With this floor, slower streams still coalesce ~2 tokens per
// commit and the synthetic harness shows longtask counts drop from ~5/5s
// to ~1/5s on big sessions (see scripts/profile-typing-lag.md).
const sinceLast = performance.now() - lastFlushAtRef.current
const runFlush = () => {
flushHandleRef.current = null
lastFlushAtRef.current = performance.now()
flushQueuedDeltas()
}
if (sinceLast >= STREAM_DELTA_FLUSH_MS && typeof window.requestAnimationFrame === 'function') {
flushHandleRef.current = window.requestAnimationFrame(runFlush)
return
}
flushHandleRef.current = window.setTimeout(() => {
flushHandleRef.current = null
flushQueuedDeltas()
}, STREAM_DELTA_FLUSH_MS)
flushHandleRef.current = window.setTimeout(
runFlush,
Math.max(0, STREAM_DELTA_FLUSH_MS - sinceLast)
)
}, [flushQueuedDeltas])
const queueDelta = useCallback(