chore(desktop): synthetic-stream perf harness + scripts

Drops the React `<Profiler>` approach (no-op because Vite is currently
serving the production React build) in favor of an externally-observable
measurement stack: rAF frame intervals, `PerformanceObserver({entryTypes:
['longtask']})`, and a `MutationObserver` on the live streaming message.

Adds a synthetic stream driver — `window.__PERF_DRIVE__.stream({...})` —
that pushes tokens through the live `$messages` atom at a controlled rate,
so the assistant-ui runtime, incremental repository, and Streamdown
markdown pipeline see the same workload they'd see during a real LLM
stream, without the LLM cost.

The driver lives in `src/app/chat/perf-probe.tsx`; `main.tsx` side-imports
it under `import.meta.env.MODE !== 'production'` so it tree-shakes out of
prod builds. (Using `MODE` rather than `DEV` because our Vite setup
currently reports `DEV=false` even under `vite dev` — see the dev-build
note in `profile-typing-lag.md`.)

Scripts:
  - measure-synthetic-stream.mjs  drive synthetic + record frame/longtask/mutation
  - profile-synth-stream.mjs      CPU profile + top self-time during synthetic
  - measure-real-stream.mjs       same harness, real LLM stream
  - profile-real-stream.mjs       CPU profile bracketing the real stream window
  - eval.mjs / reload.mjs         small CDP helpers

A real-LLM measurement on Cloud Shadows (gpt-4o-mini, 39 s window) showed
12 longtasks in the same 75-127 ms range the synthetic predicted, so the
synthetic is a faithful proxy.
This commit is contained in:
Brooklyn Nicholson
2026-05-21 19:38:26 -05:00
parent 5abf89ddd1
commit 99f2a9503c
9 changed files with 1173 additions and 0 deletions
+167
View File
@@ -0,0 +1,167 @@
import { Profiler, type ProfilerOnRenderCallback, type ReactNode } from 'react'
import { $messages, setMessages, setBusy } from '@/store/session'
type Sample = {
id: string
phase: string
actualDuration: number
baseDuration: number
startTime: number
commitTime: number
}
type SyntheticDriverHandle = { stop: () => void }
declare global {
interface Window {
__PERF_PROBE__?: {
samples: Sample[]
enabled: boolean
clear: () => void
summary: () => Record<string, { count: number; total: number; max: number; p50: number; p95: number }>
}
__PERF_DRIVE__?: {
/** Inject an assistant message and grow it by `chunk` every `intervalMs`. Returns a stop handle. */
stream: (opts?: { chunk?: string; intervalMs?: number; totalTokens?: number }) => SyntheticDriverHandle
reset: () => void
snapshotMsgs: () => number
}
}
}
if (typeof window !== 'undefined' && !window.__PERF_PROBE__) {
const samples: Sample[] = []
window.__PERF_PROBE__ = {
samples,
enabled: false,
clear: () => {
samples.length = 0
},
summary: () => {
const byId = new Map<string, number[]>()
for (const s of samples) {
const k = `${s.id}:${s.phase}`
const arr = byId.get(k) ?? []
arr.push(s.actualDuration)
byId.set(k, arr)
}
const out: Record<string, { count: number; total: number; max: number; p50: number; p95: number }> = {}
for (const [k, arr] of byId) {
arr.sort((a, b) => a - b)
const total = arr.reduce((a, b) => a + b, 0)
out[k] = {
count: arr.length,
total: Math.round(total * 100) / 100,
max: Math.round(arr[arr.length - 1] * 100) / 100,
p50: Math.round(arr[Math.floor(arr.length * 0.5)] * 100) / 100,
p95: Math.round(arr[Math.floor(arr.length * 0.95)] * 100) / 100,
}
}
return out
},
}
}
const onRender: ProfilerOnRenderCallback = (id, phase, actualDuration, baseDuration, startTime, commitTime) => {
const probe = typeof window !== 'undefined' ? window.__PERF_PROBE__ : undefined
if (!probe || !probe.enabled) return
probe.samples.push({ id, phase, actualDuration, baseDuration, startTime, commitTime })
if (probe.samples.length > 5000) probe.samples.splice(0, probe.samples.length - 5000)
}
if (typeof window !== 'undefined' && !window.__PERF_DRIVE__) {
// Synthetic stream driver — pushes tokens through the live $messages atom so the
// assistant-ui runtime + react tree sees them exactly as a real LLM stream would.
// Used by scripts/measure-real-stream.mjs when no live LLM credit is available.
let baseline: ReturnType<typeof $messages.get> | null = null
let activeHandle: SyntheticDriverHandle | null = null
const stop = () => {
activeHandle = null
setBusy(false)
}
window.__PERF_DRIVE__ = {
snapshotMsgs: () => $messages.get().length,
reset: () => {
activeHandle?.stop()
if (baseline) setMessages(baseline)
baseline = null
setBusy(false)
},
stream: ({ chunk = 'word ', intervalMs = 16, totalTokens = 400 } = {}) => {
activeHandle?.stop()
const current = $messages.get()
if (!baseline) baseline = current
const msgId = `synthetic-${Date.now()}`
// Seed an empty assistant message — assistant-ui will see it grow.
setMessages([
...current,
{
id: msgId,
role: 'assistant',
parts: [{ type: 'text', text: '' }],
timestamp: Date.now(),
pending: true
}
])
setBusy(true)
let pushed = 0
let timer: ReturnType<typeof setTimeout> | null = null
const handle: SyntheticDriverHandle = {
stop: () => {
if (timer) clearTimeout(timer)
timer = null
activeHandle = null
// Mark message finalized.
setMessages(prev =>
prev.map(m =>
m.id === msgId
? { ...m, pending: false }
: m
)
)
setBusy(false)
}
}
activeHandle = handle
const tick = () => {
if (activeHandle !== handle) return
if (pushed >= totalTokens) {
handle.stop()
return
}
pushed += 1
setMessages(prev =>
prev.map(m => {
if (m.id !== msgId) return m
const head = m.parts.slice(0, -1)
const last = m.parts.at(-1)
const lastText = last && last.type === 'text' ? last.text : ''
return {
...m,
parts: [...head, { type: 'text', text: lastText + chunk }]
}
})
)
timer = setTimeout(tick, intervalMs)
}
timer = setTimeout(tick, intervalMs)
return handle
}
}
// Suppress dead-import warning.
void stop
}
export function PerfProbe({ id, children }: { id: string; children: ReactNode }) {
return (
<Profiler id={id} onRender={onRender}>
{children}
</Profiler>
)
}
+9
View File
@@ -12,6 +12,15 @@ import { ThemeProvider } from './themes/context'
installClipboardShim()
// Dev-only: install __PERF_DRIVE__ + __PERF_PROBE__ on window so the
// scripts/ harnesses can drive a synthetic stream + record render cost.
// Tree-shaken out of production builds. (Uses MODE rather than DEV because
// our Vite setup currently bundles with PROD=true even in `vite dev`; see
// scripts/dev-no-hmr.mjs for the surrounding workarounds.)
if (import.meta.env.MODE !== 'production') {
import('./app/chat/perf-probe')
}
const queryClient = new QueryClient({
defaultOptions: {
queries: {