diff --git a/jetson/ros2_ws/src/saltybot_bringup/config/rosbridge_params.yaml b/jetson/ros2_ws/src/saltybot_bringup/config/rosbridge_params.yaml index 47bbf89..40ae7a3 100644 --- a/jetson/ros2_ws/src/saltybot_bringup/config/rosbridge_params.yaml +++ b/jetson/ros2_ws/src/saltybot_bringup/config/rosbridge_params.yaml @@ -40,6 +40,11 @@ rosbridge_websocket: "/person/target", "/person/detections", "/camera/*/image_raw/compressed", + "/camera/depth/image_rect_raw/compressed", + "/camera/panoramic/compressed", + "/social/faces/detections", + "/social/gestures", + "/social/scene/objects", "/scan", "/cmd_vel", "/saltybot/imu", diff --git a/jetson/ros2_ws/src/saltybot_bringup/launch/rosbridge.launch.py b/jetson/ros2_ws/src/saltybot_bringup/launch/rosbridge.launch.py index 31bbd8c..fefe543 100644 --- a/jetson/ros2_ws/src/saltybot_bringup/launch/rosbridge.launch.py +++ b/jetson/ros2_ws/src/saltybot_bringup/launch/rosbridge.launch.py @@ -94,4 +94,33 @@ def generate_launch_description(): for name in _CAMERAS ] - return LaunchDescription([rosbridge] + republishers) + # ── D435i colour republisher (Issue #177) ──────────────────────────────── + d435i_color = Node( + package='image_transport', + executable='republish', + name='compress_d435i_color', + arguments=['raw', 'compressed'], + remappings=[ + ('in', '/camera/color/image_raw'), + ('out/compressed', '/camera/color/image_raw/compressed'), + ], + parameters=[{'compressed.jpeg_quality': _JPEG_QUALITY}], + output='screen', + ) + + # ── D435i depth republisher (Issue #177) ───────────────────────────────── + # Depth stream as compressedDepth (PNG16) — preserves uint16 depth values. + # Browser displays as greyscale PNG (darker = closer). + d435i_depth = Node( + package='image_transport', + executable='republish', + name='compress_d435i_depth', + arguments=['raw', 'compressedDepth'], + remappings=[ + ('in', '/camera/depth/image_rect_raw'), + ('out/compressedDepth', '/camera/depth/image_rect_raw/compressed'), + ], + output='screen', + ) + + return LaunchDescription([rosbridge] + republishers + [d435i_color, d435i_depth]) diff --git a/ui/social-bot/src/App.jsx b/ui/social-bot/src/App.jsx index 6d6e65a..e614f32 100644 --- a/ui/social-bot/src/App.jsx +++ b/ui/social-bot/src/App.jsx @@ -5,13 +5,16 @@ * Status | Faces | Conversation | Personality | Navigation * * Telemetry tabs (issue #126): - * IMU | Battery | Motors | Map | Control | Health + * IMU | Battery | Motors | Map | Control | Health | Cameras * * Fleet tabs (issue #139): * Fleet (self-contained via useFleet) * * Mission tabs (issue #145): * Missions (waypoint editor, route builder, geofence, schedule, execute) + * + * Camera viewer (issue #177): + * CSI × 4 + D435i RGB/depth + panoramic, detection overlays, recording */ import { useState, useCallback } from 'react'; @@ -41,6 +44,9 @@ import { MissionPlanner } from './components/MissionPlanner.jsx'; // Settings panel (issue #160) import { SettingsPanel } from './components/SettingsPanel.jsx'; +// Camera viewer (issue #177) +import { CameraViewer } from './components/CameraViewer.jsx'; + const TAB_GROUPS = [ { label: 'SOCIAL', @@ -63,6 +69,7 @@ const TAB_GROUPS = [ { id: 'map', label: 'Map', }, { id: 'control', label: 'Control', }, { id: 'health', label: 'Health', }, + { id: 'cameras', label: 'Cameras', }, ], }, { @@ -206,6 +213,7 @@ export default function App() { {activeTab === 'map' && } {activeTab === 'control' && } {activeTab === 'health' && } + {activeTab === 'cameras' && } {activeTab === 'fleet' && } {activeTab === 'missions' && } diff --git a/ui/social-bot/src/components/CameraViewer.jsx b/ui/social-bot/src/components/CameraViewer.jsx new file mode 100644 index 0000000..b0d3695 --- /dev/null +++ b/ui/social-bot/src/components/CameraViewer.jsx @@ -0,0 +1,671 @@ +/** + * CameraViewer.jsx — Live camera stream viewer (Issue #177). + * + * Features: + * - 7 cameras: front/left/rear/right (CSI), D435i RGB/depth, panoramic + * - Detection overlays: face boxes + names, gesture icons, scene object labels + * - 360° panoramic equirect viewer with mouse drag pan + * - One-click recording (MP4/WebM) + download + * - Snapshot to PNG with annotations + timestamp + * - Picture-in-picture (up to 3 pinned cameras) + * - Per-camera FPS indicator + adaptive quality badge + * + * Topics consumed: + * /camera//image_raw/compressed sensor_msgs/CompressedImage + * /camera/color/image_raw/compressed sensor_msgs/CompressedImage (D435i) + * /camera/depth/image_rect_raw/compressed sensor_msgs/CompressedImage (D435i) + * /camera/panoramic/compressed sensor_msgs/CompressedImage + * /social/faces/detections saltybot_social_msgs/FaceDetectionArray + * /social/gestures saltybot_social_msgs/GestureArray + * /social/scene/objects saltybot_scene_msgs/SceneObjectArray + */ + +import { useEffect, useRef, useState, useCallback } from 'react'; +import { useCamera, CAMERAS, CAMERA_BY_ID, CAMERA_BY_ROS_ID } from '../hooks/useCamera.js'; + +// ── Constants ───────────────────────────────────────────────────────────────── + +const GESTURE_ICONS = { + wave: '👋', + point: '👆', + stop_palm: '✋', + thumbs_up: '👍', + thumbs_down: '👎', + come_here: '🤏', + follow: '☞', + arms_up: '🙌', + crouch: '⬇', + arms_spread: '↔', +}; + +const HAZARD_COLORS = { + 1: '#f59e0b', // stairs — amber + 2: '#ef4444', // drop — red + 3: '#60a5fa', // wet floor — blue + 4: '#a855f7', // glass door — purple + 5: '#f97316', // pet — orange +}; + +// ── Detection overlay drawing helpers ───────────────────────────────────────── + +function drawFaceBoxes(ctx, faces, scaleX, scaleY) { + for (const face of faces) { + const x = face.bbox_x * scaleX; + const y = face.bbox_y * scaleY; + const w = face.bbox_w * scaleX; + const h = face.bbox_h * scaleY; + + const isKnown = face.person_name && face.person_name !== 'unknown'; + ctx.strokeStyle = isKnown ? '#06b6d4' : '#f59e0b'; + ctx.lineWidth = 2; + ctx.shadowBlur = 6; + ctx.shadowColor = ctx.strokeStyle; + ctx.strokeRect(x, y, w, h); + ctx.shadowBlur = 0; + + // Corner accent marks + const cLen = 8; + ctx.lineWidth = 3; + [[x,y,1,1],[x+w,y,-1,1],[x,y+h,1,-1],[x+w,y+h,-1,-1]].forEach(([cx,cy,dx,dy]) => { + ctx.beginPath(); + ctx.moveTo(cx, cy + dy * cLen); + ctx.lineTo(cx, cy); + ctx.lineTo(cx + dx * cLen, cy); + ctx.stroke(); + }); + + // Label + const label = isKnown + ? `${face.person_name} ${(face.recognition_score * 100).toFixed(0)}%` + : `face #${face.face_id}`; + ctx.font = 'bold 11px monospace'; + const tw = ctx.measureText(label).width; + ctx.fillStyle = isKnown ? 'rgba(6,182,212,0.8)' : 'rgba(245,158,11,0.8)'; + ctx.fillRect(x, y - 16, tw + 6, 16); + ctx.fillStyle = '#000'; + ctx.fillText(label, x + 3, y - 4); + } +} + +function drawGestureIcons(ctx, gestures, activeCamId, scaleX, scaleY) { + for (const g of gestures) { + // Only show gestures from the currently viewed camera + const cam = CAMERA_BY_ROS_ID[g.camera_id]; + if (!cam || cam.cameraId !== activeCamId) continue; + + const x = g.hand_x * ctx.canvas.width; + const y = g.hand_y * ctx.canvas.height; + const icon = GESTURE_ICONS[g.gesture_type] ?? '?'; + + ctx.font = '24px serif'; + ctx.shadowBlur = 8; + ctx.shadowColor = '#f97316'; + ctx.fillText(icon, x - 12, y + 8); + ctx.shadowBlur = 0; + + ctx.font = 'bold 10px monospace'; + ctx.fillStyle = '#f97316'; + const label = g.gesture_type; + ctx.fillText(label, x - ctx.measureText(label).width / 2, y + 22); + } +} + +function drawSceneObjects(ctx, objects, scaleX, scaleY) { + for (const obj of objects) { + // vision_msgs/BoundingBox2D: center_x, center_y, size_x, size_y + const bb = obj.bbox; + const cx = bb?.center?.x ?? bb?.center_x; + const cy = bb?.center?.y ?? bb?.center_y; + const sw = bb?.size_x ?? 0; + const sh = bb?.size_y ?? 0; + if (cx == null) continue; + + const x = (cx - sw / 2) * scaleX; + const y = (cy - sh / 2) * scaleY; + const w = sw * scaleX; + const h = sh * scaleY; + + const color = HAZARD_COLORS[obj.hazard_type] ?? '#22c55e'; + ctx.strokeStyle = color; + ctx.lineWidth = 1.5; + ctx.setLineDash([4, 3]); + ctx.strokeRect(x, y, w, h); + ctx.setLineDash([]); + + const dist = obj.distance_m > 0 ? ` ${obj.distance_m.toFixed(1)}m` : ''; + const label = `${obj.class_name}${dist}`; + ctx.font = '10px monospace'; + const tw = ctx.measureText(label).width; + ctx.fillStyle = `${color}cc`; + ctx.fillRect(x, y + h, tw + 4, 14); + ctx.fillStyle = '#000'; + ctx.fillText(label, x + 2, y + h + 11); + } +} + +// ── Overlay canvas ───────────────────────────────────────────────────────────── + +function OverlayCanvas({ faces, gestures, sceneObjects, activeCam, containerW, containerH }) { + const canvasRef = useRef(null); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = canvas.getContext('2d'); + ctx.clearRect(0, 0, canvas.width, canvas.height); + + if (!activeCam) return; + + const scaleX = canvas.width / (activeCam.width || 640); + const scaleY = canvas.height / (activeCam.height || 480); + + // Draw overlays: only for front camera (face + gesture source) + if (activeCam.id === 'front') { + drawFaceBoxes(ctx, faces, scaleX, scaleY); + } + if (!activeCam.isPanoramic) { + drawGestureIcons(ctx, gestures, activeCam.cameraId, scaleX, scaleY); + } + if (activeCam.id === 'color') { + drawSceneObjects(ctx, sceneObjects, scaleX, scaleY); + } + }, [faces, gestures, sceneObjects, activeCam]); + + return ( + + ); +} + +// ── Panoramic equirect viewer ────────────────────────────────────────────────── + +function PanoViewer({ frameUrl }) { + const canvasRef = useRef(null); + const azRef = useRef(0); // 0–1920px offset + const dragRef = useRef(null); + const imgRef = useRef(null); + + const draw = useCallback(() => { + const canvas = canvasRef.current; + const img = imgRef.current; + if (!canvas || !img || !img.complete) return; + + const ctx = canvas.getContext('2d'); + const W = canvas.width; + const H = canvas.height; + const iW = img.naturalWidth; // 1920 + const iH = img.naturalHeight; // 960 + const vW = iW / 2; // viewport = 50% of equirect width + const vH = Math.round((H / W) * vW); + const vY = Math.round((iH - vH) / 2); + const off = Math.round(azRef.current) % iW; + + ctx.clearRect(0, 0, W, H); + + // Draw left segment + const srcX1 = off; + const srcW1 = Math.min(vW, iW - off); + const dstW1 = Math.round((srcW1 / vW) * W); + if (dstW1 > 0) { + ctx.drawImage(img, srcX1, vY, srcW1, vH, 0, 0, dstW1, H); + } + + // Draw wrapped right segment (if viewport crosses 0°) + if (srcW1 < vW) { + const srcX2 = 0; + const srcW2 = vW - srcW1; + const dstX2 = dstW1; + const dstW2 = W - dstW1; + ctx.drawImage(img, srcX2, vY, srcW2, vH, dstX2, 0, dstW2, H); + } + + // Compass badge + const azDeg = Math.round((azRef.current / iW) * 360); + ctx.fillStyle = 'rgba(0,0,0,0.5)'; + ctx.fillRect(W - 58, 6, 52, 18); + ctx.fillStyle = '#06b6d4'; + ctx.font = 'bold 11px monospace'; + ctx.fillText(`${azDeg}°`, W - 52, 19); + }, []); + + // Load image when URL changes + useEffect(() => { + if (!frameUrl) return; + const img = new Image(); + img.onload = draw; + img.src = frameUrl; + imgRef.current = img; + }, [frameUrl, draw]); + + // Re-draw when azimuth changes + const onMouseDown = e => { dragRef.current = e.clientX; }; + const onMouseMove = e => { + if (dragRef.current == null) return; + const dx = e.clientX - dragRef.current; + dragRef.current = e.clientX; + azRef.current = ((azRef.current - dx * 2) % 1920 + 1920) % 1920; + draw(); + }; + const onMouseUp = () => { dragRef.current = null; }; + + const onTouchStart = e => { dragRef.current = e.touches[0].clientX; }; + const onTouchMove = e => { + if (dragRef.current == null) return; + const dx = e.touches[0].clientX - dragRef.current; + dragRef.current = e.touches[0].clientX; + azRef.current = ((azRef.current - dx * 2) % 1920 + 1920) % 1920; + draw(); + }; + + return ( + { dragRef.current = null; }} + /> + ); +} + +// ── PiP mini window ──────────────────────────────────────────────────────────── + +function PiPWindow({ cam, frameUrl, fps, onClose, index }) { + const positions = [ + 'bottom-2 left-2', + 'bottom-2 left-40', + 'bottom-2 left-[18rem]', + ]; + return ( +
+
+ {cam.label} +
+ {fps}fps + +
+
+ {frameUrl ? ( + {cam.label} + ) : ( +
+ no signal +
+ )} +
+ ); +} + +// ── Camera selector strip ────────────────────────────────────────────────────── + +function CameraStrip({ cameras, activeId, pipList, frames, fps, onSelect, onTogglePip }) { + return ( +
+ {cameras.map(cam => { + const hasFrame = !!frames[cam.id]; + const camFps = fps[cam.id] ?? 0; + const isActive = activeId === cam.id; + const isPip = pipList.includes(cam.id); + + return ( +
+ + + {/* PiP pin button — only when NOT the active camera */} + {!isActive && ( + + )} +
+ ); + })} +
+ ); +} + +// ── Recording bar ────────────────────────────────────────────────────────────── + +function RecordingBar({ recording, recSeconds, onStart, onStop, onSnapshot, overlayRef }) { + const fmtTime = s => `${String(Math.floor(s / 60)).padStart(2, '0')}:${String(s % 60).padStart(2, '0')}`; + + return ( +
+ {!recording ? ( + + ) : ( + + )} + + + + {recording && ( + + ● RECORDING {fmtTime(recSeconds)} + + )} +
+ ); +} + +// ── Main component ───────────────────────────────────────────────────────────── + +export function CameraViewer({ subscribe }) { + const { + cameras, frames, fps, + activeId, setActiveId, + pipList, togglePip, + recording, recSeconds, + startRecording, stopRecording, + takeSnapshot, + } = useCamera({ subscribe }); + + // ── Detection state ───────────────────────────────────────────────────────── + const [faces, setFaces] = useState([]); + const [gestures, setGestures] = useState([]); + const [sceneObjects, setSceneObjects] = useState([]); + + const [showOverlay, setShowOverlay] = useState(true); + const [overlayMode, setOverlayMode] = useState('all'); // 'all' | 'faces' | 'gestures' | 'objects' | 'off' + + const overlayCanvasRef = useRef(null); + + // Subscribe to detection topics + useEffect(() => { + if (!subscribe) return; + const u1 = subscribe('/social/faces/detections', 'saltybot_social_msgs/FaceDetectionArray', msg => { + setFaces(msg.faces ?? []); + }); + const u2 = subscribe('/social/gestures', 'saltybot_social_msgs/GestureArray', msg => { + setGestures(msg.gestures ?? []); + }); + const u3 = subscribe('/social/scene/objects', 'saltybot_scene_msgs/SceneObjectArray', msg => { + setSceneObjects(msg.objects ?? []); + }); + return () => { u1?.(); u2?.(); u3?.(); }; + }, [subscribe]); + + const activeCam = CAMERA_BY_ID[activeId]; + const activeFrame = frames[activeId]; + + // Filter overlay data based on mode + const visibleFaces = (overlayMode === 'all' || overlayMode === 'faces') ? faces : []; + const visibleGestures = (overlayMode === 'all' || overlayMode === 'gestures') ? gestures : []; + const visibleObjects = (overlayMode === 'all' || overlayMode === 'objects') ? sceneObjects : []; + + // ── Container size tracking (for overlay canvas sizing) ──────────────────── + const containerRef = useRef(null); + const [containerSize, setContainerSize] = useState({ w: 640, h: 480 }); + + useEffect(() => { + if (!containerRef.current) return; + const ro = new ResizeObserver(entries => { + const e = entries[0]; + setContainerSize({ w: Math.round(e.contentRect.width), h: Math.round(e.contentRect.height) }); + }); + ro.observe(containerRef.current); + return () => ro.disconnect(); + }, []); + + // ── Quality badge ────────────────────────────────────────────────────────── + const camFps = fps[activeId] ?? 0; + const quality = camFps >= 13 ? 'FULL' : camFps >= 8 ? 'GOOD' : camFps > 0 ? 'LOW' : 'NO SIGNAL'; + const qualColor = camFps >= 13 ? 'text-green-500' : camFps >= 8 ? 'text-amber-500' : camFps > 0 ? 'text-red-500' : 'text-gray-700'; + + return ( +
+ + {/* ── Camera strip ── */} +
+
+
CAMERA SELECT
+ {quality} {camFps > 0 ? `${camFps}fps` : ''} +
+ +
+ + {/* ── Main viewer ── */} +
+ + {/* Viewer toolbar */} +
+
+ {activeCam?.label ?? '—'} + {activeCam?.isDepth && ( + DEPTH · greyscale + )} + {activeCam?.isPanoramic && ( + 360° · drag to pan + )} +
+ + {/* Overlay mode selector */} +
+ {['off','faces','gestures','objects','all'].map(mode => ( + + ))} +
+
+ + {/* Image + overlay */} +
+ {activeCam?.isPanoramic ? ( + + ) : activeFrame ? ( + {activeCam?.label + ) : ( +
+
+
📷
+
Waiting for {activeCam?.label ?? '—'}…
+
{activeCam?.topic}
+
+
+ )} + + {/* Detection overlay canvas */} + {overlayMode !== 'off' && !activeCam?.isPanoramic && ( + + )} + + {/* PiP windows */} + {pipList.map((id, idx) => { + const cam = CAMERA_BY_ID[id]; + if (!cam) return null; + return ( + togglePip(id)} + /> + ); + })} +
+
+ + {/* ── Recording controls ── */} +
+
+
CAPTURE
+
+ +
+ Recording saves as MP4/WebM to your Downloads. + Snapshot includes detection overlay + timestamp. +
+
+ + {/* ── Detection status ── */} +
+
+
FACES
+
0 ? 'text-cyan-400' : 'text-gray-700'}`}> + {faces.length > 0 ? `${faces.length} detected` : 'none'} +
+ {faces.slice(0, 2).map((f, i) => ( +
+ {f.person_name && f.person_name !== 'unknown' + ? `↳ ${f.person_name}` + : `↳ unknown #${f.face_id}`} +
+ ))} +
+ +
+
GESTURES
+
0 ? 'text-amber-400' : 'text-gray-700'}`}> + {gestures.length > 0 ? `${gestures.length} active` : 'none'} +
+ {gestures.slice(0, 2).map((g, i) => { + const icon = GESTURE_ICONS[g.gesture_type] ?? '?'; + return ( +
+ {icon} {g.gesture_type} cam{g.camera_id} +
+ ); + })} +
+ +
+
OBJECTS
+
0 ? 'text-green-400' : 'text-gray-700'}`}> + {sceneObjects.length > 0 ? `${sceneObjects.length} objects` : 'none'} +
+ {sceneObjects + .filter(o => o.hazard_type > 0) + .slice(0, 2) + .map((o, i) => ( +
⚠ {o.class_name}
+ )) + } + {sceneObjects.filter(o => o.hazard_type === 0).slice(0, 2).map((o, i) => ( +
+ {o.class_name} {o.distance_m > 0 ? `${o.distance_m.toFixed(1)}m` : ''} +
+ ))} +
+
+ + {/* ── Legend ── */} +
+
+
+ Known face +
+
+
+ Unknown face +
+
+ 👆 Gesture +
+
+
+ Object +
+
+
+ Hazard +
+
+ ⊕ pin = PiP · overlay: {overlayMode} +
+
+
+ ); +} diff --git a/ui/social-bot/src/hooks/useCamera.js b/ui/social-bot/src/hooks/useCamera.js new file mode 100644 index 0000000..7ad5bd9 --- /dev/null +++ b/ui/social-bot/src/hooks/useCamera.js @@ -0,0 +1,325 @@ +/** + * useCamera.js — Multi-camera stream manager (Issue #177). + * + * Subscribes to sensor_msgs/CompressedImage topics via rosbridge. + * Decodes base64 JPEG/PNG → data URL for / display. + * Tracks per-camera FPS. Manages MediaRecorder for recording + snapshots. + * + * Camera sources: + * front / left / rear / right — 4× CSI IMX219, 640×480 + * topic: /camera//image_raw/compressed + * color — D435i RGB, 640×480 + * topic: /camera/color/image_raw/compressed + * depth — D435i depth, 640×480 greyscale (PNG16) + * topic: /camera/depth/image_rect_raw/compressed + * panoramic — equirect stitch 1920×960 + * topic: /camera/panoramic/compressed + */ + +import { useState, useEffect, useRef, useCallback } from 'react'; + +// ── Camera catalogue ────────────────────────────────────────────────────────── + +export const CAMERAS = [ + { + id: 'front', + label: 'Front', + shortLabel: 'F', + topic: '/camera/front/image_raw/compressed', + msgType: 'sensor_msgs/CompressedImage', + cameraId: 0, // matches gesture_node camera_id + width: 640, height: 480, + }, + { + id: 'left', + label: 'Left', + shortLabel: 'L', + topic: '/camera/left/image_raw/compressed', + msgType: 'sensor_msgs/CompressedImage', + cameraId: 1, + width: 640, height: 480, + }, + { + id: 'rear', + label: 'Rear', + shortLabel: 'R', + topic: '/camera/rear/image_raw/compressed', + msgType: 'sensor_msgs/CompressedImage', + cameraId: 2, + width: 640, height: 480, + }, + { + id: 'right', + label: 'Right', + shortLabel: 'Rt', + topic: '/camera/right/image_raw/compressed', + msgType: 'sensor_msgs/CompressedImage', + cameraId: 3, + width: 640, height: 480, + }, + { + id: 'color', + label: 'D435i RGB', + shortLabel: 'D', + topic: '/camera/color/image_raw/compressed', + msgType: 'sensor_msgs/CompressedImage', + cameraId: 4, + width: 640, height: 480, + }, + { + id: 'depth', + label: 'Depth', + shortLabel: '≋', + topic: '/camera/depth/image_rect_raw/compressed', + msgType: 'sensor_msgs/CompressedImage', + cameraId: 5, + width: 640, height: 480, + isDepth: true, + }, + { + id: 'panoramic', + label: 'Panoramic', + shortLabel: '360', + topic: '/camera/panoramic/compressed', + msgType: 'sensor_msgs/CompressedImage', + cameraId: -1, + width: 1920, height: 960, + isPanoramic: true, + }, +]; + +export const CAMERA_BY_ID = Object.fromEntries(CAMERAS.map(c => [c.id, c])); +export const CAMERA_BY_ROS_ID = Object.fromEntries( + CAMERAS.filter(c => c.cameraId >= 0).map(c => [c.cameraId, c]) +); + +const TARGET_FPS = 15; +const FPS_INTERVAL = 1000; // ms between FPS counter resets + +// ── Hook ────────────────────────────────────────────────────────────────────── + +export function useCamera({ subscribe } = {}) { + const [frames, setFrames] = useState(() => + Object.fromEntries(CAMERAS.map(c => [c.id, null])) + ); + const [fps, setFps] = useState(() => + Object.fromEntries(CAMERAS.map(c => [c.id, 0])) + ); + const [activeId, setActiveId] = useState('front'); + const [pipList, setPipList] = useState([]); // up to 3 extra camera ids + const [recording, setRecording] = useState(false); + const [recSeconds, setRecSeconds] = useState(0); + + // ── Refs (not state — no re-render needed) ───────────────────────────────── + const countRef = useRef(Object.fromEntries(CAMERAS.map(c => [c.id, 0]))); + const mediaRecRef = useRef(null); + const chunksRef = useRef([]); + const recTimerRef = useRef(null); + const recordCanvas = useRef(null); // hidden canvas used for recording + const recAnimRef = useRef(null); // rAF handle for record-canvas loop + const latestFrameRef = useRef(Object.fromEntries(CAMERAS.map(c => [c.id, null]))); + const latestTsRef = useRef(Object.fromEntries(CAMERAS.map(c => [c.id, 0]))); + + // ── FPS counter ──────────────────────────────────────────────────────────── + useEffect(() => { + const timer = setInterval(() => { + setFps({ ...countRef.current }); + const reset = Object.fromEntries(CAMERAS.map(c => [c.id, 0])); + countRef.current = reset; + }, FPS_INTERVAL); + return () => clearInterval(timer); + }, []); + + // ── Subscribe all camera topics ──────────────────────────────────────────── + useEffect(() => { + if (!subscribe) return; + + const unsubs = CAMERAS.map(cam => { + let lastTs = 0; + const interval = Math.floor(1000 / TARGET_FPS); // client-side 15fps gate + + return subscribe(cam.topic, cam.msgType, (msg) => { + const now = Date.now(); + if (now - lastTs < interval) return; // drop frames > 15fps + lastTs = now; + + const fmt = msg.format || 'jpeg'; + const mime = fmt.includes('png') || fmt.includes('16UC') ? 'image/png' : 'image/jpeg'; + const dataUrl = `data:${mime};base64,${msg.data}`; + + latestFrameRef.current[cam.id] = dataUrl; + latestTsRef.current[cam.id] = now; + countRef.current[cam.id] = (countRef.current[cam.id] ?? 0) + 1; + + setFrames(prev => ({ ...prev, [cam.id]: dataUrl })); + }); + }); + + return () => unsubs.forEach(fn => fn?.()); + }, [subscribe]); + + // ── Create hidden record canvas ──────────────────────────────────────────── + useEffect(() => { + const c = document.createElement('canvas'); + c.width = 640; + c.height = 480; + c.style.display = 'none'; + document.body.appendChild(c); + recordCanvas.current = c; + return () => { c.remove(); }; + }, []); + + // ── Draw loop for record canvas ──────────────────────────────────────────── + // Runs at TARGET_FPS when recording — draws active frame to hidden canvas + const startRecordLoop = useCallback(() => { + const canvas = recordCanvas.current; + if (!canvas) return; + + const step = () => { + const cam = CAMERA_BY_ID[activeId]; + const src = latestFrameRef.current[activeId]; + const ctx = canvas.getContext('2d'); + + if (!cam || !src) { + recAnimRef.current = requestAnimationFrame(step); + return; + } + + // Resize canvas to match source + if (canvas.width !== cam.width || canvas.height !== cam.height) { + canvas.width = cam.width; + canvas.height = cam.height; + } + + const img = new Image(); + img.onload = () => { + ctx.drawImage(img, 0, 0, canvas.width, canvas.height); + }; + img.src = src; + + recAnimRef.current = setTimeout(step, Math.floor(1000 / TARGET_FPS)); + }; + + recAnimRef.current = setTimeout(step, 0); + }, [activeId]); + + const stopRecordLoop = useCallback(() => { + if (recAnimRef.current) { + clearTimeout(recAnimRef.current); + cancelAnimationFrame(recAnimRef.current); + recAnimRef.current = null; + } + }, []); + + // ── Recording ────────────────────────────────────────────────────────────── + + const startRecording = useCallback(() => { + const canvas = recordCanvas.current; + if (!canvas || recording) return; + + startRecordLoop(); + + const stream = canvas.captureStream(TARGET_FPS); + const mimeType = + MediaRecorder.isTypeSupported('video/mp4') ? 'video/mp4' : + MediaRecorder.isTypeSupported('video/webm;codecs=vp9') ? 'video/webm;codecs=vp9' : + MediaRecorder.isTypeSupported('video/webm;codecs=vp8') ? 'video/webm;codecs=vp8' : + 'video/webm'; + + chunksRef.current = []; + const mr = new MediaRecorder(stream, { mimeType, videoBitsPerSecond: 2_500_000 }); + mr.ondataavailable = e => { if (e.data?.size > 0) chunksRef.current.push(e.data); }; + mr.start(200); + mediaRecRef.current = mr; + setRecording(true); + setRecSeconds(0); + recTimerRef.current = setInterval(() => setRecSeconds(s => s + 1), 1000); + }, [recording, startRecordLoop]); + + const stopRecording = useCallback(() => { + const mr = mediaRecRef.current; + if (!mr || mr.state === 'inactive') return; + + mr.onstop = () => { + const ext = mr.mimeType.includes('mp4') ? 'mp4' : 'webm'; + const blob = new Blob(chunksRef.current, { type: mr.mimeType }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `saltybot-${activeId}-${Date.now()}.${ext}`; + a.click(); + URL.revokeObjectURL(url); + }; + + mr.stop(); + stopRecordLoop(); + clearInterval(recTimerRef.current); + setRecording(false); + }, [activeId, stopRecordLoop]); + + // ── Snapshot ─────────────────────────────────────────────────────────────── + + const takeSnapshot = useCallback((overlayCanvasEl) => { + const src = latestFrameRef.current[activeId]; + if (!src) return; + + const cam = CAMERA_BY_ID[activeId]; + const canvas = document.createElement('canvas'); + canvas.width = cam.width; + canvas.height = cam.height; + const ctx = canvas.getContext('2d'); + + const img = new Image(); + img.onload = () => { + ctx.drawImage(img, 0, 0, canvas.width, canvas.height); + + // Composite detection overlay if provided + if (overlayCanvasEl) { + ctx.drawImage(overlayCanvasEl, 0, 0, canvas.width, canvas.height); + } + + // Timestamp watermark + ctx.fillStyle = 'rgba(0,0,0,0.5)'; + ctx.fillRect(0, canvas.height - 20, canvas.width, 20); + ctx.fillStyle = '#06b6d4'; + ctx.font = '11px monospace'; + ctx.fillText(`SALTYBOT ${cam.label} ${new Date().toISOString()}`, 8, canvas.height - 6); + + canvas.toBlob(blob => { + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `saltybot-snap-${activeId}-${Date.now()}.png`; + a.click(); + URL.revokeObjectURL(url); + }, 'image/png'); + }; + img.src = src; + }, [activeId]); + + // ── PiP management ───────────────────────────────────────────────────────── + + const togglePip = useCallback(id => { + setPipList(prev => { + if (prev.includes(id)) return prev.filter(x => x !== id); + const next = [...prev, id].filter(x => x !== activeId); + return next.slice(-3); // max 3 PIPs + }); + }, [activeId]); + + // Remove PiP if it becomes the active camera + useEffect(() => { + setPipList(prev => prev.filter(id => id !== activeId)); + }, [activeId]); + + return { + cameras: CAMERAS, + frames, + fps, + activeId, setActiveId, + pipList, togglePip, + recording, recSeconds, + startRecording, stopRecording, + takeSnapshot, + }; +}