/** * CameraViewer.jsx — Live camera stream viewer (Issue #177). * * Features: * - 7 cameras: front/left/rear/right (CSI), D435i RGB/depth, panoramic * - Detection overlays: face boxes + names, gesture icons, scene object labels * - 360° panoramic equirect viewer with mouse drag pan * - One-click recording (MP4/WebM) + download * - Snapshot to PNG with annotations + timestamp * - Picture-in-picture (up to 3 pinned cameras) * - Per-camera FPS indicator + adaptive quality badge * * Topics consumed: * /camera//image_raw/compressed sensor_msgs/CompressedImage * /camera/color/image_raw/compressed sensor_msgs/CompressedImage (D435i) * /camera/depth/image_rect_raw/compressed sensor_msgs/CompressedImage (D435i) * /camera/panoramic/compressed sensor_msgs/CompressedImage * /social/faces/detections saltybot_social_msgs/FaceDetectionArray * /social/gestures saltybot_social_msgs/GestureArray * /social/scene/objects saltybot_scene_msgs/SceneObjectArray */ import { useEffect, useRef, useState, useCallback } from 'react'; import { useCamera, CAMERAS, CAMERA_BY_ID, CAMERA_BY_ROS_ID } from '../hooks/useCamera.js'; // ── Constants ───────────────────────────────────────────────────────────────── const GESTURE_ICONS = { wave: '👋', point: '👆', stop_palm: '✋', thumbs_up: '👍', thumbs_down: '👎', come_here: '🤏', follow: '☞', arms_up: '🙌', crouch: '⬇', arms_spread: '↔', }; const HAZARD_COLORS = { 1: '#f59e0b', // stairs — amber 2: '#ef4444', // drop — red 3: '#60a5fa', // wet floor — blue 4: '#a855f7', // glass door — purple 5: '#f97316', // pet — orange }; // ── Detection overlay drawing helpers ───────────────────────────────────────── function drawFaceBoxes(ctx, faces, scaleX, scaleY) { for (const face of faces) { const x = face.bbox_x * scaleX; const y = face.bbox_y * scaleY; const w = face.bbox_w * scaleX; const h = face.bbox_h * scaleY; const isKnown = face.person_name && face.person_name !== 'unknown'; ctx.strokeStyle = isKnown ? '#06b6d4' : '#f59e0b'; ctx.lineWidth = 2; ctx.shadowBlur = 6; ctx.shadowColor = ctx.strokeStyle; ctx.strokeRect(x, y, w, h); ctx.shadowBlur = 0; // Corner accent marks const cLen = 8; ctx.lineWidth = 3; [[x,y,1,1],[x+w,y,-1,1],[x,y+h,1,-1],[x+w,y+h,-1,-1]].forEach(([cx,cy,dx,dy]) => { ctx.beginPath(); ctx.moveTo(cx, cy + dy * cLen); ctx.lineTo(cx, cy); ctx.lineTo(cx + dx * cLen, cy); ctx.stroke(); }); // Label const label = isKnown ? `${face.person_name} ${(face.recognition_score * 100).toFixed(0)}%` : `face #${face.face_id}`; ctx.font = 'bold 11px monospace'; const tw = ctx.measureText(label).width; ctx.fillStyle = isKnown ? 'rgba(6,182,212,0.8)' : 'rgba(245,158,11,0.8)'; ctx.fillRect(x, y - 16, tw + 6, 16); ctx.fillStyle = '#000'; ctx.fillText(label, x + 3, y - 4); } } function drawGestureIcons(ctx, gestures, activeCamId, scaleX, scaleY) { for (const g of gestures) { // Only show gestures from the currently viewed camera const cam = CAMERA_BY_ROS_ID[g.camera_id]; if (!cam || cam.cameraId !== activeCamId) continue; const x = g.hand_x * ctx.canvas.width; const y = g.hand_y * ctx.canvas.height; const icon = GESTURE_ICONS[g.gesture_type] ?? '?'; ctx.font = '24px serif'; ctx.shadowBlur = 8; ctx.shadowColor = '#f97316'; ctx.fillText(icon, x - 12, y + 8); ctx.shadowBlur = 0; ctx.font = 'bold 10px monospace'; ctx.fillStyle = '#f97316'; const label = g.gesture_type; ctx.fillText(label, x - ctx.measureText(label).width / 2, y + 22); } } function drawSceneObjects(ctx, objects, scaleX, scaleY) { for (const obj of objects) { // vision_msgs/BoundingBox2D: center_x, center_y, size_x, size_y const bb = obj.bbox; const cx = bb?.center?.x ?? bb?.center_x; const cy = bb?.center?.y ?? bb?.center_y; const sw = bb?.size_x ?? 0; const sh = bb?.size_y ?? 0; if (cx == null) continue; const x = (cx - sw / 2) * scaleX; const y = (cy - sh / 2) * scaleY; const w = sw * scaleX; const h = sh * scaleY; const color = HAZARD_COLORS[obj.hazard_type] ?? '#22c55e'; ctx.strokeStyle = color; ctx.lineWidth = 1.5; ctx.setLineDash([4, 3]); ctx.strokeRect(x, y, w, h); ctx.setLineDash([]); const dist = obj.distance_m > 0 ? ` ${obj.distance_m.toFixed(1)}m` : ''; const label = `${obj.class_name}${dist}`; ctx.font = '10px monospace'; const tw = ctx.measureText(label).width; ctx.fillStyle = `${color}cc`; ctx.fillRect(x, y + h, tw + 4, 14); ctx.fillStyle = '#000'; ctx.fillText(label, x + 2, y + h + 11); } } // ── Overlay canvas ───────────────────────────────────────────────────────────── function OverlayCanvas({ faces, gestures, sceneObjects, activeCam, containerW, containerH }) { const canvasRef = useRef(null); useEffect(() => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext('2d'); ctx.clearRect(0, 0, canvas.width, canvas.height); if (!activeCam) return; const scaleX = canvas.width / (activeCam.width || 640); const scaleY = canvas.height / (activeCam.height || 480); // Draw overlays: only for front camera (face + gesture source) if (activeCam.id === 'front') { drawFaceBoxes(ctx, faces, scaleX, scaleY); } if (!activeCam.isPanoramic) { drawGestureIcons(ctx, gestures, activeCam.cameraId, scaleX, scaleY); } if (activeCam.id === 'color') { drawSceneObjects(ctx, sceneObjects, scaleX, scaleY); } }, [faces, gestures, sceneObjects, activeCam]); return ( ); } // ── Panoramic equirect viewer ────────────────────────────────────────────────── function PanoViewer({ frameUrl }) { const canvasRef = useRef(null); const azRef = useRef(0); // 0–1920px offset const dragRef = useRef(null); const imgRef = useRef(null); const draw = useCallback(() => { const canvas = canvasRef.current; const img = imgRef.current; if (!canvas || !img || !img.complete) return; const ctx = canvas.getContext('2d'); const W = canvas.width; const H = canvas.height; const iW = img.naturalWidth; // 1920 const iH = img.naturalHeight; // 960 const vW = iW / 2; // viewport = 50% of equirect width const vH = Math.round((H / W) * vW); const vY = Math.round((iH - vH) / 2); const off = Math.round(azRef.current) % iW; ctx.clearRect(0, 0, W, H); // Draw left segment const srcX1 = off; const srcW1 = Math.min(vW, iW - off); const dstW1 = Math.round((srcW1 / vW) * W); if (dstW1 > 0) { ctx.drawImage(img, srcX1, vY, srcW1, vH, 0, 0, dstW1, H); } // Draw wrapped right segment (if viewport crosses 0°) if (srcW1 < vW) { const srcX2 = 0; const srcW2 = vW - srcW1; const dstX2 = dstW1; const dstW2 = W - dstW1; ctx.drawImage(img, srcX2, vY, srcW2, vH, dstX2, 0, dstW2, H); } // Compass badge const azDeg = Math.round((azRef.current / iW) * 360); ctx.fillStyle = 'rgba(0,0,0,0.5)'; ctx.fillRect(W - 58, 6, 52, 18); ctx.fillStyle = '#06b6d4'; ctx.font = 'bold 11px monospace'; ctx.fillText(`${azDeg}°`, W - 52, 19); }, []); // Load image when URL changes useEffect(() => { if (!frameUrl) return; const img = new Image(); img.onload = draw; img.src = frameUrl; imgRef.current = img; }, [frameUrl, draw]); // Re-draw when azimuth changes const onMouseDown = e => { dragRef.current = e.clientX; }; const onMouseMove = e => { if (dragRef.current == null) return; const dx = e.clientX - dragRef.current; dragRef.current = e.clientX; azRef.current = ((azRef.current - dx * 2) % 1920 + 1920) % 1920; draw(); }; const onMouseUp = () => { dragRef.current = null; }; const onTouchStart = e => { dragRef.current = e.touches[0].clientX; }; const onTouchMove = e => { if (dragRef.current == null) return; const dx = e.touches[0].clientX - dragRef.current; dragRef.current = e.touches[0].clientX; azRef.current = ((azRef.current - dx * 2) % 1920 + 1920) % 1920; draw(); }; return ( { dragRef.current = null; }} /> ); } // ── PiP mini window ──────────────────────────────────────────────────────────── function PiPWindow({ cam, frameUrl, fps, onClose, index }) { const positions = [ 'bottom-2 left-2', 'bottom-2 left-40', 'bottom-2 left-[18rem]', ]; return (
{cam.label}
{fps}fps
{frameUrl ? ( {cam.label} ) : (
no signal
)}
); } // ── Camera selector strip ────────────────────────────────────────────────────── function CameraStrip({ cameras, activeId, pipList, frames, fps, onSelect, onTogglePip }) { return (
{cameras.map(cam => { const hasFrame = !!frames[cam.id]; const camFps = fps[cam.id] ?? 0; const isActive = activeId === cam.id; const isPip = pipList.includes(cam.id); return (
{/* PiP pin button — only when NOT the active camera */} {!isActive && ( )}
); })}
); } // ── Recording bar ────────────────────────────────────────────────────────────── function RecordingBar({ recording, recSeconds, onStart, onStop, onSnapshot, overlayRef }) { const fmtTime = s => `${String(Math.floor(s / 60)).padStart(2, '0')}:${String(s % 60).padStart(2, '0')}`; return (
{!recording ? ( ) : ( )} {recording && ( ● RECORDING {fmtTime(recSeconds)} )}
); } // ── Main component ───────────────────────────────────────────────────────────── export function CameraViewer({ subscribe }) { const { cameras, frames, fps, activeId, setActiveId, pipList, togglePip, recording, recSeconds, startRecording, stopRecording, takeSnapshot, } = useCamera({ subscribe }); // ── Detection state ───────────────────────────────────────────────────────── const [faces, setFaces] = useState([]); const [gestures, setGestures] = useState([]); const [sceneObjects, setSceneObjects] = useState([]); const [showOverlay, setShowOverlay] = useState(true); const [overlayMode, setOverlayMode] = useState('all'); // 'all' | 'faces' | 'gestures' | 'objects' | 'off' const overlayCanvasRef = useRef(null); // Subscribe to detection topics useEffect(() => { if (!subscribe) return; const u1 = subscribe('/social/faces/detections', 'saltybot_social_msgs/FaceDetectionArray', msg => { setFaces(msg.faces ?? []); }); const u2 = subscribe('/social/gestures', 'saltybot_social_msgs/GestureArray', msg => { setGestures(msg.gestures ?? []); }); const u3 = subscribe('/social/scene/objects', 'saltybot_scene_msgs/SceneObjectArray', msg => { setSceneObjects(msg.objects ?? []); }); return () => { u1?.(); u2?.(); u3?.(); }; }, [subscribe]); const activeCam = CAMERA_BY_ID[activeId]; const activeFrame = frames[activeId]; // Filter overlay data based on mode const visibleFaces = (overlayMode === 'all' || overlayMode === 'faces') ? faces : []; const visibleGestures = (overlayMode === 'all' || overlayMode === 'gestures') ? gestures : []; const visibleObjects = (overlayMode === 'all' || overlayMode === 'objects') ? sceneObjects : []; // ── Container size tracking (for overlay canvas sizing) ──────────────────── const containerRef = useRef(null); const [containerSize, setContainerSize] = useState({ w: 640, h: 480 }); useEffect(() => { if (!containerRef.current) return; const ro = new ResizeObserver(entries => { const e = entries[0]; setContainerSize({ w: Math.round(e.contentRect.width), h: Math.round(e.contentRect.height) }); }); ro.observe(containerRef.current); return () => ro.disconnect(); }, []); // ── Quality badge ────────────────────────────────────────────────────────── const camFps = fps[activeId] ?? 0; const quality = camFps >= 13 ? 'FULL' : camFps >= 8 ? 'GOOD' : camFps > 0 ? 'LOW' : 'NO SIGNAL'; const qualColor = camFps >= 13 ? 'text-green-500' : camFps >= 8 ? 'text-amber-500' : camFps > 0 ? 'text-red-500' : 'text-gray-700'; return (
{/* ── Camera strip ── */}
CAMERA SELECT
{quality} {camFps > 0 ? `${camFps}fps` : ''}
{/* ── Main viewer ── */}
{/* Viewer toolbar */}
{activeCam?.label ?? '—'} {activeCam?.isDepth && ( DEPTH · greyscale )} {activeCam?.isPanoramic && ( 360° · drag to pan )}
{/* Overlay mode selector */}
{['off','faces','gestures','objects','all'].map(mode => ( ))}
{/* Image + overlay */}
{activeCam?.isPanoramic ? ( ) : activeFrame ? ( {activeCam?.label ) : (
📷
Waiting for {activeCam?.label ?? '—'}…
{activeCam?.topic}
)} {/* Detection overlay canvas */} {overlayMode !== 'off' && !activeCam?.isPanoramic && ( )} {/* PiP windows */} {pipList.map((id, idx) => { const cam = CAMERA_BY_ID[id]; if (!cam) return null; return ( togglePip(id)} /> ); })}
{/* ── Recording controls ── */}
CAPTURE
Recording saves as MP4/WebM to your Downloads. Snapshot includes detection overlay + timestamp.
{/* ── Detection status ── */}
FACES
0 ? 'text-cyan-400' : 'text-gray-700'}`}> {faces.length > 0 ? `${faces.length} detected` : 'none'}
{faces.slice(0, 2).map((f, i) => (
{f.person_name && f.person_name !== 'unknown' ? `↳ ${f.person_name}` : `↳ unknown #${f.face_id}`}
))}
GESTURES
0 ? 'text-amber-400' : 'text-gray-700'}`}> {gestures.length > 0 ? `${gestures.length} active` : 'none'}
{gestures.slice(0, 2).map((g, i) => { const icon = GESTURE_ICONS[g.gesture_type] ?? '?'; return (
{icon} {g.gesture_type} cam{g.camera_id}
); })}
OBJECTS
0 ? 'text-green-400' : 'text-gray-700'}`}> {sceneObjects.length > 0 ? `${sceneObjects.length} objects` : 'none'}
{sceneObjects .filter(o => o.hazard_type > 0) .slice(0, 2) .map((o, i) => (
⚠ {o.class_name}
)) } {sceneObjects.filter(o => o.hazard_type === 0).slice(0, 2).map((o, i) => (
{o.class_name} {o.distance_m > 0 ? `${o.distance_m.toFixed(1)}m` : ''}
))}
{/* ── Legend ── */}
Known face
Unknown face
👆 Gesture
Object
Hazard
⊕ pin = PiP · overlay: {overlayMode}
); }