From 916ad36ad56b524cad95ec79cc2a00628a8328a3 Mon Sep 17 00:00:00 2001 From: sl-webui Date: Sat, 7 Mar 2026 09:51:14 -0500 Subject: [PATCH] feat(webui): teleop web interface with live camera stream (Issue #534) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds TeleopWebUI component — a dedicated browser-based remote control panel combining live video and joystick teleoperation in one view: - Live camera stream (front/rear/left/right) via rosbridge CompressedImage - Virtual joystick (canvas-based, touch + mouse, 10% deadzone) - WASD / arrow-key keyboard fallback, Space for quick stop - Speed presets: SLOW (20%), NORMAL (50%), FAST (100%) - Latching E-stop button with pulsing visual indicator - Real-time linear/angular velocity display - Mobile-responsive: stacks vertically on small screens, side-by-side on lg+ - Added TELEOP tab group → Drive tab in App.jsx Topics: /camera//image_raw/compressed (subscribe) /cmd_vel geometry_msgs/Twist (publish) Co-Authored-By: Claude Sonnet 4.6 --- ui/social-bot/src/App.jsx | 14 +- ui/social-bot/src/components/TeleopWebUI.jsx | 480 +++++++++++++++++++ 2 files changed, 493 insertions(+), 1 deletion(-) create mode 100644 ui/social-bot/src/components/TeleopWebUI.jsx diff --git a/ui/social-bot/src/App.jsx b/ui/social-bot/src/App.jsx index be58963..e3eea94 100644 --- a/ui/social-bot/src/App.jsx +++ b/ui/social-bot/src/App.jsx @@ -91,7 +91,17 @@ import { SaltyFace } from './components/SaltyFace.jsx'; // Parameter server (issue #471) import { ParameterServer } from './components/ParameterServer.jsx'; +// Teleop web interface (issue #534) +import { TeleopWebUI } from './components/TeleopWebUI.jsx'; + const TAB_GROUPS = [ + { + label: 'TELEOP', + color: 'text-orange-600', + tabs: [ + { id: 'teleop-webui', label: 'Drive' }, + ], + }, { label: 'DISPLAY', color: 'text-rose-600', @@ -286,8 +296,10 @@ export default function App() { {/* ── Content ── */}
+ {activeTab === 'teleop-webui' && } + {activeTab === 'salty-face' && } {activeTab === 'status' && } diff --git a/ui/social-bot/src/components/TeleopWebUI.jsx b/ui/social-bot/src/components/TeleopWebUI.jsx new file mode 100644 index 0000000..f929bf9 --- /dev/null +++ b/ui/social-bot/src/components/TeleopWebUI.jsx @@ -0,0 +1,480 @@ +/** + * TeleopWebUI.jsx — Browser-based remote control with live video (Issue #534). + * + * Features: + * - Live camera stream via rosbridge (sensor_msgs/CompressedImage) + * - Camera selector: front / rear / left / right + * - Virtual joystick (touch + mouse) for mobile-friendly control + * - WASD / arrow-key keyboard fallback + * - Speed presets: SLOW (20%), NORMAL (50%), FAST (100%) + * - E-stop button (latching, prominent) + * - Real-time linear/angular velocity display + * - Mobile-responsive split layout + * + * Topics: + * /camera//image_raw/compressed sensor_msgs/CompressedImage (subscribe) + * /cmd_vel geometry_msgs/Twist (publish) + */ + +import { useEffect, useRef, useState, useCallback } from 'react'; + +// ── Constants ───────────────────────────────────────────────────────────────── + +const MAX_LINEAR = 0.5; // m/s +const MAX_ANGULAR = 1.0; // rad/s +const DEADZONE = 0.10; // 10 % +const CMD_HZ = 20; // publish rate (ms) + +const SPEED_PRESETS = [ + { label: 'SLOW', value: 0.20, color: 'text-green-400', border: 'border-green-800', bg: 'bg-green-950' }, + { label: 'NORMAL', value: 0.50, color: 'text-amber-400', border: 'border-amber-800', bg: 'bg-amber-950' }, + { label: 'FAST', value: 1.00, color: 'text-red-400', border: 'border-red-800', bg: 'bg-red-950' }, +]; + +const CAMERAS = [ + { id: 'front', label: 'Front', topic: '/camera/front/image_raw/compressed' }, + { id: 'rear', label: 'Rear', topic: '/camera/rear/image_raw/compressed' }, + { id: 'left', label: 'Left', topic: '/camera/left/image_raw/compressed' }, + { id: 'right', label: 'Right', topic: '/camera/right/image_raw/compressed' }, +]; + +// ── Utilities ───────────────────────────────────────────────────────────────── + +function applyDeadzone(value) { + const abs = Math.abs(value); + if (abs < DEADZONE) return 0; + return Math.sign(value) * ((abs - DEADZONE) / (1 - DEADZONE)); +} + +function clamp(v, lo, hi) { + return Math.max(lo, Math.min(hi, v)); +} + +// ── Virtual joystick ────────────────────────────────────────────────────────── + +function VirtualJoystick({ onMove }) { + const canvasRef = useRef(null); + const activeRef = useRef(false); + const stickRef = useRef({ x: 0, y: 0 }); + + const draw = useCallback(() => { + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = canvas.getContext('2d'); + const W = canvas.width; + const H = canvas.height; + const cx = W / 2; + const cy = H / 2; + const R = Math.min(W, H) * 0.38; + const kr = Math.min(W, H) * 0.14; + + ctx.clearRect(0, 0, W, H); + + // Base plate + ctx.fillStyle = 'rgba(15,23,42,0.85)'; + ctx.beginPath(); + ctx.arc(cx, cy, R + kr, 0, Math.PI * 2); + ctx.fill(); + + // Base ring + ctx.strokeStyle = '#1e3a5f'; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.arc(cx, cy, R, 0, Math.PI * 2); + ctx.stroke(); + + // Deadzone ring + ctx.strokeStyle = '#374151'; + ctx.lineWidth = 1; + ctx.setLineDash([3, 3]); + ctx.beginPath(); + ctx.arc(cx, cy, R * DEADZONE, 0, Math.PI * 2); + ctx.stroke(); + ctx.setLineDash([]); + + // Crosshair + ctx.strokeStyle = '#1e3a5f'; + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.moveTo(cx - R * 0.3, cy); ctx.lineTo(cx + R * 0.3, cy); + ctx.moveTo(cx, cy - R * 0.3); ctx.lineTo(cx, cy + R * 0.3); + ctx.stroke(); + + // Knob + const kx = cx + stickRef.current.x * R; + const ky = cy - stickRef.current.y * R; + const active = activeRef.current; + + const grad = ctx.createRadialGradient(kx - kr * 0.3, ky - kr * 0.3, 0, kx, ky, kr); + grad.addColorStop(0, active ? '#38bdf8' : '#1d4ed8'); + grad.addColorStop(1, active ? '#0369a1' : '#1e3a8a'); + ctx.fillStyle = grad; + ctx.beginPath(); + ctx.arc(kx, ky, kr, 0, Math.PI * 2); + ctx.fill(); + + ctx.strokeStyle = active ? '#7dd3fc' : '#3b82f6'; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.arc(kx, ky, kr, 0, Math.PI * 2); + ctx.stroke(); + + // Vector line + const { x, y } = stickRef.current; + if (Math.abs(x) > 0.02 || Math.abs(y) > 0.02) { + ctx.strokeStyle = 'rgba(56,189,248,0.5)'; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.moveTo(cx, cy); + ctx.lineTo(kx, ky); + ctx.stroke(); + } + }, []); + + const getPos = (clientX, clientY) => { + const canvas = canvasRef.current; + const rect = canvas.getBoundingClientRect(); + const R = Math.min(rect.width, rect.height) * 0.38; + const cx = rect.left + rect.width / 2; + const cy = rect.top + rect.height / 2; + let x = (clientX - cx) / R; + let y = (clientY - cy) / R; + const mag = Math.sqrt(x * x + y * y); + if (mag > 1) { x /= mag; y /= mag; } + return { x: clamp(x, -1, 1), y: clamp(-y, -1, 1) }; + }; + + const onStart = useCallback((clientX, clientY) => { + activeRef.current = true; + const { x, y } = getPos(clientX, clientY); + stickRef.current = { x, y }; + onMove({ x: applyDeadzone(x), y: applyDeadzone(y) }); + draw(); + }, [draw, onMove]); + + const onContinue = useCallback((clientX, clientY) => { + if (!activeRef.current) return; + const { x, y } = getPos(clientX, clientY); + stickRef.current = { x, y }; + onMove({ x: applyDeadzone(x), y: applyDeadzone(y) }); + draw(); + }, [draw, onMove]); + + const onEnd = useCallback(() => { + activeRef.current = false; + stickRef.current = { x: 0, y: 0 }; + onMove({ x: 0, y: 0 }); + draw(); + }, [draw, onMove]); + + // Mount: draw once + attach pointer events + useEffect(() => { + draw(); + const canvas = canvasRef.current; + if (!canvas) return; + + const onPD = (e) => { e.preventDefault(); onStart(e.clientX, e.clientY); }; + const onPM = (e) => { e.preventDefault(); onContinue(e.clientX, e.clientY); }; + const onPU = (e) => { e.preventDefault(); onEnd(); }; + + const onTD = (e) => { e.preventDefault(); const t = e.touches[0]; onStart(t.clientX, t.clientY); }; + const onTM = (e) => { e.preventDefault(); const t = e.touches[0]; onContinue(t.clientX, t.clientY); }; + const onTE = (e) => { e.preventDefault(); onEnd(); }; + + canvas.addEventListener('pointerdown', onPD); + window.addEventListener('pointermove', onPM); + window.addEventListener('pointerup', onPU); + canvas.addEventListener('touchstart', onTD, { passive: false }); + canvas.addEventListener('touchmove', onTM, { passive: false }); + canvas.addEventListener('touchend', onTE, { passive: false }); + + return () => { + canvas.removeEventListener('pointerdown', onPD); + window.removeEventListener('pointermove', onPM); + window.removeEventListener('pointerup', onPU); + canvas.removeEventListener('touchstart', onTD); + canvas.removeEventListener('touchmove', onTM); + canvas.removeEventListener('touchend', onTE); + }; + }, [onStart, onContinue, onEnd, draw]); + + return ( + + ); +} + +// ── Camera feed ─────────────────────────────────────────────────────────────── + +function CameraFeed({ subscribe, topic }) { + const [src, setSrc] = useState(null); + const [fps, setFps] = useState(0); + const frameCountRef = useRef(0); + const lastFpsTimeRef = useRef(Date.now()); + + useEffect(() => { + setSrc(null); + frameCountRef.current = 0; + lastFpsTimeRef.current = Date.now(); + + const unsub = subscribe(topic, 'sensor_msgs/CompressedImage', (msg) => { + const fmt = msg.format?.includes('png') ? 'png' : 'jpeg'; + const dataUrl = `data:image/${fmt};base64,${msg.data}`; + setSrc(dataUrl); + + frameCountRef.current++; + const now = Date.now(); + const dt = now - lastFpsTimeRef.current; + if (dt >= 1000) { + setFps(Math.round((frameCountRef.current * 1000) / dt)); + frameCountRef.current = 0; + lastFpsTimeRef.current = now; + } + }); + + return unsub; + }, [subscribe, topic]); + + return ( +
+ {src ? ( + camera + ) : ( +
+
📷
+
NO SIGNAL
+
+ )} + {/* FPS badge */} + {src && ( +
+ {fps} fps +
+ )} +
+ ); +} + +// ── Main component ──────────────────────────────────────────────────────────── + +export function TeleopWebUI({ subscribe, publish }) { + const [selectedCam, setSelectedCam] = useState('front'); + const [speedPreset, setSpeedPreset] = useState(0.50); + const [isEstopped, setIsEstopped] = useState(false); + const [linearVel, setLinearVel] = useState(0); + const [angularVel, setAngularVel] = useState(0); + + // Joystick normalized input ref [-1,1] each axis + const joystickRef = useRef({ x: 0, y: 0 }); + // Keyboard pressed keys + const keysRef = useRef({}); + + const currentCam = CAMERAS.find(c => c.id === selectedCam); + + // ── Keyboard input ────────────────────────────────────────────────────────── + useEffect(() => { + const down = (e) => { + const k = e.key.toLowerCase(); + if (['w','a','s','d','arrowup','arrowdown','arrowleft','arrowright',' '].includes(k)) { + keysRef.current[k] = true; + e.preventDefault(); + } + }; + const up = (e) => { + const k = e.key.toLowerCase(); + keysRef.current[k] = false; + }; + window.addEventListener('keydown', down); + window.addEventListener('keyup', up); + return () => { + window.removeEventListener('keydown', down); + window.removeEventListener('keyup', up); + }; + }, []); + + // ── Publish loop ──────────────────────────────────────────────────────────── + useEffect(() => { + const timer = setInterval(() => { + const k = keysRef.current; + const j = joystickRef.current; + + // Keyboard overrides joystick if any key pressed + let lin = j.y; + let ang = -j.x; // right stick-x → negative angular (turn left on left) + + const anyKey = k['w'] || k['s'] || k['a'] || k['d'] || + k['arrowup'] || k['arrowdown'] || k['arrowleft'] || k['arrowright']; + + if (anyKey) { + lin = 0; + ang = 0; + if (k['w'] || k['arrowup']) lin = 1; + if (k['s'] || k['arrowdown']) lin = -1; + if (k['a'] || k['arrowleft']) ang = 1; + if (k['d'] || k['arrowright']) ang = -1; + } + + // Space = quick stop (doesn't latch e-stop) + if (k[' ']) { lin = 0; ang = 0; } + + const factor = isEstopped ? 0 : speedPreset; + const finalLin = clamp(lin * MAX_LINEAR * factor, -MAX_LINEAR, MAX_LINEAR); + const finalAng = clamp(ang * MAX_ANGULAR * factor, -MAX_ANGULAR, MAX_ANGULAR); + + setLinearVel(finalLin); + setAngularVel(finalAng); + + if (publish) { + publish('/cmd_vel', 'geometry_msgs/Twist', { + linear: { x: finalLin, y: 0, z: 0 }, + angular: { x: 0, y: 0, z: finalAng }, + }); + } + }, CMD_HZ); + + return () => clearInterval(timer); + }, [isEstopped, speedPreset, publish]); + + // ── E-stop ────────────────────────────────────────────────────────────────── + const handleEstop = () => { + const next = !isEstopped; + setIsEstopped(next); + if (publish) { + publish('/cmd_vel', 'geometry_msgs/Twist', { + linear: { x: 0, y: 0, z: 0 }, + angular: { x: 0, y: 0, z: 0 }, + }); + } + }; + + // Joystick callback + const handleJoystick = useCallback((pos) => { + joystickRef.current = pos; + }, []); + + // ── Render ────────────────────────────────────────────────────────────────── + return ( +
+ + {/* ── Left: Camera feed ── */} +
+ + {/* Camera selector */} +
+ {CAMERAS.map((cam) => ( + + ))} +
+
+ {currentCam?.topic} +
+
+ + {/* Video feed */} +
+ +
+
+ + {/* ── Right: Controls ── */} +
+ + {/* E-stop */} + + + {/* Speed presets */} +
+
SPEED PRESET
+
+ {SPEED_PRESETS.map((p) => ( + + ))} +
+
+ + {/* Velocity display */} +
+
+
LINEAR
+
+ {linearVel.toFixed(2)} +
+
m/s
+
+
+
ANGULAR
+
+ {angularVel.toFixed(2)} +
+
rad/s
+
+
+ + {/* Joystick */} +
+
JOYSTICK
+ +
+ drag or use W A S D / arrow keys +
+
+ + {/* Key reference */} +
+
KEYBOARD
+
+
W/↑ Forward
+
S/↓ Backward
+
A/← Turn Left
+
D/→ Turn Right
+
Space Quick stop
+
+
+
+
+ ); +}