sl-webui 347449ed95 feat(webui): hand pose tracking and gesture visualization (Issue #344)
Features:
- Subscribes to /saltybot/hands (21 landmarks per hand - MediaPipe format)
- Subscribes to /saltybot/hand_gesture (String gesture label)
- Canvas-based hand skeleton rendering with bone connections
- Support for dual hand tracking (left and right)
- Handedness indicators with color coding
  * Left hand: green
  * Right hand: yellow
- Real-time gesture display with confidence indicator
- Per-landmark confidence visualization
- Bone connections between all 21 joints

Hand Skeleton Features:
- 21 MediaPipe landmarks per hand
  * Wrist (1)
  * Thumb (4)
  * Index finger (4)
  * Middle finger (4)
  * Ring finger (4)
  * Pinky finger (4)
- 20 bone connections between joints
- Confidence-based rendering (only show high-confidence points)
- Scaling and normalization for viewport
- Joint type indicators (tips with ring outline)
- Glow effects around landmarks

Gesture Recognition:
- Real-time gesture label display
- Confidence percentage (0-100%)
- Color-coded confidence:
  * Green: >80% (high confidence)
  * Yellow: 50-80% (medium confidence)
  * Blue: <50% (detecting)

Hand Status Display:
- Live detection status for both hands
- Visual indicators (✓ detected / ◯ not detected)
- Dual-hand canvas rendering
- Gesture info panel with confidence bar

Integration:
- Added to SOCIAL tab group as "Hands" tab
- Positioned after "Faces" tab
- Uses subscribe hook for real-time updates
- Dark theme with color-coded hands
- Canvas-based rendering for smooth visualization

Build: 125 modules, no errors
Main bundle: 270.08 KB

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
2026-03-03 12:43:19 -05:00

324 lines
12 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* App.jsx — Saltybot Social + Telemetry + Fleet + Mission Dashboard root component.
*
* Social tabs (issue #107):
* Status | Faces | Conversation | Personality | Navigation
*
* Telemetry tabs (issue #126):
* IMU | Battery | Motors | Map | Control | Health | Cameras
*
* Fleet tabs (issue #139):
* Fleet (self-contained via useFleet)
*
* Mission tabs (issue #145):
* Missions (waypoint editor, route builder, geofence, schedule, execute)
*
* Camera viewer (issue #177):
* CSI × 4 + D435i RGB/depth + panoramic, detection overlays, recording
*/
import { useState, useCallback } from 'react';
import { useRosbridge } from './hooks/useRosbridge.js';
// Social panels
import { StatusPanel } from './components/StatusPanel.jsx';
import { StatusHeader } from './components/StatusHeader.jsx';
import { FaceGallery } from './components/FaceGallery.jsx';
import { ConversationLog } from './components/ConversationLog.jsx';
import { ConversationHistory } from './components/ConversationHistory.jsx';
import { PersonalityTuner } from './components/PersonalityTuner.jsx';
import { NavModeSelector } from './components/NavModeSelector.jsx';
import { AudioMeter } from './components/AudioMeter.jsx';
// Telemetry panels
import PoseViewer from './components/PoseViewer.jsx';
import { BatteryPanel } from './components/BatteryPanel.jsx';
import { BatteryChart } from './components/BatteryChart.jsx';
import { MotorPanel } from './components/MotorPanel.jsx';
import { MotorCurrentGraph } from './components/MotorCurrentGraph.jsx';
import { MapViewer } from './components/MapViewer.jsx';
import { ControlMode } from './components/ControlMode.jsx';
import { SystemHealth } from './components/SystemHealth.jsx';
// Fleet panel (issue #139)
import { FleetPanel } from './components/FleetPanel.jsx';
// Mission planner (issue #145)
import { MissionPlanner } from './components/MissionPlanner.jsx';
// Settings panel (issue #160)
import { SettingsPanel } from './components/SettingsPanel.jsx';
// Camera viewer (issue #177)
import { CameraViewer } from './components/CameraViewer.jsx';
// Event log (issue #192)
import { EventLog } from './components/EventLog.jsx';
// Log viewer (issue #275)
import { LogViewer } from './components/LogViewer.jsx';
// Joystick teleop (issue #212)
import JoystickTeleop from './components/JoystickTeleop.jsx';
// Network diagnostics (issue #222)
import { NetworkPanel } from './components/NetworkPanel.jsx';
// Waypoint editor (issue #261)
import { WaypointEditor } from './components/WaypointEditor.jsx';
// Bandwidth monitor (issue #287)
import { BandwidthMonitor } from './components/BandwidthMonitor.jsx';
// Temperature gauge (issue #308)
import { TempGauge } from './components/TempGauge.jsx';
// Node list viewer
import { NodeList } from './components/NodeList.jsx';
// Gamepad teleoperation (issue #319)
import { Teleop } from './components/Teleop.jsx';
// System diagnostics (issue #340)
import { Diagnostics } from './components/Diagnostics.jsx';
// Hand tracking visualization (issue #344)
import { HandTracker } from './components/HandTracker.jsx';
const TAB_GROUPS = [
{
label: 'SOCIAL',
color: 'text-cyan-600',
tabs: [
{ id: 'status', label: 'Status', },
{ id: 'faces', label: 'Faces', },
{ id: 'hands', label: 'Hands', },
{ id: 'conversation', label: 'Convo', },
{ id: 'history', label: 'History', },
{ id: 'personality', label: 'Personality', },
{ id: 'navigation', label: 'Nav Mode', },
{ id: 'audio', label: 'Audio', },
],
},
{
label: 'TELEMETRY',
color: 'text-amber-600',
tabs: [
{ id: 'imu', label: 'IMU', },
{ id: 'battery', label: 'Battery', },
{ id: 'battery-chart', label: 'Battery History', },
{ id: 'motors', label: 'Motors', },
{ id: 'thermal', label: 'Thermal', },
{ id: 'map', label: 'Map', },
{ id: 'control', label: 'Control', },
{ id: 'health', label: 'Health', },
{ id: 'cameras', label: 'Cameras', },
],
},
{
label: 'NAVIGATION',
color: 'text-teal-600',
tabs: [
{ id: 'waypoints', label: 'Waypoints' },
],
},
{
label: 'FLEET',
color: 'text-green-600',
tabs: [
{ id: 'fleet', label: 'Fleet' },
{ id: 'missions', label: 'Missions' },
],
},
{
label: 'MONITORING',
color: 'text-yellow-600',
tabs: [
{ id: 'diagnostics', label: 'Diagnostics' },
{ id: 'eventlog', label: 'Events' },
{ id: 'bandwidth', label: 'Bandwidth' },
{ id: 'nodes', label: 'Nodes' },
],
},
{
label: 'CONFIG',
color: 'text-purple-600',
tabs: [
{ id: 'network', label: 'Network' },
{ id: 'settings', label: 'Settings' },
],
},
];
const FLEET_TABS = new Set(['fleet', 'missions']);
const DEFAULT_WS_URL = 'ws://localhost:9090';
function ConnectionBar({ url, setUrl, connected, error }) {
const [editing, setEditing] = useState(false);
const [draft, setDraft] = useState(url);
const handleApply = () => { setUrl(draft); setEditing(false); };
return (
<div className="flex items-center gap-2 text-xs">
<div className={`w-2 h-2 rounded-full shrink-0 ${
connected ? 'bg-green-400' : error ? 'bg-red-500' : 'bg-gray-600'
}`} />
{editing ? (
<div className="flex items-center gap-1">
<input
value={draft}
onChange={(e) => setDraft(e.target.value)}
onKeyDown={(e) => {
if (e.key === 'Enter') handleApply();
if (e.key === 'Escape') setEditing(false);
}}
autoFocus
className="bg-gray-900 border border-cyan-800 rounded px-2 py-0.5 text-cyan-300 w-52 focus:outline-none"
/>
<button
onClick={handleApply}
className="px-2 py-0.5 rounded bg-cyan-950 border border-cyan-700 text-cyan-400 hover:bg-cyan-900 text-xs"
>
Connect
</button>
<button onClick={() => setEditing(false)} className="text-gray-600 hover:text-gray-400 px-1">
</button>
</div>
) : (
<button
onClick={() => { setDraft(url); setEditing(true); }}
className="text-gray-500 hover:text-cyan-400 transition-colors truncate max-w-52"
title={url}
>
{connected ? (
<span className="text-green-400">{url}</span>
) : error ? (
<span className="text-red-400"> {url}</span>
) : (
<span className="text-gray-500">{url}</span>
)}
</button>
)}
</div>
);
}
export default function App() {
const [wsUrl, setWsUrl] = useState(DEFAULT_WS_URL);
const [activeTab, setActiveTab] = useState('status');
const { connected, error, subscribe, publish, callService, setParam } = useRosbridge(wsUrl);
const publishFn = useCallback((name, type, data) => publish(name, type, data), [publish]);
return (
<div className="min-h-screen flex flex-col bg-[#050510] text-gray-300 font-mono">
{/* ── Top Bar ── */}
<header className="flex items-center justify-between px-4 py-2 bg-[#070712] border-b border-cyan-950 shrink-0 gap-2 flex-wrap">
<div className="flex items-center gap-3">
<span className="text-orange-500 font-bold tracking-widest text-sm"> SALTYBOT</span>
<span className="text-cyan-800 text-xs hidden sm:inline">DASHBOARD</span>
</div>
{!FLEET_TABS.has(activeTab) && (
<ConnectionBar url={wsUrl} setUrl={setWsUrl} connected={connected} error={error} />
)}
</header>
{/* ── Status Header ── */}
<StatusHeader subscribe={subscribe} />
{/* ── Tab Navigation ── */}
<nav className="bg-[#070712] border-b border-cyan-950 shrink-0 overflow-x-auto">
<div className="flex min-w-max">
{TAB_GROUPS.map((group, gi) => (
<div key={group.label} className="flex items-stretch">
{gi > 0 && (
<div className="flex items-center px-1">
<div className="w-px h-4 bg-cyan-950" />
</div>
)}
<div className="flex items-center px-1.5 shrink-0">
<span className={`text-xs font-bold tracking-widest ${group.color} opacity-60`}>
{group.label}
</span>
</div>
{group.tabs.map((tab) => (
<button
key={tab.id}
onClick={() => setActiveTab(tab.id)}
className={`px-3 py-2.5 text-xs font-bold tracking-wider whitespace-nowrap border-b-2 transition-colors ${
activeTab === tab.id
? 'border-cyan-500 text-cyan-300 bg-cyan-950 bg-opacity-30'
: 'border-transparent text-gray-600 hover:text-gray-300 hover:border-gray-700'
}`}
>
{tab.label.toUpperCase()}
</button>
))}
</div>
))}
</div>
</nav>
{/* ── Content ── */}
<main className={`flex-1 ${['eventlog', 'control', 'imu'].includes(activeTab) ? 'flex flex-col' : 'overflow-y-auto'} p-4`}>
{activeTab === 'status' && <StatusPanel subscribe={subscribe} />}
{activeTab === 'faces' && <FaceGallery subscribe={subscribe} callService={callService} />}
{activeTab === 'hands' && <HandTracker subscribe={subscribe} />}
{activeTab === 'conversation' && <ConversationLog subscribe={subscribe} />}
{activeTab === 'history' && <ConversationHistory subscribe={subscribe} />}
{activeTab === 'personality' && <PersonalityTuner subscribe={subscribe} setParam={setParam} />}
{activeTab === 'navigation' && <NavModeSelector subscribe={subscribe} publish={publishFn} />}
{activeTab === 'audio' && <AudioMeter subscribe={subscribe} />}
{activeTab === 'imu' && <PoseViewer subscribe={subscribe} />}
{activeTab === 'battery' && <BatteryPanel subscribe={subscribe} />}
{activeTab === 'battery-chart' && <BatteryChart subscribe={subscribe} />}
{activeTab === 'motors' && <MotorPanel subscribe={subscribe} />}
{activeTab === 'motor-current-graph' && <MotorCurrentGraph subscribe={subscribe} />}
{activeTab === 'thermal' && <TempGauge subscribe={subscribe} />}
{activeTab === 'map' && <MapViewer subscribe={subscribe} />}
{activeTab === 'control' && <Teleop publish={publishFn} />}
{activeTab === 'health' && <SystemHealth subscribe={subscribe} />}
{activeTab === 'cameras' && <CameraViewer subscribe={subscribe} />}
{activeTab === 'waypoints' && <WaypointEditor subscribe={subscribe} publish={publishFn} callService={callService} />}
{activeTab === 'fleet' && <FleetPanel />}
{activeTab === 'missions' && <MissionPlanner />}
{activeTab === 'diagnostics' && <Diagnostics subscribe={subscribe} />}
{activeTab === 'eventlog' && <EventLog subscribe={subscribe} />}
{activeTab === 'bandwidth' && <BandwidthMonitor />}
{activeTab === 'nodes' && <NodeList subscribe={subscribe} publish={publishFn} callService={callService} />}
{activeTab === 'logs' && <LogViewer subscribe={subscribe} />}
{activeTab === 'network' && <NetworkPanel subscribe={subscribe} connected={connected} wsUrl={wsUrl} />}
{activeTab === 'settings' && <SettingsPanel subscribe={subscribe} callService={callService} connected={connected} wsUrl={wsUrl} />}
</main>
{/* ── Footer ── */}
<footer className="bg-[#070712] border-t border-cyan-950 px-4 py-1.5 flex items-center justify-between text-xs text-gray-700 shrink-0">
{!FLEET_TABS.has(activeTab) ? (
<>
<span>rosbridge: <code className="text-gray-600">{wsUrl}</code></span>
<span className={connected ? 'text-green-700' : 'text-red-900'}>
{connected ? 'CONNECTED' : 'DISCONNECTED'}
</span>
</>
) : (
<span className="text-green-800">FLEET MODE multi-robot</span>
)}
</footer>
</div>
);
}