diff --git a/ui/social-bot/src/App.jsx b/ui/social-bot/src/App.jsx index cf763c6..f2c4b25 100644 --- a/ui/social-bot/src/App.jsx +++ b/ui/social-bot/src/App.jsx @@ -82,6 +82,9 @@ import { Teleop } from './components/Teleop.jsx'; // System diagnostics (issue #340) import { Diagnostics } from './components/Diagnostics.jsx'; +// Hand tracking visualization (issue #344) +import { HandTracker } from './components/HandTracker.jsx'; + const TAB_GROUPS = [ { label: 'SOCIAL', @@ -89,6 +92,7 @@ const TAB_GROUPS = [ tabs: [ { id: 'status', label: 'Status', }, { id: 'faces', label: 'Faces', }, + { id: 'hands', label: 'Hands', }, { id: 'conversation', label: 'Convo', }, { id: 'history', label: 'History', }, { id: 'personality', label: 'Personality', }, @@ -263,6 +267,7 @@ export default function App() {
{activeTab === 'status' && } {activeTab === 'faces' && } + {activeTab === 'hands' && } {activeTab === 'conversation' && } {activeTab === 'history' && } {activeTab === 'personality' && } diff --git a/ui/social-bot/src/components/HandTracker.jsx b/ui/social-bot/src/components/HandTracker.jsx new file mode 100644 index 0000000..df659ff --- /dev/null +++ b/ui/social-bot/src/components/HandTracker.jsx @@ -0,0 +1,331 @@ +/** + * HandTracker.jsx — Hand pose and gesture visualization + * + * Features: + * - Subscribes to /saltybot/hands (21 landmarks per hand) + * - Subscribes to /saltybot/hand_gesture (String gesture label) + * - Canvas-based hand skeleton rendering + * - Bone connections between landmarks + * - Support for dual hands (left and right) + * - Handedness indicator + * - Real-time gesture display + * - Confidence-based landmark rendering + */ + +import { useEffect, useRef, useState } from 'react'; + +// MediaPipe hand landmark connections (bones) +const HAND_CONNECTIONS = [ + // Thumb + [0, 1], [1, 2], [2, 3], [3, 4], + // Index finger + [0, 5], [5, 6], [6, 7], [7, 8], + // Middle finger + [0, 9], [9, 10], [10, 11], [11, 12], + // Ring finger + [0, 13], [13, 14], [14, 15], [15, 16], + // Pinky finger + [0, 17], [17, 18], [18, 19], [19, 20], + // Palm connections + [5, 9], [9, 13], [13, 17], +]; + +const LANDMARK_NAMES = [ + 'Wrist', + 'Thumb CMC', 'Thumb MCP', 'Thumb IP', 'Thumb Tip', + 'Index MCP', 'Index PIP', 'Index DIP', 'Index Tip', + 'Middle MCP', 'Middle PIP', 'Middle DIP', 'Middle Tip', + 'Ring MCP', 'Ring PIP', 'Ring DIP', 'Ring Tip', + 'Pinky MCP', 'Pinky PIP', 'Pinky DIP', 'Pinky Tip', +]; + +function HandCanvas({ hand, color, label }) { + const canvasRef = useRef(null); + const [flipped, setFlipped] = useState(false); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas || !hand || !hand.landmarks || hand.landmarks.length === 0) return; + + const ctx = canvas.getContext('2d'); + const width = canvas.width; + const height = canvas.height; + + // Clear canvas + ctx.fillStyle = '#1f2937'; + ctx.fillRect(0, 0, width, height); + + // Find min/max coordinates for scaling + let minX = Infinity, maxX = -Infinity; + let minY = Infinity, maxY = -Infinity; + + hand.landmarks.forEach((lm) => { + minX = Math.min(minX, lm.x); + maxX = Math.max(maxX, lm.x); + minY = Math.min(minY, lm.y); + maxY = Math.max(maxY, lm.y); + }); + + const padding = 20; + const rangeX = maxX - minX || 1; + const rangeY = maxY - minY || 1; + const scaleX = (width - padding * 2) / rangeX; + const scaleY = (height - padding * 2) / rangeY; + const scale = Math.min(scaleX, scaleY); + + // Convert landmark coordinates to canvas positions + const getCanvasPos = (lm) => { + const x = padding + (lm.x - minX) * scale; + const y = padding + (lm.y - minY) * scale; + return { x, y }; + }; + + // Draw bones + ctx.strokeStyle = color; + ctx.lineWidth = 2; + ctx.lineCap = 'round'; + ctx.lineJoin = 'round'; + + HAND_CONNECTIONS.forEach(([start, end]) => { + if (start < hand.landmarks.length && end < hand.landmarks.length) { + const startLm = hand.landmarks[start]; + const endLm = hand.landmarks[end]; + + if (startLm.confidence > 0.1 && endLm.confidence > 0.1) { + const startPos = getCanvasPos(startLm); + const endPos = getCanvasPos(endLm); + + ctx.globalAlpha = Math.min(startLm.confidence, endLm.confidence); + ctx.beginPath(); + ctx.moveTo(startPos.x, startPos.y); + ctx.lineTo(endPos.x, endPos.y); + ctx.stroke(); + ctx.globalAlpha = 1.0; + } + } + }); + + // Draw landmarks + hand.landmarks.forEach((lm, i) => { + if (lm.confidence > 0.1) { + const pos = getCanvasPos(lm); + const radius = 4 + lm.confidence * 2; + + // Landmark glow + ctx.fillStyle = color; + ctx.globalAlpha = lm.confidence * 0.3; + ctx.beginPath(); + ctx.arc(pos.x, pos.y, radius * 2, 0, Math.PI * 2); + ctx.fill(); + + // Landmark point + ctx.fillStyle = color; + ctx.globalAlpha = lm.confidence; + ctx.beginPath(); + ctx.arc(pos.x, pos.y, radius, 0, Math.PI * 2); + ctx.fill(); + + // Joint type marker + const isJoint = i > 0 && i % 4 === 0; // Tip joints + if (isJoint) { + ctx.strokeStyle = '#fff'; + ctx.lineWidth = 1; + ctx.globalAlpha = lm.confidence; + ctx.beginPath(); + ctx.arc(pos.x, pos.y, radius + 2, 0, Math.PI * 2); + ctx.stroke(); + } + } + }); + + ctx.globalAlpha = 1.0; + + // Draw handedness label on canvas + ctx.fillStyle = color; + ctx.font = 'bold 12px monospace'; + ctx.textAlign = 'left'; + ctx.fillText(label, 10, height - 10); + }, [hand, color, label]); + + return ( +
+
{label}
+ +
+ ); +} + +export function HandTracker({ subscribe }) { + const [leftHand, setLeftHand] = useState(null); + const [rightHand, setRightHand] = useState(null); + const [gesture, setGesture] = useState(''); + const [confidence, setConfidence] = useState(0); + const handsRef = useRef({}); + + // Subscribe to hand poses + useEffect(() => { + const unsubscribe = subscribe( + '/saltybot/hands', + 'saltybot_msgs/HandPose', + (msg) => { + try { + if (!msg) return; + + // Handle both single hand and multi-hand formats + if (Array.isArray(msg)) { + // Multi-hand format: array of hands + const left = msg.find((h) => h.handedness === 'Left' || h.handedness === 0); + const right = msg.find((h) => h.handedness === 'Right' || h.handedness === 1); + if (left) setLeftHand(left); + if (right) setRightHand(right); + } else if (msg.handedness) { + // Single hand format + if (msg.handedness === 'Left' || msg.handedness === 0) { + setLeftHand(msg); + } else { + setRightHand(msg); + } + } + + handsRef.current = { left: leftHand, right: rightHand }; + } catch (e) { + console.error('Error parsing hand pose:', e); + } + } + ); + + return unsubscribe; + }, [subscribe, leftHand, rightHand]); + + // Subscribe to gesture + useEffect(() => { + const unsubscribe = subscribe( + '/saltybot/hand_gesture', + 'std_msgs/String', + (msg) => { + try { + if (msg.data) { + // Parse gesture data (format: "gesture_name confidence") + const parts = msg.data.split(' '); + setGesture(parts[0] || ''); + if (parts[1]) { + setConfidence(parseFloat(parts[1]) || 0); + } + } + } catch (e) { + console.error('Error parsing gesture:', e); + } + } + ); + + return unsubscribe; + }, [subscribe]); + + const hasLeftHand = leftHand && leftHand.landmarks && leftHand.landmarks.length > 0; + const hasRightHand = rightHand && rightHand.landmarks && rightHand.landmarks.length > 0; + const gestureConfidentColor = + confidence > 0.8 ? 'text-green-400' : confidence > 0.5 ? 'text-yellow-400' : 'text-gray-400'; + + return ( +
+ {/* Gesture Display */} +
+
+
GESTURE
+
+ {gesture || 'Detecting...'} +
+
+ + {/* Confidence bar */} + {gesture && ( +
+
Confidence
+
+
0.8 + ? 'bg-green-500' + : confidence > 0.5 + ? 'bg-yellow-500' + : 'bg-blue-500' + }`} + style={{ width: `${Math.round(confidence * 100)}%` }} + /> +
+
+ {Math.round(confidence * 100)}% +
+
+ )} +
+ + {/* Hand Renders */} +
+ {hasLeftHand || hasRightHand ? ( +
+ {hasLeftHand && ( + + )} + {hasRightHand && ( + + )} +
+ ) : ( +
+
+
Waiting for hand detection
+
+ Hands will appear when detected on /saltybot/hands +
+
+
+ )} +
+ + {/* Hand Info */} +
+
HAND STATUS
+
+
+
LEFT
+
+ {hasLeftHand ? '✓' : '◯'} +
+
+
+
RIGHT
+
+ {hasRightHand ? '✓' : '◯'} +
+
+
+
+ + {/* Landmark Info */} +
+
+ Hand Format: + 21 landmarks per hand (MediaPipe) +
+
+ Topics: + /saltybot/hands, /saltybot/hand_gesture +
+
+ Landmarks: + Wrist + fingers (5×4 joints) +
+
+ Color Code: + 🟢 Left | 🟠 Right +
+
+
+ ); +}