/** * HandTracker.jsx — Hand pose and gesture visualization * * Features: * - Subscribes to /saltybot/hands (21 landmarks per hand) * - Subscribes to /saltybot/hand_gesture (String gesture label) * - Canvas-based hand skeleton rendering * - Bone connections between landmarks * - Support for dual hands (left and right) * - Handedness indicator * - Real-time gesture display * - Confidence-based landmark rendering */ import { useEffect, useRef, useState } from 'react'; // MediaPipe hand landmark connections (bones) const HAND_CONNECTIONS = [ // Thumb [0, 1], [1, 2], [2, 3], [3, 4], // Index finger [0, 5], [5, 6], [6, 7], [7, 8], // Middle finger [0, 9], [9, 10], [10, 11], [11, 12], // Ring finger [0, 13], [13, 14], [14, 15], [15, 16], // Pinky finger [0, 17], [17, 18], [18, 19], [19, 20], // Palm connections [5, 9], [9, 13], [13, 17], ]; const LANDMARK_NAMES = [ 'Wrist', 'Thumb CMC', 'Thumb MCP', 'Thumb IP', 'Thumb Tip', 'Index MCP', 'Index PIP', 'Index DIP', 'Index Tip', 'Middle MCP', 'Middle PIP', 'Middle DIP', 'Middle Tip', 'Ring MCP', 'Ring PIP', 'Ring DIP', 'Ring Tip', 'Pinky MCP', 'Pinky PIP', 'Pinky DIP', 'Pinky Tip', ]; function HandCanvas({ hand, color, label }) { const canvasRef = useRef(null); const [flipped, setFlipped] = useState(false); useEffect(() => { const canvas = canvasRef.current; if (!canvas || !hand || !hand.landmarks || hand.landmarks.length === 0) return; const ctx = canvas.getContext('2d'); const width = canvas.width; const height = canvas.height; // Clear canvas ctx.fillStyle = '#1f2937'; ctx.fillRect(0, 0, width, height); // Find min/max coordinates for scaling let minX = Infinity, maxX = -Infinity; let minY = Infinity, maxY = -Infinity; hand.landmarks.forEach((lm) => { minX = Math.min(minX, lm.x); maxX = Math.max(maxX, lm.x); minY = Math.min(minY, lm.y); maxY = Math.max(maxY, lm.y); }); const padding = 20; const rangeX = maxX - minX || 1; const rangeY = maxY - minY || 1; const scaleX = (width - padding * 2) / rangeX; const scaleY = (height - padding * 2) / rangeY; const scale = Math.min(scaleX, scaleY); // Convert landmark coordinates to canvas positions const getCanvasPos = (lm) => { const x = padding + (lm.x - minX) * scale; const y = padding + (lm.y - minY) * scale; return { x, y }; }; // Draw bones ctx.strokeStyle = color; ctx.lineWidth = 2; ctx.lineCap = 'round'; ctx.lineJoin = 'round'; HAND_CONNECTIONS.forEach(([start, end]) => { if (start < hand.landmarks.length && end < hand.landmarks.length) { const startLm = hand.landmarks[start]; const endLm = hand.landmarks[end]; if (startLm.confidence > 0.1 && endLm.confidence > 0.1) { const startPos = getCanvasPos(startLm); const endPos = getCanvasPos(endLm); ctx.globalAlpha = Math.min(startLm.confidence, endLm.confidence); ctx.beginPath(); ctx.moveTo(startPos.x, startPos.y); ctx.lineTo(endPos.x, endPos.y); ctx.stroke(); ctx.globalAlpha = 1.0; } } }); // Draw landmarks hand.landmarks.forEach((lm, i) => { if (lm.confidence > 0.1) { const pos = getCanvasPos(lm); const radius = 4 + lm.confidence * 2; // Landmark glow ctx.fillStyle = color; ctx.globalAlpha = lm.confidence * 0.3; ctx.beginPath(); ctx.arc(pos.x, pos.y, radius * 2, 0, Math.PI * 2); ctx.fill(); // Landmark point ctx.fillStyle = color; ctx.globalAlpha = lm.confidence; ctx.beginPath(); ctx.arc(pos.x, pos.y, radius, 0, Math.PI * 2); ctx.fill(); // Joint type marker const isJoint = i > 0 && i % 4 === 0; // Tip joints if (isJoint) { ctx.strokeStyle = '#fff'; ctx.lineWidth = 1; ctx.globalAlpha = lm.confidence; ctx.beginPath(); ctx.arc(pos.x, pos.y, radius + 2, 0, Math.PI * 2); ctx.stroke(); } } }); ctx.globalAlpha = 1.0; // Draw handedness label on canvas ctx.fillStyle = color; ctx.font = 'bold 12px monospace'; ctx.textAlign = 'left'; ctx.fillText(label, 10, height - 10); }, [hand, color, label]); return (
{label}
); } export function HandTracker({ subscribe }) { const [leftHand, setLeftHand] = useState(null); const [rightHand, setRightHand] = useState(null); const [gesture, setGesture] = useState(''); const [confidence, setConfidence] = useState(0); const handsRef = useRef({}); // Subscribe to hand poses useEffect(() => { const unsubscribe = subscribe( '/saltybot/hands', 'saltybot_msgs/HandPose', (msg) => { try { if (!msg) return; // Handle both single hand and multi-hand formats if (Array.isArray(msg)) { // Multi-hand format: array of hands const left = msg.find((h) => h.handedness === 'Left' || h.handedness === 0); const right = msg.find((h) => h.handedness === 'Right' || h.handedness === 1); if (left) setLeftHand(left); if (right) setRightHand(right); } else if (msg.handedness) { // Single hand format if (msg.handedness === 'Left' || msg.handedness === 0) { setLeftHand(msg); } else { setRightHand(msg); } } handsRef.current = { left: leftHand, right: rightHand }; } catch (e) { console.error('Error parsing hand pose:', e); } } ); return unsubscribe; }, [subscribe, leftHand, rightHand]); // Subscribe to gesture useEffect(() => { const unsubscribe = subscribe( '/saltybot/hand_gesture', 'std_msgs/String', (msg) => { try { if (msg.data) { // Parse gesture data (format: "gesture_name confidence") const parts = msg.data.split(' '); setGesture(parts[0] || ''); if (parts[1]) { setConfidence(parseFloat(parts[1]) || 0); } } } catch (e) { console.error('Error parsing gesture:', e); } } ); return unsubscribe; }, [subscribe]); const hasLeftHand = leftHand && leftHand.landmarks && leftHand.landmarks.length > 0; const hasRightHand = rightHand && rightHand.landmarks && rightHand.landmarks.length > 0; const gestureConfidentColor = confidence > 0.8 ? 'text-green-400' : confidence > 0.5 ? 'text-yellow-400' : 'text-gray-400'; return (
{/* Gesture Display */}
GESTURE
{gesture || 'Detecting...'}
{/* Confidence bar */} {gesture && (
Confidence
0.8 ? 'bg-green-500' : confidence > 0.5 ? 'bg-yellow-500' : 'bg-blue-500' }`} style={{ width: `${Math.round(confidence * 100)}%` }} />
{Math.round(confidence * 100)}%
)}
{/* Hand Renders */}
{hasLeftHand || hasRightHand ? (
{hasLeftHand && ( )} {hasRightHand && ( )}
) : (
Waiting for hand detection
Hands will appear when detected on /saltybot/hands
)}
{/* Hand Info */}
HAND STATUS
LEFT
{hasLeftHand ? '✓' : '◯'}
RIGHT
{hasRightHand ? '✓' : '◯'}
{/* Landmark Info */}
Hand Format: 21 landmarks per hand (MediaPipe)
Topics: /saltybot/hands, /saltybot/hand_gesture
Landmarks: Wrist + fingers (5×4 joints)
Color Code: 🟢 Left | 🟠 Right
); }