/** * HandTracker.jsx — Hand pose and gesture visualization * * Features: * - Subscribes to /saltybot/hands (21 landmarks per hand) * - Subscribes to /saltybot/hand_gesture (String gesture label) * - Canvas-based hand skeleton rendering * - Bone connections between landmarks * - Support for dual hands (left and right) * - Handedness indicator * - Real-time gesture display * - Confidence-based landmark rendering */ import { useEffect, useRef, useState } from 'react'; // MediaPipe hand landmark connections (bones) const HAND_CONNECTIONS = [ // Thumb [0, 1], [1, 2], [2, 3], [3, 4], // Index finger [0, 5], [5, 6], [6, 7], [7, 8], // Middle finger [0, 9], [9, 10], [10, 11], [11, 12], // Ring finger [0, 13], [13, 14], [14, 15], [15, 16], // Pinky finger [0, 17], [17, 18], [18, 19], [19, 20], // Palm connections [5, 9], [9, 13], [13, 17], ]; const LANDMARK_NAMES = [ 'Wrist', 'Thumb CMC', 'Thumb MCP', 'Thumb IP', 'Thumb Tip', 'Index MCP', 'Index PIP', 'Index DIP', 'Index Tip', 'Middle MCP', 'Middle PIP', 'Middle DIP', 'Middle Tip', 'Ring MCP', 'Ring PIP', 'Ring DIP', 'Ring Tip', 'Pinky MCP', 'Pinky PIP', 'Pinky DIP', 'Pinky Tip', ]; function HandCanvas({ hand, color, label }) { const canvasRef = useRef(null); const [flipped, setFlipped] = useState(false); useEffect(() => { const canvas = canvasRef.current; if (!canvas || !hand || !hand.landmarks || hand.landmarks.length === 0) return; const ctx = canvas.getContext('2d'); const width = canvas.width; const height = canvas.height; // Clear canvas ctx.fillStyle = '#1f2937'; ctx.fillRect(0, 0, width, height); // Find min/max coordinates for scaling let minX = Infinity, maxX = -Infinity; let minY = Infinity, maxY = -Infinity; hand.landmarks.forEach((lm) => { minX = Math.min(minX, lm.x); maxX = Math.max(maxX, lm.x); minY = Math.min(minY, lm.y); maxY = Math.max(maxY, lm.y); }); const padding = 20; const rangeX = maxX - minX || 1; const rangeY = maxY - minY || 1; const scaleX = (width - padding * 2) / rangeX; const scaleY = (height - padding * 2) / rangeY; const scale = Math.min(scaleX, scaleY); // Convert landmark coordinates to canvas positions const getCanvasPos = (lm) => { const x = padding + (lm.x - minX) * scale; const y = padding + (lm.y - minY) * scale; return { x, y }; }; // Draw bones ctx.strokeStyle = color; ctx.lineWidth = 2; ctx.lineCap = 'round'; ctx.lineJoin = 'round'; HAND_CONNECTIONS.forEach(([start, end]) => { if (start < hand.landmarks.length && end < hand.landmarks.length) { const startLm = hand.landmarks[start]; const endLm = hand.landmarks[end]; if (startLm.confidence > 0.1 && endLm.confidence > 0.1) { const startPos = getCanvasPos(startLm); const endPos = getCanvasPos(endLm); ctx.globalAlpha = Math.min(startLm.confidence, endLm.confidence); ctx.beginPath(); ctx.moveTo(startPos.x, startPos.y); ctx.lineTo(endPos.x, endPos.y); ctx.stroke(); ctx.globalAlpha = 1.0; } } }); // Draw landmarks hand.landmarks.forEach((lm, i) => { if (lm.confidence > 0.1) { const pos = getCanvasPos(lm); const radius = 4 + lm.confidence * 2; // Landmark glow ctx.fillStyle = color; ctx.globalAlpha = lm.confidence * 0.3; ctx.beginPath(); ctx.arc(pos.x, pos.y, radius * 2, 0, Math.PI * 2); ctx.fill(); // Landmark point ctx.fillStyle = color; ctx.globalAlpha = lm.confidence; ctx.beginPath(); ctx.arc(pos.x, pos.y, radius, 0, Math.PI * 2); ctx.fill(); // Joint type marker const isJoint = i > 0 && i % 4 === 0; // Tip joints if (isJoint) { ctx.strokeStyle = '#fff'; ctx.lineWidth = 1; ctx.globalAlpha = lm.confidence; ctx.beginPath(); ctx.arc(pos.x, pos.y, radius + 2, 0, Math.PI * 2); ctx.stroke(); } } }); ctx.globalAlpha = 1.0; // Draw handedness label on canvas ctx.fillStyle = color; ctx.font = 'bold 12px monospace'; ctx.textAlign = 'left'; ctx.fillText(label, 10, height - 10); }, [hand, color, label]); return (