sl-webui
347449ed95
Features: - Subscribes to /saltybot/hands (21 landmarks per hand - MediaPipe format) - Subscribes to /saltybot/hand_gesture (String gesture label) - Canvas-based hand skeleton rendering with bone connections - Support for dual hand tracking (left and right) - Handedness indicators with color coding * Left hand: green * Right hand: yellow - Real-time gesture display with confidence indicator - Per-landmark confidence visualization - Bone connections between all 21 joints Hand Skeleton Features: - 21 MediaPipe landmarks per hand * Wrist (1) * Thumb (4) * Index finger (4) * Middle finger (4) * Ring finger (4) * Pinky finger (4) - 20 bone connections between joints - Confidence-based rendering (only show high-confidence points) - Scaling and normalization for viewport - Joint type indicators (tips with ring outline) - Glow effects around landmarks Gesture Recognition: - Real-time gesture label display - Confidence percentage (0-100%) - Color-coded confidence: * Green: >80% (high confidence) * Yellow: 50-80% (medium confidence) * Blue: <50% (detecting) Hand Status Display: - Live detection status for both hands - Visual indicators (✓ detected / ◯ not detected) - Dual-hand canvas rendering - Gesture info panel with confidence bar Integration: - Added to SOCIAL tab group as "Hands" tab - Positioned after "Faces" tab - Uses subscribe hook for real-time updates - Dark theme with color-coded hands - Canvas-based rendering for smooth visualization Build: 125 modules, no errors Main bundle: 270.08 KB Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
332 lines
11 KiB
JavaScript
332 lines
11 KiB
JavaScript
/**
|
||
* HandTracker.jsx — Hand pose and gesture visualization
|
||
*
|
||
* Features:
|
||
* - Subscribes to /saltybot/hands (21 landmarks per hand)
|
||
* - Subscribes to /saltybot/hand_gesture (String gesture label)
|
||
* - Canvas-based hand skeleton rendering
|
||
* - Bone connections between landmarks
|
||
* - Support for dual hands (left and right)
|
||
* - Handedness indicator
|
||
* - Real-time gesture display
|
||
* - Confidence-based landmark rendering
|
||
*/
|
||
|
||
import { useEffect, useRef, useState } from 'react';
|
||
|
||
// MediaPipe hand landmark connections (bones)
|
||
const HAND_CONNECTIONS = [
|
||
// Thumb
|
||
[0, 1], [1, 2], [2, 3], [3, 4],
|
||
// Index finger
|
||
[0, 5], [5, 6], [6, 7], [7, 8],
|
||
// Middle finger
|
||
[0, 9], [9, 10], [10, 11], [11, 12],
|
||
// Ring finger
|
||
[0, 13], [13, 14], [14, 15], [15, 16],
|
||
// Pinky finger
|
||
[0, 17], [17, 18], [18, 19], [19, 20],
|
||
// Palm connections
|
||
[5, 9], [9, 13], [13, 17],
|
||
];
|
||
|
||
const LANDMARK_NAMES = [
|
||
'Wrist',
|
||
'Thumb CMC', 'Thumb MCP', 'Thumb IP', 'Thumb Tip',
|
||
'Index MCP', 'Index PIP', 'Index DIP', 'Index Tip',
|
||
'Middle MCP', 'Middle PIP', 'Middle DIP', 'Middle Tip',
|
||
'Ring MCP', 'Ring PIP', 'Ring DIP', 'Ring Tip',
|
||
'Pinky MCP', 'Pinky PIP', 'Pinky DIP', 'Pinky Tip',
|
||
];
|
||
|
||
function HandCanvas({ hand, color, label }) {
|
||
const canvasRef = useRef(null);
|
||
const [flipped, setFlipped] = useState(false);
|
||
|
||
useEffect(() => {
|
||
const canvas = canvasRef.current;
|
||
if (!canvas || !hand || !hand.landmarks || hand.landmarks.length === 0) return;
|
||
|
||
const ctx = canvas.getContext('2d');
|
||
const width = canvas.width;
|
||
const height = canvas.height;
|
||
|
||
// Clear canvas
|
||
ctx.fillStyle = '#1f2937';
|
||
ctx.fillRect(0, 0, width, height);
|
||
|
||
// Find min/max coordinates for scaling
|
||
let minX = Infinity, maxX = -Infinity;
|
||
let minY = Infinity, maxY = -Infinity;
|
||
|
||
hand.landmarks.forEach((lm) => {
|
||
minX = Math.min(minX, lm.x);
|
||
maxX = Math.max(maxX, lm.x);
|
||
minY = Math.min(minY, lm.y);
|
||
maxY = Math.max(maxY, lm.y);
|
||
});
|
||
|
||
const padding = 20;
|
||
const rangeX = maxX - minX || 1;
|
||
const rangeY = maxY - minY || 1;
|
||
const scaleX = (width - padding * 2) / rangeX;
|
||
const scaleY = (height - padding * 2) / rangeY;
|
||
const scale = Math.min(scaleX, scaleY);
|
||
|
||
// Convert landmark coordinates to canvas positions
|
||
const getCanvasPos = (lm) => {
|
||
const x = padding + (lm.x - minX) * scale;
|
||
const y = padding + (lm.y - minY) * scale;
|
||
return { x, y };
|
||
};
|
||
|
||
// Draw bones
|
||
ctx.strokeStyle = color;
|
||
ctx.lineWidth = 2;
|
||
ctx.lineCap = 'round';
|
||
ctx.lineJoin = 'round';
|
||
|
||
HAND_CONNECTIONS.forEach(([start, end]) => {
|
||
if (start < hand.landmarks.length && end < hand.landmarks.length) {
|
||
const startLm = hand.landmarks[start];
|
||
const endLm = hand.landmarks[end];
|
||
|
||
if (startLm.confidence > 0.1 && endLm.confidence > 0.1) {
|
||
const startPos = getCanvasPos(startLm);
|
||
const endPos = getCanvasPos(endLm);
|
||
|
||
ctx.globalAlpha = Math.min(startLm.confidence, endLm.confidence);
|
||
ctx.beginPath();
|
||
ctx.moveTo(startPos.x, startPos.y);
|
||
ctx.lineTo(endPos.x, endPos.y);
|
||
ctx.stroke();
|
||
ctx.globalAlpha = 1.0;
|
||
}
|
||
}
|
||
});
|
||
|
||
// Draw landmarks
|
||
hand.landmarks.forEach((lm, i) => {
|
||
if (lm.confidence > 0.1) {
|
||
const pos = getCanvasPos(lm);
|
||
const radius = 4 + lm.confidence * 2;
|
||
|
||
// Landmark glow
|
||
ctx.fillStyle = color;
|
||
ctx.globalAlpha = lm.confidence * 0.3;
|
||
ctx.beginPath();
|
||
ctx.arc(pos.x, pos.y, radius * 2, 0, Math.PI * 2);
|
||
ctx.fill();
|
||
|
||
// Landmark point
|
||
ctx.fillStyle = color;
|
||
ctx.globalAlpha = lm.confidence;
|
||
ctx.beginPath();
|
||
ctx.arc(pos.x, pos.y, radius, 0, Math.PI * 2);
|
||
ctx.fill();
|
||
|
||
// Joint type marker
|
||
const isJoint = i > 0 && i % 4 === 0; // Tip joints
|
||
if (isJoint) {
|
||
ctx.strokeStyle = '#fff';
|
||
ctx.lineWidth = 1;
|
||
ctx.globalAlpha = lm.confidence;
|
||
ctx.beginPath();
|
||
ctx.arc(pos.x, pos.y, radius + 2, 0, Math.PI * 2);
|
||
ctx.stroke();
|
||
}
|
||
}
|
||
});
|
||
|
||
ctx.globalAlpha = 1.0;
|
||
|
||
// Draw handedness label on canvas
|
||
ctx.fillStyle = color;
|
||
ctx.font = 'bold 12px monospace';
|
||
ctx.textAlign = 'left';
|
||
ctx.fillText(label, 10, height - 10);
|
||
}, [hand, color, label]);
|
||
|
||
return (
|
||
<div className="flex flex-col items-center gap-2">
|
||
<div className="text-xs font-bold text-gray-400">{label}</div>
|
||
<canvas
|
||
ref={canvasRef}
|
||
width={280}
|
||
height={320}
|
||
className="border-2 border-gray-800 rounded-lg bg-gray-900"
|
||
/>
|
||
</div>
|
||
);
|
||
}
|
||
|
||
export function HandTracker({ subscribe }) {
|
||
const [leftHand, setLeftHand] = useState(null);
|
||
const [rightHand, setRightHand] = useState(null);
|
||
const [gesture, setGesture] = useState('');
|
||
const [confidence, setConfidence] = useState(0);
|
||
const handsRef = useRef({});
|
||
|
||
// Subscribe to hand poses
|
||
useEffect(() => {
|
||
const unsubscribe = subscribe(
|
||
'/saltybot/hands',
|
||
'saltybot_msgs/HandPose',
|
||
(msg) => {
|
||
try {
|
||
if (!msg) return;
|
||
|
||
// Handle both single hand and multi-hand formats
|
||
if (Array.isArray(msg)) {
|
||
// Multi-hand format: array of hands
|
||
const left = msg.find((h) => h.handedness === 'Left' || h.handedness === 0);
|
||
const right = msg.find((h) => h.handedness === 'Right' || h.handedness === 1);
|
||
if (left) setLeftHand(left);
|
||
if (right) setRightHand(right);
|
||
} else if (msg.handedness) {
|
||
// Single hand format
|
||
if (msg.handedness === 'Left' || msg.handedness === 0) {
|
||
setLeftHand(msg);
|
||
} else {
|
||
setRightHand(msg);
|
||
}
|
||
}
|
||
|
||
handsRef.current = { left: leftHand, right: rightHand };
|
||
} catch (e) {
|
||
console.error('Error parsing hand pose:', e);
|
||
}
|
||
}
|
||
);
|
||
|
||
return unsubscribe;
|
||
}, [subscribe, leftHand, rightHand]);
|
||
|
||
// Subscribe to gesture
|
||
useEffect(() => {
|
||
const unsubscribe = subscribe(
|
||
'/saltybot/hand_gesture',
|
||
'std_msgs/String',
|
||
(msg) => {
|
||
try {
|
||
if (msg.data) {
|
||
// Parse gesture data (format: "gesture_name confidence")
|
||
const parts = msg.data.split(' ');
|
||
setGesture(parts[0] || '');
|
||
if (parts[1]) {
|
||
setConfidence(parseFloat(parts[1]) || 0);
|
||
}
|
||
}
|
||
} catch (e) {
|
||
console.error('Error parsing gesture:', e);
|
||
}
|
||
}
|
||
);
|
||
|
||
return unsubscribe;
|
||
}, [subscribe]);
|
||
|
||
const hasLeftHand = leftHand && leftHand.landmarks && leftHand.landmarks.length > 0;
|
||
const hasRightHand = rightHand && rightHand.landmarks && rightHand.landmarks.length > 0;
|
||
const gestureConfidentColor =
|
||
confidence > 0.8 ? 'text-green-400' : confidence > 0.5 ? 'text-yellow-400' : 'text-gray-400';
|
||
|
||
return (
|
||
<div className="flex flex-col h-full space-y-3">
|
||
{/* Gesture Display */}
|
||
<div className="bg-gray-950 rounded-lg border border-cyan-950 p-3 space-y-2">
|
||
<div className="flex justify-between items-center">
|
||
<div className="text-cyan-700 text-xs font-bold tracking-widest">GESTURE</div>
|
||
<div className={`text-sm font-mono font-bold ${gestureConfidentColor}`}>
|
||
{gesture || 'Detecting...'}
|
||
</div>
|
||
</div>
|
||
|
||
{/* Confidence bar */}
|
||
{gesture && (
|
||
<div className="space-y-1">
|
||
<div className="text-xs text-gray-600">Confidence</div>
|
||
<div className="w-full bg-gray-900 rounded-full h-2 overflow-hidden">
|
||
<div
|
||
className={`h-full transition-all ${
|
||
confidence > 0.8
|
||
? 'bg-green-500'
|
||
: confidence > 0.5
|
||
? 'bg-yellow-500'
|
||
: 'bg-blue-500'
|
||
}`}
|
||
style={{ width: `${Math.round(confidence * 100)}%` }}
|
||
/>
|
||
</div>
|
||
<div className="text-right text-xs text-gray-500">
|
||
{Math.round(confidence * 100)}%
|
||
</div>
|
||
</div>
|
||
)}
|
||
</div>
|
||
|
||
{/* Hand Renders */}
|
||
<div className="flex-1 overflow-y-auto">
|
||
{hasLeftHand || hasRightHand ? (
|
||
<div className="flex gap-3 justify-center flex-wrap">
|
||
{hasLeftHand && (
|
||
<HandCanvas hand={leftHand} color="#10b981" label="LEFT HAND" />
|
||
)}
|
||
{hasRightHand && (
|
||
<HandCanvas hand={rightHand} color="#f59e0b" label="RIGHT HAND" />
|
||
)}
|
||
</div>
|
||
) : (
|
||
<div className="flex items-center justify-center h-full text-gray-600">
|
||
<div className="text-center">
|
||
<div className="text-sm mb-2">Waiting for hand detection</div>
|
||
<div className="text-xs text-gray-700">
|
||
Hands will appear when detected on /saltybot/hands
|
||
</div>
|
||
</div>
|
||
</div>
|
||
)}
|
||
</div>
|
||
|
||
{/* Hand Info */}
|
||
<div className="bg-gray-950 rounded-lg border border-cyan-950 p-3 space-y-2">
|
||
<div className="text-xs font-bold text-gray-400 tracking-widest mb-2">HAND STATUS</div>
|
||
<div className="grid grid-cols-2 gap-2">
|
||
<div className="bg-green-950 rounded p-2">
|
||
<div className="text-xs text-gray-600">LEFT</div>
|
||
<div className="text-lg font-mono text-green-400">
|
||
{hasLeftHand ? '✓' : '◯'}
|
||
</div>
|
||
</div>
|
||
<div className="bg-yellow-950 rounded p-2">
|
||
<div className="text-xs text-gray-600">RIGHT</div>
|
||
<div className="text-lg font-mono text-yellow-400">
|
||
{hasRightHand ? '✓' : '◯'}
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
{/* Landmark Info */}
|
||
<div className="bg-gray-950 rounded border border-gray-800 p-2 text-xs text-gray-600 space-y-1">
|
||
<div className="flex justify-between">
|
||
<span>Hand Format:</span>
|
||
<span className="text-gray-500">21 landmarks per hand (MediaPipe)</span>
|
||
</div>
|
||
<div className="flex justify-between">
|
||
<span>Topics:</span>
|
||
<span className="text-gray-500">/saltybot/hands, /saltybot/hand_gesture</span>
|
||
</div>
|
||
<div className="flex justify-between">
|
||
<span>Landmarks:</span>
|
||
<span className="text-gray-500">Wrist + fingers (5×4 joints)</span>
|
||
</div>
|
||
<div className="flex justify-between">
|
||
<span>Color Code:</span>
|
||
<span className="text-gray-500">🟢 Left | 🟠 Right</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
);
|
||
}
|