feat(webui): hand pose tracking and gesture visualization (Issue #344)
Features: - Subscribes to /saltybot/hands (21 landmarks per hand - MediaPipe format) - Subscribes to /saltybot/hand_gesture (String gesture label) - Canvas-based hand skeleton rendering with bone connections - Support for dual hand tracking (left and right) - Handedness indicators with color coding * Left hand: green * Right hand: yellow - Real-time gesture display with confidence indicator - Per-landmark confidence visualization - Bone connections between all 21 joints Hand Skeleton Features: - 21 MediaPipe landmarks per hand * Wrist (1) * Thumb (4) * Index finger (4) * Middle finger (4) * Ring finger (4) * Pinky finger (4) - 20 bone connections between joints - Confidence-based rendering (only show high-confidence points) - Scaling and normalization for viewport - Joint type indicators (tips with ring outline) - Glow effects around landmarks Gesture Recognition: - Real-time gesture label display - Confidence percentage (0-100%) - Color-coded confidence: * Green: >80% (high confidence) * Yellow: 50-80% (medium confidence) * Blue: <50% (detecting) Hand Status Display: - Live detection status for both hands - Visual indicators (✓ detected / ◯ not detected) - Dual-hand canvas rendering - Gesture info panel with confidence bar Integration: - Added to SOCIAL tab group as "Hands" tab - Positioned after "Faces" tab - Uses subscribe hook for real-time updates - Dark theme with color-coded hands - Canvas-based rendering for smooth visualization Build: 125 modules, no errors Main bundle: 270.08 KB Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
5156100197
commit
347449ed95
@ -82,6 +82,9 @@ import { Teleop } from './components/Teleop.jsx';
|
||||
// System diagnostics (issue #340)
|
||||
import { Diagnostics } from './components/Diagnostics.jsx';
|
||||
|
||||
// Hand tracking visualization (issue #344)
|
||||
import { HandTracker } from './components/HandTracker.jsx';
|
||||
|
||||
const TAB_GROUPS = [
|
||||
{
|
||||
label: 'SOCIAL',
|
||||
@ -89,6 +92,7 @@ const TAB_GROUPS = [
|
||||
tabs: [
|
||||
{ id: 'status', label: 'Status', },
|
||||
{ id: 'faces', label: 'Faces', },
|
||||
{ id: 'hands', label: 'Hands', },
|
||||
{ id: 'conversation', label: 'Convo', },
|
||||
{ id: 'history', label: 'History', },
|
||||
{ id: 'personality', label: 'Personality', },
|
||||
@ -263,6 +267,7 @@ export default function App() {
|
||||
<main className={`flex-1 ${['eventlog', 'control', 'imu'].includes(activeTab) ? 'flex flex-col' : 'overflow-y-auto'} p-4`}>
|
||||
{activeTab === 'status' && <StatusPanel subscribe={subscribe} />}
|
||||
{activeTab === 'faces' && <FaceGallery subscribe={subscribe} callService={callService} />}
|
||||
{activeTab === 'hands' && <HandTracker subscribe={subscribe} />}
|
||||
{activeTab === 'conversation' && <ConversationLog subscribe={subscribe} />}
|
||||
{activeTab === 'history' && <ConversationHistory subscribe={subscribe} />}
|
||||
{activeTab === 'personality' && <PersonalityTuner subscribe={subscribe} setParam={setParam} />}
|
||||
|
||||
331
ui/social-bot/src/components/HandTracker.jsx
Normal file
331
ui/social-bot/src/components/HandTracker.jsx
Normal file
@ -0,0 +1,331 @@
|
||||
/**
|
||||
* HandTracker.jsx — Hand pose and gesture visualization
|
||||
*
|
||||
* Features:
|
||||
* - Subscribes to /saltybot/hands (21 landmarks per hand)
|
||||
* - Subscribes to /saltybot/hand_gesture (String gesture label)
|
||||
* - Canvas-based hand skeleton rendering
|
||||
* - Bone connections between landmarks
|
||||
* - Support for dual hands (left and right)
|
||||
* - Handedness indicator
|
||||
* - Real-time gesture display
|
||||
* - Confidence-based landmark rendering
|
||||
*/
|
||||
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
|
||||
// MediaPipe hand landmark connections (bones)
|
||||
const HAND_CONNECTIONS = [
|
||||
// Thumb
|
||||
[0, 1], [1, 2], [2, 3], [3, 4],
|
||||
// Index finger
|
||||
[0, 5], [5, 6], [6, 7], [7, 8],
|
||||
// Middle finger
|
||||
[0, 9], [9, 10], [10, 11], [11, 12],
|
||||
// Ring finger
|
||||
[0, 13], [13, 14], [14, 15], [15, 16],
|
||||
// Pinky finger
|
||||
[0, 17], [17, 18], [18, 19], [19, 20],
|
||||
// Palm connections
|
||||
[5, 9], [9, 13], [13, 17],
|
||||
];
|
||||
|
||||
const LANDMARK_NAMES = [
|
||||
'Wrist',
|
||||
'Thumb CMC', 'Thumb MCP', 'Thumb IP', 'Thumb Tip',
|
||||
'Index MCP', 'Index PIP', 'Index DIP', 'Index Tip',
|
||||
'Middle MCP', 'Middle PIP', 'Middle DIP', 'Middle Tip',
|
||||
'Ring MCP', 'Ring PIP', 'Ring DIP', 'Ring Tip',
|
||||
'Pinky MCP', 'Pinky PIP', 'Pinky DIP', 'Pinky Tip',
|
||||
];
|
||||
|
||||
function HandCanvas({ hand, color, label }) {
|
||||
const canvasRef = useRef(null);
|
||||
const [flipped, setFlipped] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas || !hand || !hand.landmarks || hand.landmarks.length === 0) return;
|
||||
|
||||
const ctx = canvas.getContext('2d');
|
||||
const width = canvas.width;
|
||||
const height = canvas.height;
|
||||
|
||||
// Clear canvas
|
||||
ctx.fillStyle = '#1f2937';
|
||||
ctx.fillRect(0, 0, width, height);
|
||||
|
||||
// Find min/max coordinates for scaling
|
||||
let minX = Infinity, maxX = -Infinity;
|
||||
let minY = Infinity, maxY = -Infinity;
|
||||
|
||||
hand.landmarks.forEach((lm) => {
|
||||
minX = Math.min(minX, lm.x);
|
||||
maxX = Math.max(maxX, lm.x);
|
||||
minY = Math.min(minY, lm.y);
|
||||
maxY = Math.max(maxY, lm.y);
|
||||
});
|
||||
|
||||
const padding = 20;
|
||||
const rangeX = maxX - minX || 1;
|
||||
const rangeY = maxY - minY || 1;
|
||||
const scaleX = (width - padding * 2) / rangeX;
|
||||
const scaleY = (height - padding * 2) / rangeY;
|
||||
const scale = Math.min(scaleX, scaleY);
|
||||
|
||||
// Convert landmark coordinates to canvas positions
|
||||
const getCanvasPos = (lm) => {
|
||||
const x = padding + (lm.x - minX) * scale;
|
||||
const y = padding + (lm.y - minY) * scale;
|
||||
return { x, y };
|
||||
};
|
||||
|
||||
// Draw bones
|
||||
ctx.strokeStyle = color;
|
||||
ctx.lineWidth = 2;
|
||||
ctx.lineCap = 'round';
|
||||
ctx.lineJoin = 'round';
|
||||
|
||||
HAND_CONNECTIONS.forEach(([start, end]) => {
|
||||
if (start < hand.landmarks.length && end < hand.landmarks.length) {
|
||||
const startLm = hand.landmarks[start];
|
||||
const endLm = hand.landmarks[end];
|
||||
|
||||
if (startLm.confidence > 0.1 && endLm.confidence > 0.1) {
|
||||
const startPos = getCanvasPos(startLm);
|
||||
const endPos = getCanvasPos(endLm);
|
||||
|
||||
ctx.globalAlpha = Math.min(startLm.confidence, endLm.confidence);
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(startPos.x, startPos.y);
|
||||
ctx.lineTo(endPos.x, endPos.y);
|
||||
ctx.stroke();
|
||||
ctx.globalAlpha = 1.0;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Draw landmarks
|
||||
hand.landmarks.forEach((lm, i) => {
|
||||
if (lm.confidence > 0.1) {
|
||||
const pos = getCanvasPos(lm);
|
||||
const radius = 4 + lm.confidence * 2;
|
||||
|
||||
// Landmark glow
|
||||
ctx.fillStyle = color;
|
||||
ctx.globalAlpha = lm.confidence * 0.3;
|
||||
ctx.beginPath();
|
||||
ctx.arc(pos.x, pos.y, radius * 2, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
|
||||
// Landmark point
|
||||
ctx.fillStyle = color;
|
||||
ctx.globalAlpha = lm.confidence;
|
||||
ctx.beginPath();
|
||||
ctx.arc(pos.x, pos.y, radius, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
|
||||
// Joint type marker
|
||||
const isJoint = i > 0 && i % 4 === 0; // Tip joints
|
||||
if (isJoint) {
|
||||
ctx.strokeStyle = '#fff';
|
||||
ctx.lineWidth = 1;
|
||||
ctx.globalAlpha = lm.confidence;
|
||||
ctx.beginPath();
|
||||
ctx.arc(pos.x, pos.y, radius + 2, 0, Math.PI * 2);
|
||||
ctx.stroke();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
ctx.globalAlpha = 1.0;
|
||||
|
||||
// Draw handedness label on canvas
|
||||
ctx.fillStyle = color;
|
||||
ctx.font = 'bold 12px monospace';
|
||||
ctx.textAlign = 'left';
|
||||
ctx.fillText(label, 10, height - 10);
|
||||
}, [hand, color, label]);
|
||||
|
||||
return (
|
||||
<div className="flex flex-col items-center gap-2">
|
||||
<div className="text-xs font-bold text-gray-400">{label}</div>
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
width={280}
|
||||
height={320}
|
||||
className="border-2 border-gray-800 rounded-lg bg-gray-900"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function HandTracker({ subscribe }) {
|
||||
const [leftHand, setLeftHand] = useState(null);
|
||||
const [rightHand, setRightHand] = useState(null);
|
||||
const [gesture, setGesture] = useState('');
|
||||
const [confidence, setConfidence] = useState(0);
|
||||
const handsRef = useRef({});
|
||||
|
||||
// Subscribe to hand poses
|
||||
useEffect(() => {
|
||||
const unsubscribe = subscribe(
|
||||
'/saltybot/hands',
|
||||
'saltybot_msgs/HandPose',
|
||||
(msg) => {
|
||||
try {
|
||||
if (!msg) return;
|
||||
|
||||
// Handle both single hand and multi-hand formats
|
||||
if (Array.isArray(msg)) {
|
||||
// Multi-hand format: array of hands
|
||||
const left = msg.find((h) => h.handedness === 'Left' || h.handedness === 0);
|
||||
const right = msg.find((h) => h.handedness === 'Right' || h.handedness === 1);
|
||||
if (left) setLeftHand(left);
|
||||
if (right) setRightHand(right);
|
||||
} else if (msg.handedness) {
|
||||
// Single hand format
|
||||
if (msg.handedness === 'Left' || msg.handedness === 0) {
|
||||
setLeftHand(msg);
|
||||
} else {
|
||||
setRightHand(msg);
|
||||
}
|
||||
}
|
||||
|
||||
handsRef.current = { left: leftHand, right: rightHand };
|
||||
} catch (e) {
|
||||
console.error('Error parsing hand pose:', e);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
return unsubscribe;
|
||||
}, [subscribe, leftHand, rightHand]);
|
||||
|
||||
// Subscribe to gesture
|
||||
useEffect(() => {
|
||||
const unsubscribe = subscribe(
|
||||
'/saltybot/hand_gesture',
|
||||
'std_msgs/String',
|
||||
(msg) => {
|
||||
try {
|
||||
if (msg.data) {
|
||||
// Parse gesture data (format: "gesture_name confidence")
|
||||
const parts = msg.data.split(' ');
|
||||
setGesture(parts[0] || '');
|
||||
if (parts[1]) {
|
||||
setConfidence(parseFloat(parts[1]) || 0);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing gesture:', e);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
return unsubscribe;
|
||||
}, [subscribe]);
|
||||
|
||||
const hasLeftHand = leftHand && leftHand.landmarks && leftHand.landmarks.length > 0;
|
||||
const hasRightHand = rightHand && rightHand.landmarks && rightHand.landmarks.length > 0;
|
||||
const gestureConfidentColor =
|
||||
confidence > 0.8 ? 'text-green-400' : confidence > 0.5 ? 'text-yellow-400' : 'text-gray-400';
|
||||
|
||||
return (
|
||||
<div className="flex flex-col h-full space-y-3">
|
||||
{/* Gesture Display */}
|
||||
<div className="bg-gray-950 rounded-lg border border-cyan-950 p-3 space-y-2">
|
||||
<div className="flex justify-between items-center">
|
||||
<div className="text-cyan-700 text-xs font-bold tracking-widest">GESTURE</div>
|
||||
<div className={`text-sm font-mono font-bold ${gestureConfidentColor}`}>
|
||||
{gesture || 'Detecting...'}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Confidence bar */}
|
||||
{gesture && (
|
||||
<div className="space-y-1">
|
||||
<div className="text-xs text-gray-600">Confidence</div>
|
||||
<div className="w-full bg-gray-900 rounded-full h-2 overflow-hidden">
|
||||
<div
|
||||
className={`h-full transition-all ${
|
||||
confidence > 0.8
|
||||
? 'bg-green-500'
|
||||
: confidence > 0.5
|
||||
? 'bg-yellow-500'
|
||||
: 'bg-blue-500'
|
||||
}`}
|
||||
style={{ width: `${Math.round(confidence * 100)}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className="text-right text-xs text-gray-500">
|
||||
{Math.round(confidence * 100)}%
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Hand Renders */}
|
||||
<div className="flex-1 overflow-y-auto">
|
||||
{hasLeftHand || hasRightHand ? (
|
||||
<div className="flex gap-3 justify-center flex-wrap">
|
||||
{hasLeftHand && (
|
||||
<HandCanvas hand={leftHand} color="#10b981" label="LEFT HAND" />
|
||||
)}
|
||||
{hasRightHand && (
|
||||
<HandCanvas hand={rightHand} color="#f59e0b" label="RIGHT HAND" />
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex items-center justify-center h-full text-gray-600">
|
||||
<div className="text-center">
|
||||
<div className="text-sm mb-2">Waiting for hand detection</div>
|
||||
<div className="text-xs text-gray-700">
|
||||
Hands will appear when detected on /saltybot/hands
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Hand Info */}
|
||||
<div className="bg-gray-950 rounded-lg border border-cyan-950 p-3 space-y-2">
|
||||
<div className="text-xs font-bold text-gray-400 tracking-widest mb-2">HAND STATUS</div>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<div className="bg-green-950 rounded p-2">
|
||||
<div className="text-xs text-gray-600">LEFT</div>
|
||||
<div className="text-lg font-mono text-green-400">
|
||||
{hasLeftHand ? '✓' : '◯'}
|
||||
</div>
|
||||
</div>
|
||||
<div className="bg-yellow-950 rounded p-2">
|
||||
<div className="text-xs text-gray-600">RIGHT</div>
|
||||
<div className="text-lg font-mono text-yellow-400">
|
||||
{hasRightHand ? '✓' : '◯'}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Landmark Info */}
|
||||
<div className="bg-gray-950 rounded border border-gray-800 p-2 text-xs text-gray-600 space-y-1">
|
||||
<div className="flex justify-between">
|
||||
<span>Hand Format:</span>
|
||||
<span className="text-gray-500">21 landmarks per hand (MediaPipe)</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span>Topics:</span>
|
||||
<span className="text-gray-500">/saltybot/hands, /saltybot/hand_gesture</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span>Landmarks:</span>
|
||||
<span className="text-gray-500">Wrist + fingers (5×4 joints)</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span>Color Code:</span>
|
||||
<span className="text-gray-500">🟢 Left | 🟠 Right</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user