feat(webui): hand tracking skeleton visualization (Issue #344) #346
@ -82,6 +82,9 @@ import { Teleop } from './components/Teleop.jsx';
|
|||||||
// System diagnostics (issue #340)
|
// System diagnostics (issue #340)
|
||||||
import { Diagnostics } from './components/Diagnostics.jsx';
|
import { Diagnostics } from './components/Diagnostics.jsx';
|
||||||
|
|
||||||
|
// Hand tracking visualization (issue #344)
|
||||||
|
import { HandTracker } from './components/HandTracker.jsx';
|
||||||
|
|
||||||
const TAB_GROUPS = [
|
const TAB_GROUPS = [
|
||||||
{
|
{
|
||||||
label: 'SOCIAL',
|
label: 'SOCIAL',
|
||||||
@ -89,6 +92,7 @@ const TAB_GROUPS = [
|
|||||||
tabs: [
|
tabs: [
|
||||||
{ id: 'status', label: 'Status', },
|
{ id: 'status', label: 'Status', },
|
||||||
{ id: 'faces', label: 'Faces', },
|
{ id: 'faces', label: 'Faces', },
|
||||||
|
{ id: 'hands', label: 'Hands', },
|
||||||
{ id: 'conversation', label: 'Convo', },
|
{ id: 'conversation', label: 'Convo', },
|
||||||
{ id: 'history', label: 'History', },
|
{ id: 'history', label: 'History', },
|
||||||
{ id: 'personality', label: 'Personality', },
|
{ id: 'personality', label: 'Personality', },
|
||||||
@ -263,6 +267,7 @@ export default function App() {
|
|||||||
<main className={`flex-1 ${['eventlog', 'control', 'imu'].includes(activeTab) ? 'flex flex-col' : 'overflow-y-auto'} p-4`}>
|
<main className={`flex-1 ${['eventlog', 'control', 'imu'].includes(activeTab) ? 'flex flex-col' : 'overflow-y-auto'} p-4`}>
|
||||||
{activeTab === 'status' && <StatusPanel subscribe={subscribe} />}
|
{activeTab === 'status' && <StatusPanel subscribe={subscribe} />}
|
||||||
{activeTab === 'faces' && <FaceGallery subscribe={subscribe} callService={callService} />}
|
{activeTab === 'faces' && <FaceGallery subscribe={subscribe} callService={callService} />}
|
||||||
|
{activeTab === 'hands' && <HandTracker subscribe={subscribe} />}
|
||||||
{activeTab === 'conversation' && <ConversationLog subscribe={subscribe} />}
|
{activeTab === 'conversation' && <ConversationLog subscribe={subscribe} />}
|
||||||
{activeTab === 'history' && <ConversationHistory subscribe={subscribe} />}
|
{activeTab === 'history' && <ConversationHistory subscribe={subscribe} />}
|
||||||
{activeTab === 'personality' && <PersonalityTuner subscribe={subscribe} setParam={setParam} />}
|
{activeTab === 'personality' && <PersonalityTuner subscribe={subscribe} setParam={setParam} />}
|
||||||
|
|||||||
331
ui/social-bot/src/components/HandTracker.jsx
Normal file
331
ui/social-bot/src/components/HandTracker.jsx
Normal file
@ -0,0 +1,331 @@
|
|||||||
|
/**
|
||||||
|
* HandTracker.jsx — Hand pose and gesture visualization
|
||||||
|
*
|
||||||
|
* Features:
|
||||||
|
* - Subscribes to /saltybot/hands (21 landmarks per hand)
|
||||||
|
* - Subscribes to /saltybot/hand_gesture (String gesture label)
|
||||||
|
* - Canvas-based hand skeleton rendering
|
||||||
|
* - Bone connections between landmarks
|
||||||
|
* - Support for dual hands (left and right)
|
||||||
|
* - Handedness indicator
|
||||||
|
* - Real-time gesture display
|
||||||
|
* - Confidence-based landmark rendering
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { useEffect, useRef, useState } from 'react';
|
||||||
|
|
||||||
|
// MediaPipe hand landmark connections (bones)
|
||||||
|
const HAND_CONNECTIONS = [
|
||||||
|
// Thumb
|
||||||
|
[0, 1], [1, 2], [2, 3], [3, 4],
|
||||||
|
// Index finger
|
||||||
|
[0, 5], [5, 6], [6, 7], [7, 8],
|
||||||
|
// Middle finger
|
||||||
|
[0, 9], [9, 10], [10, 11], [11, 12],
|
||||||
|
// Ring finger
|
||||||
|
[0, 13], [13, 14], [14, 15], [15, 16],
|
||||||
|
// Pinky finger
|
||||||
|
[0, 17], [17, 18], [18, 19], [19, 20],
|
||||||
|
// Palm connections
|
||||||
|
[5, 9], [9, 13], [13, 17],
|
||||||
|
];
|
||||||
|
|
||||||
|
const LANDMARK_NAMES = [
|
||||||
|
'Wrist',
|
||||||
|
'Thumb CMC', 'Thumb MCP', 'Thumb IP', 'Thumb Tip',
|
||||||
|
'Index MCP', 'Index PIP', 'Index DIP', 'Index Tip',
|
||||||
|
'Middle MCP', 'Middle PIP', 'Middle DIP', 'Middle Tip',
|
||||||
|
'Ring MCP', 'Ring PIP', 'Ring DIP', 'Ring Tip',
|
||||||
|
'Pinky MCP', 'Pinky PIP', 'Pinky DIP', 'Pinky Tip',
|
||||||
|
];
|
||||||
|
|
||||||
|
function HandCanvas({ hand, color, label }) {
|
||||||
|
const canvasRef = useRef(null);
|
||||||
|
const [flipped, setFlipped] = useState(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const canvas = canvasRef.current;
|
||||||
|
if (!canvas || !hand || !hand.landmarks || hand.landmarks.length === 0) return;
|
||||||
|
|
||||||
|
const ctx = canvas.getContext('2d');
|
||||||
|
const width = canvas.width;
|
||||||
|
const height = canvas.height;
|
||||||
|
|
||||||
|
// Clear canvas
|
||||||
|
ctx.fillStyle = '#1f2937';
|
||||||
|
ctx.fillRect(0, 0, width, height);
|
||||||
|
|
||||||
|
// Find min/max coordinates for scaling
|
||||||
|
let minX = Infinity, maxX = -Infinity;
|
||||||
|
let minY = Infinity, maxY = -Infinity;
|
||||||
|
|
||||||
|
hand.landmarks.forEach((lm) => {
|
||||||
|
minX = Math.min(minX, lm.x);
|
||||||
|
maxX = Math.max(maxX, lm.x);
|
||||||
|
minY = Math.min(minY, lm.y);
|
||||||
|
maxY = Math.max(maxY, lm.y);
|
||||||
|
});
|
||||||
|
|
||||||
|
const padding = 20;
|
||||||
|
const rangeX = maxX - minX || 1;
|
||||||
|
const rangeY = maxY - minY || 1;
|
||||||
|
const scaleX = (width - padding * 2) / rangeX;
|
||||||
|
const scaleY = (height - padding * 2) / rangeY;
|
||||||
|
const scale = Math.min(scaleX, scaleY);
|
||||||
|
|
||||||
|
// Convert landmark coordinates to canvas positions
|
||||||
|
const getCanvasPos = (lm) => {
|
||||||
|
const x = padding + (lm.x - minX) * scale;
|
||||||
|
const y = padding + (lm.y - minY) * scale;
|
||||||
|
return { x, y };
|
||||||
|
};
|
||||||
|
|
||||||
|
// Draw bones
|
||||||
|
ctx.strokeStyle = color;
|
||||||
|
ctx.lineWidth = 2;
|
||||||
|
ctx.lineCap = 'round';
|
||||||
|
ctx.lineJoin = 'round';
|
||||||
|
|
||||||
|
HAND_CONNECTIONS.forEach(([start, end]) => {
|
||||||
|
if (start < hand.landmarks.length && end < hand.landmarks.length) {
|
||||||
|
const startLm = hand.landmarks[start];
|
||||||
|
const endLm = hand.landmarks[end];
|
||||||
|
|
||||||
|
if (startLm.confidence > 0.1 && endLm.confidence > 0.1) {
|
||||||
|
const startPos = getCanvasPos(startLm);
|
||||||
|
const endPos = getCanvasPos(endLm);
|
||||||
|
|
||||||
|
ctx.globalAlpha = Math.min(startLm.confidence, endLm.confidence);
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.moveTo(startPos.x, startPos.y);
|
||||||
|
ctx.lineTo(endPos.x, endPos.y);
|
||||||
|
ctx.stroke();
|
||||||
|
ctx.globalAlpha = 1.0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Draw landmarks
|
||||||
|
hand.landmarks.forEach((lm, i) => {
|
||||||
|
if (lm.confidence > 0.1) {
|
||||||
|
const pos = getCanvasPos(lm);
|
||||||
|
const radius = 4 + lm.confidence * 2;
|
||||||
|
|
||||||
|
// Landmark glow
|
||||||
|
ctx.fillStyle = color;
|
||||||
|
ctx.globalAlpha = lm.confidence * 0.3;
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.arc(pos.x, pos.y, radius * 2, 0, Math.PI * 2);
|
||||||
|
ctx.fill();
|
||||||
|
|
||||||
|
// Landmark point
|
||||||
|
ctx.fillStyle = color;
|
||||||
|
ctx.globalAlpha = lm.confidence;
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.arc(pos.x, pos.y, radius, 0, Math.PI * 2);
|
||||||
|
ctx.fill();
|
||||||
|
|
||||||
|
// Joint type marker
|
||||||
|
const isJoint = i > 0 && i % 4 === 0; // Tip joints
|
||||||
|
if (isJoint) {
|
||||||
|
ctx.strokeStyle = '#fff';
|
||||||
|
ctx.lineWidth = 1;
|
||||||
|
ctx.globalAlpha = lm.confidence;
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.arc(pos.x, pos.y, radius + 2, 0, Math.PI * 2);
|
||||||
|
ctx.stroke();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
ctx.globalAlpha = 1.0;
|
||||||
|
|
||||||
|
// Draw handedness label on canvas
|
||||||
|
ctx.fillStyle = color;
|
||||||
|
ctx.font = 'bold 12px monospace';
|
||||||
|
ctx.textAlign = 'left';
|
||||||
|
ctx.fillText(label, 10, height - 10);
|
||||||
|
}, [hand, color, label]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col items-center gap-2">
|
||||||
|
<div className="text-xs font-bold text-gray-400">{label}</div>
|
||||||
|
<canvas
|
||||||
|
ref={canvasRef}
|
||||||
|
width={280}
|
||||||
|
height={320}
|
||||||
|
className="border-2 border-gray-800 rounded-lg bg-gray-900"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function HandTracker({ subscribe }) {
|
||||||
|
const [leftHand, setLeftHand] = useState(null);
|
||||||
|
const [rightHand, setRightHand] = useState(null);
|
||||||
|
const [gesture, setGesture] = useState('');
|
||||||
|
const [confidence, setConfidence] = useState(0);
|
||||||
|
const handsRef = useRef({});
|
||||||
|
|
||||||
|
// Subscribe to hand poses
|
||||||
|
useEffect(() => {
|
||||||
|
const unsubscribe = subscribe(
|
||||||
|
'/saltybot/hands',
|
||||||
|
'saltybot_msgs/HandPose',
|
||||||
|
(msg) => {
|
||||||
|
try {
|
||||||
|
if (!msg) return;
|
||||||
|
|
||||||
|
// Handle both single hand and multi-hand formats
|
||||||
|
if (Array.isArray(msg)) {
|
||||||
|
// Multi-hand format: array of hands
|
||||||
|
const left = msg.find((h) => h.handedness === 'Left' || h.handedness === 0);
|
||||||
|
const right = msg.find((h) => h.handedness === 'Right' || h.handedness === 1);
|
||||||
|
if (left) setLeftHand(left);
|
||||||
|
if (right) setRightHand(right);
|
||||||
|
} else if (msg.handedness) {
|
||||||
|
// Single hand format
|
||||||
|
if (msg.handedness === 'Left' || msg.handedness === 0) {
|
||||||
|
setLeftHand(msg);
|
||||||
|
} else {
|
||||||
|
setRightHand(msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
handsRef.current = { left: leftHand, right: rightHand };
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Error parsing hand pose:', e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
return unsubscribe;
|
||||||
|
}, [subscribe, leftHand, rightHand]);
|
||||||
|
|
||||||
|
// Subscribe to gesture
|
||||||
|
useEffect(() => {
|
||||||
|
const unsubscribe = subscribe(
|
||||||
|
'/saltybot/hand_gesture',
|
||||||
|
'std_msgs/String',
|
||||||
|
(msg) => {
|
||||||
|
try {
|
||||||
|
if (msg.data) {
|
||||||
|
// Parse gesture data (format: "gesture_name confidence")
|
||||||
|
const parts = msg.data.split(' ');
|
||||||
|
setGesture(parts[0] || '');
|
||||||
|
if (parts[1]) {
|
||||||
|
setConfidence(parseFloat(parts[1]) || 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Error parsing gesture:', e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
return unsubscribe;
|
||||||
|
}, [subscribe]);
|
||||||
|
|
||||||
|
const hasLeftHand = leftHand && leftHand.landmarks && leftHand.landmarks.length > 0;
|
||||||
|
const hasRightHand = rightHand && rightHand.landmarks && rightHand.landmarks.length > 0;
|
||||||
|
const gestureConfidentColor =
|
||||||
|
confidence > 0.8 ? 'text-green-400' : confidence > 0.5 ? 'text-yellow-400' : 'text-gray-400';
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col h-full space-y-3">
|
||||||
|
{/* Gesture Display */}
|
||||||
|
<div className="bg-gray-950 rounded-lg border border-cyan-950 p-3 space-y-2">
|
||||||
|
<div className="flex justify-between items-center">
|
||||||
|
<div className="text-cyan-700 text-xs font-bold tracking-widest">GESTURE</div>
|
||||||
|
<div className={`text-sm font-mono font-bold ${gestureConfidentColor}`}>
|
||||||
|
{gesture || 'Detecting...'}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Confidence bar */}
|
||||||
|
{gesture && (
|
||||||
|
<div className="space-y-1">
|
||||||
|
<div className="text-xs text-gray-600">Confidence</div>
|
||||||
|
<div className="w-full bg-gray-900 rounded-full h-2 overflow-hidden">
|
||||||
|
<div
|
||||||
|
className={`h-full transition-all ${
|
||||||
|
confidence > 0.8
|
||||||
|
? 'bg-green-500'
|
||||||
|
: confidence > 0.5
|
||||||
|
? 'bg-yellow-500'
|
||||||
|
: 'bg-blue-500'
|
||||||
|
}`}
|
||||||
|
style={{ width: `${Math.round(confidence * 100)}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className="text-right text-xs text-gray-500">
|
||||||
|
{Math.round(confidence * 100)}%
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Hand Renders */}
|
||||||
|
<div className="flex-1 overflow-y-auto">
|
||||||
|
{hasLeftHand || hasRightHand ? (
|
||||||
|
<div className="flex gap-3 justify-center flex-wrap">
|
||||||
|
{hasLeftHand && (
|
||||||
|
<HandCanvas hand={leftHand} color="#10b981" label="LEFT HAND" />
|
||||||
|
)}
|
||||||
|
{hasRightHand && (
|
||||||
|
<HandCanvas hand={rightHand} color="#f59e0b" label="RIGHT HAND" />
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="flex items-center justify-center h-full text-gray-600">
|
||||||
|
<div className="text-center">
|
||||||
|
<div className="text-sm mb-2">Waiting for hand detection</div>
|
||||||
|
<div className="text-xs text-gray-700">
|
||||||
|
Hands will appear when detected on /saltybot/hands
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Hand Info */}
|
||||||
|
<div className="bg-gray-950 rounded-lg border border-cyan-950 p-3 space-y-2">
|
||||||
|
<div className="text-xs font-bold text-gray-400 tracking-widest mb-2">HAND STATUS</div>
|
||||||
|
<div className="grid grid-cols-2 gap-2">
|
||||||
|
<div className="bg-green-950 rounded p-2">
|
||||||
|
<div className="text-xs text-gray-600">LEFT</div>
|
||||||
|
<div className="text-lg font-mono text-green-400">
|
||||||
|
{hasLeftHand ? '✓' : '◯'}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="bg-yellow-950 rounded p-2">
|
||||||
|
<div className="text-xs text-gray-600">RIGHT</div>
|
||||||
|
<div className="text-lg font-mono text-yellow-400">
|
||||||
|
{hasRightHand ? '✓' : '◯'}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Landmark Info */}
|
||||||
|
<div className="bg-gray-950 rounded border border-gray-800 p-2 text-xs text-gray-600 space-y-1">
|
||||||
|
<div className="flex justify-between">
|
||||||
|
<span>Hand Format:</span>
|
||||||
|
<span className="text-gray-500">21 landmarks per hand (MediaPipe)</span>
|
||||||
|
</div>
|
||||||
|
<div className="flex justify-between">
|
||||||
|
<span>Topics:</span>
|
||||||
|
<span className="text-gray-500">/saltybot/hands, /saltybot/hand_gesture</span>
|
||||||
|
</div>
|
||||||
|
<div className="flex justify-between">
|
||||||
|
<span>Landmarks:</span>
|
||||||
|
<span className="text-gray-500">Wrist + fingers (5×4 joints)</span>
|
||||||
|
</div>
|
||||||
|
<div className="flex justify-between">
|
||||||
|
<span>Color Code:</span>
|
||||||
|
<span className="text-gray-500">🟢 Left | 🟠 Right</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user