Compare commits
No commits in common. "9b538395c007b0ce9330f8034ed7101ab9c92ee5" and "a0f36777328bb2d0b344c9fcf2a02c8299f93df3" have entirely different histories.
9b538395c0
...
a0f3677732
@ -82,9 +82,6 @@ import { Teleop } from './components/Teleop.jsx';
|
|||||||
// System diagnostics (issue #340)
|
// System diagnostics (issue #340)
|
||||||
import { Diagnostics } from './components/Diagnostics.jsx';
|
import { Diagnostics } from './components/Diagnostics.jsx';
|
||||||
|
|
||||||
// Hand tracking visualization (issue #344)
|
|
||||||
import { HandTracker } from './components/HandTracker.jsx';
|
|
||||||
|
|
||||||
const TAB_GROUPS = [
|
const TAB_GROUPS = [
|
||||||
{
|
{
|
||||||
label: 'SOCIAL',
|
label: 'SOCIAL',
|
||||||
@ -92,7 +89,6 @@ const TAB_GROUPS = [
|
|||||||
tabs: [
|
tabs: [
|
||||||
{ id: 'status', label: 'Status', },
|
{ id: 'status', label: 'Status', },
|
||||||
{ id: 'faces', label: 'Faces', },
|
{ id: 'faces', label: 'Faces', },
|
||||||
{ id: 'hands', label: 'Hands', },
|
|
||||||
{ id: 'conversation', label: 'Convo', },
|
{ id: 'conversation', label: 'Convo', },
|
||||||
{ id: 'history', label: 'History', },
|
{ id: 'history', label: 'History', },
|
||||||
{ id: 'personality', label: 'Personality', },
|
{ id: 'personality', label: 'Personality', },
|
||||||
@ -267,7 +263,6 @@ export default function App() {
|
|||||||
<main className={`flex-1 ${['eventlog', 'control', 'imu'].includes(activeTab) ? 'flex flex-col' : 'overflow-y-auto'} p-4`}>
|
<main className={`flex-1 ${['eventlog', 'control', 'imu'].includes(activeTab) ? 'flex flex-col' : 'overflow-y-auto'} p-4`}>
|
||||||
{activeTab === 'status' && <StatusPanel subscribe={subscribe} />}
|
{activeTab === 'status' && <StatusPanel subscribe={subscribe} />}
|
||||||
{activeTab === 'faces' && <FaceGallery subscribe={subscribe} callService={callService} />}
|
{activeTab === 'faces' && <FaceGallery subscribe={subscribe} callService={callService} />}
|
||||||
{activeTab === 'hands' && <HandTracker subscribe={subscribe} />}
|
|
||||||
{activeTab === 'conversation' && <ConversationLog subscribe={subscribe} />}
|
{activeTab === 'conversation' && <ConversationLog subscribe={subscribe} />}
|
||||||
{activeTab === 'history' && <ConversationHistory subscribe={subscribe} />}
|
{activeTab === 'history' && <ConversationHistory subscribe={subscribe} />}
|
||||||
{activeTab === 'personality' && <PersonalityTuner subscribe={subscribe} setParam={setParam} />}
|
{activeTab === 'personality' && <PersonalityTuner subscribe={subscribe} setParam={setParam} />}
|
||||||
|
|||||||
@ -1,331 +0,0 @@
|
|||||||
/**
|
|
||||||
* HandTracker.jsx — Hand pose and gesture visualization
|
|
||||||
*
|
|
||||||
* Features:
|
|
||||||
* - Subscribes to /saltybot/hands (21 landmarks per hand)
|
|
||||||
* - Subscribes to /saltybot/hand_gesture (String gesture label)
|
|
||||||
* - Canvas-based hand skeleton rendering
|
|
||||||
* - Bone connections between landmarks
|
|
||||||
* - Support for dual hands (left and right)
|
|
||||||
* - Handedness indicator
|
|
||||||
* - Real-time gesture display
|
|
||||||
* - Confidence-based landmark rendering
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { useEffect, useRef, useState } from 'react';
|
|
||||||
|
|
||||||
// MediaPipe hand landmark connections (bones)
|
|
||||||
const HAND_CONNECTIONS = [
|
|
||||||
// Thumb
|
|
||||||
[0, 1], [1, 2], [2, 3], [3, 4],
|
|
||||||
// Index finger
|
|
||||||
[0, 5], [5, 6], [6, 7], [7, 8],
|
|
||||||
// Middle finger
|
|
||||||
[0, 9], [9, 10], [10, 11], [11, 12],
|
|
||||||
// Ring finger
|
|
||||||
[0, 13], [13, 14], [14, 15], [15, 16],
|
|
||||||
// Pinky finger
|
|
||||||
[0, 17], [17, 18], [18, 19], [19, 20],
|
|
||||||
// Palm connections
|
|
||||||
[5, 9], [9, 13], [13, 17],
|
|
||||||
];
|
|
||||||
|
|
||||||
const LANDMARK_NAMES = [
|
|
||||||
'Wrist',
|
|
||||||
'Thumb CMC', 'Thumb MCP', 'Thumb IP', 'Thumb Tip',
|
|
||||||
'Index MCP', 'Index PIP', 'Index DIP', 'Index Tip',
|
|
||||||
'Middle MCP', 'Middle PIP', 'Middle DIP', 'Middle Tip',
|
|
||||||
'Ring MCP', 'Ring PIP', 'Ring DIP', 'Ring Tip',
|
|
||||||
'Pinky MCP', 'Pinky PIP', 'Pinky DIP', 'Pinky Tip',
|
|
||||||
];
|
|
||||||
|
|
||||||
function HandCanvas({ hand, color, label }) {
|
|
||||||
const canvasRef = useRef(null);
|
|
||||||
const [flipped, setFlipped] = useState(false);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
const canvas = canvasRef.current;
|
|
||||||
if (!canvas || !hand || !hand.landmarks || hand.landmarks.length === 0) return;
|
|
||||||
|
|
||||||
const ctx = canvas.getContext('2d');
|
|
||||||
const width = canvas.width;
|
|
||||||
const height = canvas.height;
|
|
||||||
|
|
||||||
// Clear canvas
|
|
||||||
ctx.fillStyle = '#1f2937';
|
|
||||||
ctx.fillRect(0, 0, width, height);
|
|
||||||
|
|
||||||
// Find min/max coordinates for scaling
|
|
||||||
let minX = Infinity, maxX = -Infinity;
|
|
||||||
let minY = Infinity, maxY = -Infinity;
|
|
||||||
|
|
||||||
hand.landmarks.forEach((lm) => {
|
|
||||||
minX = Math.min(minX, lm.x);
|
|
||||||
maxX = Math.max(maxX, lm.x);
|
|
||||||
minY = Math.min(minY, lm.y);
|
|
||||||
maxY = Math.max(maxY, lm.y);
|
|
||||||
});
|
|
||||||
|
|
||||||
const padding = 20;
|
|
||||||
const rangeX = maxX - minX || 1;
|
|
||||||
const rangeY = maxY - minY || 1;
|
|
||||||
const scaleX = (width - padding * 2) / rangeX;
|
|
||||||
const scaleY = (height - padding * 2) / rangeY;
|
|
||||||
const scale = Math.min(scaleX, scaleY);
|
|
||||||
|
|
||||||
// Convert landmark coordinates to canvas positions
|
|
||||||
const getCanvasPos = (lm) => {
|
|
||||||
const x = padding + (lm.x - minX) * scale;
|
|
||||||
const y = padding + (lm.y - minY) * scale;
|
|
||||||
return { x, y };
|
|
||||||
};
|
|
||||||
|
|
||||||
// Draw bones
|
|
||||||
ctx.strokeStyle = color;
|
|
||||||
ctx.lineWidth = 2;
|
|
||||||
ctx.lineCap = 'round';
|
|
||||||
ctx.lineJoin = 'round';
|
|
||||||
|
|
||||||
HAND_CONNECTIONS.forEach(([start, end]) => {
|
|
||||||
if (start < hand.landmarks.length && end < hand.landmarks.length) {
|
|
||||||
const startLm = hand.landmarks[start];
|
|
||||||
const endLm = hand.landmarks[end];
|
|
||||||
|
|
||||||
if (startLm.confidence > 0.1 && endLm.confidence > 0.1) {
|
|
||||||
const startPos = getCanvasPos(startLm);
|
|
||||||
const endPos = getCanvasPos(endLm);
|
|
||||||
|
|
||||||
ctx.globalAlpha = Math.min(startLm.confidence, endLm.confidence);
|
|
||||||
ctx.beginPath();
|
|
||||||
ctx.moveTo(startPos.x, startPos.y);
|
|
||||||
ctx.lineTo(endPos.x, endPos.y);
|
|
||||||
ctx.stroke();
|
|
||||||
ctx.globalAlpha = 1.0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Draw landmarks
|
|
||||||
hand.landmarks.forEach((lm, i) => {
|
|
||||||
if (lm.confidence > 0.1) {
|
|
||||||
const pos = getCanvasPos(lm);
|
|
||||||
const radius = 4 + lm.confidence * 2;
|
|
||||||
|
|
||||||
// Landmark glow
|
|
||||||
ctx.fillStyle = color;
|
|
||||||
ctx.globalAlpha = lm.confidence * 0.3;
|
|
||||||
ctx.beginPath();
|
|
||||||
ctx.arc(pos.x, pos.y, radius * 2, 0, Math.PI * 2);
|
|
||||||
ctx.fill();
|
|
||||||
|
|
||||||
// Landmark point
|
|
||||||
ctx.fillStyle = color;
|
|
||||||
ctx.globalAlpha = lm.confidence;
|
|
||||||
ctx.beginPath();
|
|
||||||
ctx.arc(pos.x, pos.y, radius, 0, Math.PI * 2);
|
|
||||||
ctx.fill();
|
|
||||||
|
|
||||||
// Joint type marker
|
|
||||||
const isJoint = i > 0 && i % 4 === 0; // Tip joints
|
|
||||||
if (isJoint) {
|
|
||||||
ctx.strokeStyle = '#fff';
|
|
||||||
ctx.lineWidth = 1;
|
|
||||||
ctx.globalAlpha = lm.confidence;
|
|
||||||
ctx.beginPath();
|
|
||||||
ctx.arc(pos.x, pos.y, radius + 2, 0, Math.PI * 2);
|
|
||||||
ctx.stroke();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
ctx.globalAlpha = 1.0;
|
|
||||||
|
|
||||||
// Draw handedness label on canvas
|
|
||||||
ctx.fillStyle = color;
|
|
||||||
ctx.font = 'bold 12px monospace';
|
|
||||||
ctx.textAlign = 'left';
|
|
||||||
ctx.fillText(label, 10, height - 10);
|
|
||||||
}, [hand, color, label]);
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="flex flex-col items-center gap-2">
|
|
||||||
<div className="text-xs font-bold text-gray-400">{label}</div>
|
|
||||||
<canvas
|
|
||||||
ref={canvasRef}
|
|
||||||
width={280}
|
|
||||||
height={320}
|
|
||||||
className="border-2 border-gray-800 rounded-lg bg-gray-900"
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function HandTracker({ subscribe }) {
|
|
||||||
const [leftHand, setLeftHand] = useState(null);
|
|
||||||
const [rightHand, setRightHand] = useState(null);
|
|
||||||
const [gesture, setGesture] = useState('');
|
|
||||||
const [confidence, setConfidence] = useState(0);
|
|
||||||
const handsRef = useRef({});
|
|
||||||
|
|
||||||
// Subscribe to hand poses
|
|
||||||
useEffect(() => {
|
|
||||||
const unsubscribe = subscribe(
|
|
||||||
'/saltybot/hands',
|
|
||||||
'saltybot_msgs/HandPose',
|
|
||||||
(msg) => {
|
|
||||||
try {
|
|
||||||
if (!msg) return;
|
|
||||||
|
|
||||||
// Handle both single hand and multi-hand formats
|
|
||||||
if (Array.isArray(msg)) {
|
|
||||||
// Multi-hand format: array of hands
|
|
||||||
const left = msg.find((h) => h.handedness === 'Left' || h.handedness === 0);
|
|
||||||
const right = msg.find((h) => h.handedness === 'Right' || h.handedness === 1);
|
|
||||||
if (left) setLeftHand(left);
|
|
||||||
if (right) setRightHand(right);
|
|
||||||
} else if (msg.handedness) {
|
|
||||||
// Single hand format
|
|
||||||
if (msg.handedness === 'Left' || msg.handedness === 0) {
|
|
||||||
setLeftHand(msg);
|
|
||||||
} else {
|
|
||||||
setRightHand(msg);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
handsRef.current = { left: leftHand, right: rightHand };
|
|
||||||
} catch (e) {
|
|
||||||
console.error('Error parsing hand pose:', e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
return unsubscribe;
|
|
||||||
}, [subscribe, leftHand, rightHand]);
|
|
||||||
|
|
||||||
// Subscribe to gesture
|
|
||||||
useEffect(() => {
|
|
||||||
const unsubscribe = subscribe(
|
|
||||||
'/saltybot/hand_gesture',
|
|
||||||
'std_msgs/String',
|
|
||||||
(msg) => {
|
|
||||||
try {
|
|
||||||
if (msg.data) {
|
|
||||||
// Parse gesture data (format: "gesture_name confidence")
|
|
||||||
const parts = msg.data.split(' ');
|
|
||||||
setGesture(parts[0] || '');
|
|
||||||
if (parts[1]) {
|
|
||||||
setConfidence(parseFloat(parts[1]) || 0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
console.error('Error parsing gesture:', e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
return unsubscribe;
|
|
||||||
}, [subscribe]);
|
|
||||||
|
|
||||||
const hasLeftHand = leftHand && leftHand.landmarks && leftHand.landmarks.length > 0;
|
|
||||||
const hasRightHand = rightHand && rightHand.landmarks && rightHand.landmarks.length > 0;
|
|
||||||
const gestureConfidentColor =
|
|
||||||
confidence > 0.8 ? 'text-green-400' : confidence > 0.5 ? 'text-yellow-400' : 'text-gray-400';
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="flex flex-col h-full space-y-3">
|
|
||||||
{/* Gesture Display */}
|
|
||||||
<div className="bg-gray-950 rounded-lg border border-cyan-950 p-3 space-y-2">
|
|
||||||
<div className="flex justify-between items-center">
|
|
||||||
<div className="text-cyan-700 text-xs font-bold tracking-widest">GESTURE</div>
|
|
||||||
<div className={`text-sm font-mono font-bold ${gestureConfidentColor}`}>
|
|
||||||
{gesture || 'Detecting...'}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{/* Confidence bar */}
|
|
||||||
{gesture && (
|
|
||||||
<div className="space-y-1">
|
|
||||||
<div className="text-xs text-gray-600">Confidence</div>
|
|
||||||
<div className="w-full bg-gray-900 rounded-full h-2 overflow-hidden">
|
|
||||||
<div
|
|
||||||
className={`h-full transition-all ${
|
|
||||||
confidence > 0.8
|
|
||||||
? 'bg-green-500'
|
|
||||||
: confidence > 0.5
|
|
||||||
? 'bg-yellow-500'
|
|
||||||
: 'bg-blue-500'
|
|
||||||
}`}
|
|
||||||
style={{ width: `${Math.round(confidence * 100)}%` }}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
<div className="text-right text-xs text-gray-500">
|
|
||||||
{Math.round(confidence * 100)}%
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{/* Hand Renders */}
|
|
||||||
<div className="flex-1 overflow-y-auto">
|
|
||||||
{hasLeftHand || hasRightHand ? (
|
|
||||||
<div className="flex gap-3 justify-center flex-wrap">
|
|
||||||
{hasLeftHand && (
|
|
||||||
<HandCanvas hand={leftHand} color="#10b981" label="LEFT HAND" />
|
|
||||||
)}
|
|
||||||
{hasRightHand && (
|
|
||||||
<HandCanvas hand={rightHand} color="#f59e0b" label="RIGHT HAND" />
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
) : (
|
|
||||||
<div className="flex items-center justify-center h-full text-gray-600">
|
|
||||||
<div className="text-center">
|
|
||||||
<div className="text-sm mb-2">Waiting for hand detection</div>
|
|
||||||
<div className="text-xs text-gray-700">
|
|
||||||
Hands will appear when detected on /saltybot/hands
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{/* Hand Info */}
|
|
||||||
<div className="bg-gray-950 rounded-lg border border-cyan-950 p-3 space-y-2">
|
|
||||||
<div className="text-xs font-bold text-gray-400 tracking-widest mb-2">HAND STATUS</div>
|
|
||||||
<div className="grid grid-cols-2 gap-2">
|
|
||||||
<div className="bg-green-950 rounded p-2">
|
|
||||||
<div className="text-xs text-gray-600">LEFT</div>
|
|
||||||
<div className="text-lg font-mono text-green-400">
|
|
||||||
{hasLeftHand ? '✓' : '◯'}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div className="bg-yellow-950 rounded p-2">
|
|
||||||
<div className="text-xs text-gray-600">RIGHT</div>
|
|
||||||
<div className="text-lg font-mono text-yellow-400">
|
|
||||||
{hasRightHand ? '✓' : '◯'}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{/* Landmark Info */}
|
|
||||||
<div className="bg-gray-950 rounded border border-gray-800 p-2 text-xs text-gray-600 space-y-1">
|
|
||||||
<div className="flex justify-between">
|
|
||||||
<span>Hand Format:</span>
|
|
||||||
<span className="text-gray-500">21 landmarks per hand (MediaPipe)</span>
|
|
||||||
</div>
|
|
||||||
<div className="flex justify-between">
|
|
||||||
<span>Topics:</span>
|
|
||||||
<span className="text-gray-500">/saltybot/hands, /saltybot/hand_gesture</span>
|
|
||||||
</div>
|
|
||||||
<div className="flex justify-between">
|
|
||||||
<span>Landmarks:</span>
|
|
||||||
<span className="text-gray-500">Wrist + fingers (5×4 joints)</span>
|
|
||||||
</div>
|
|
||||||
<div className="flex justify-between">
|
|
||||||
<span>Color Code:</span>
|
|
||||||
<span className="text-gray-500">🟢 Left | 🟠 Right</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Loading…
x
Reference in New Issue
Block a user