Merge pull request 'feat: Salty Face animated expression UI (Issue #370)' (#379) from sl-webui/issue-370-salty-face into main
This commit is contained in:
commit
3a639507c7
@ -85,7 +85,17 @@ import { Diagnostics } from './components/Diagnostics.jsx';
|
||||
// Hand tracking visualization (issue #344)
|
||||
import { HandTracker } from './components/HandTracker.jsx';
|
||||
|
||||
// Salty Face animated expression UI (issue #370)
|
||||
import { SaltyFace } from './components/SaltyFace.jsx';
|
||||
|
||||
const TAB_GROUPS = [
|
||||
{
|
||||
label: 'DISPLAY',
|
||||
color: 'text-rose-600',
|
||||
tabs: [
|
||||
{ id: 'salty-face', label: 'Salty Face', },
|
||||
],
|
||||
},
|
||||
{
|
||||
label: 'SOCIAL',
|
||||
color: 'text-cyan-600',
|
||||
@ -270,7 +280,12 @@ export default function App() {
|
||||
</nav>
|
||||
|
||||
{/* ── Content ── */}
|
||||
<main className={`flex-1 ${['eventlog', 'control', 'imu'].includes(activeTab) ? 'flex flex-col' : 'overflow-y-auto'} p-4`}>
|
||||
<main className={`flex-1 ${
|
||||
activeTab === 'salty-face' ? '' :
|
||||
['eventlog', 'control', 'imu'].includes(activeTab) ? 'flex flex-col' : 'overflow-y-auto'
|
||||
} ${activeTab === 'salty-face' ? '' : 'p-4'}`}>
|
||||
{activeTab === 'salty-face' && <SaltyFace subscribe={subscribe} />}
|
||||
|
||||
{activeTab === 'status' && <StatusPanel subscribe={subscribe} />}
|
||||
{activeTab === 'faces' && <FaceGallery subscribe={subscribe} callService={callService} />}
|
||||
{activeTab === 'hands' && <HandTracker subscribe={subscribe} />}
|
||||
|
||||
400
ui/social-bot/src/components/SaltyFace.jsx
Normal file
400
ui/social-bot/src/components/SaltyFace.jsx
Normal file
@ -0,0 +1,400 @@
|
||||
/**
|
||||
* SaltyFace.jsx — Animated facial expression UI for MageDok 7" display
|
||||
*
|
||||
* Features:
|
||||
* - 8 emotional states (happy, alert, confused, sleeping, excited, emergency, listening, talking)
|
||||
* - GPU-accelerated Canvas/SVG rendering (target 30fps on Orin Nano)
|
||||
* - ROS2 integration: /saltybot/state, /saltybot/target_track, /saltybot/obstacles
|
||||
* - Mouth animation synchronized with TTS audio
|
||||
* - HUD overlay: battery, speed, distance, sensor health
|
||||
* - Tap-to-toggle status overlay
|
||||
* - Inspired by Cozmo/Vector minimalist design
|
||||
*/
|
||||
|
||||
import React, { useState, useEffect, useRef, useCallback } from 'react';
|
||||
|
||||
// Emotion states
|
||||
const EMOTIONS = {
|
||||
HAPPY: 'happy', // Default, normal operation
|
||||
ALERT: 'alert', // Obstacles detected
|
||||
CONFUSED: 'confused', // Target lost, searching
|
||||
SLEEPING: 'sleeping', // Prolonged inactivity
|
||||
EXCITED: 'excited', // Target reacquired
|
||||
EMERGENCY: 'emergency', // E-stop activated
|
||||
LISTENING: 'listening', // Microphone active
|
||||
TALKING: 'talking', // Text-to-speech output
|
||||
};
|
||||
|
||||
// Eye characteristics per emotion
|
||||
const EMOTION_CONFIG = {
|
||||
[EMOTIONS.HAPPY]: {
|
||||
eyeScale: 1.0,
|
||||
pupilPos: { x: 0, y: 0 },
|
||||
blink: true,
|
||||
blinkRate: 3000,
|
||||
color: '#10b981',
|
||||
},
|
||||
[EMOTIONS.ALERT]: {
|
||||
eyeScale: 1.3,
|
||||
pupilPos: { x: 0, y: -3 },
|
||||
blink: false,
|
||||
blinkRate: 0,
|
||||
color: '#ef4444',
|
||||
},
|
||||
[EMOTIONS.CONFUSED]: {
|
||||
eyeScale: 1.1,
|
||||
pupilPos: { x: 0, y: 0 },
|
||||
blink: true,
|
||||
blinkRate: 1500,
|
||||
eyeWander: true,
|
||||
color: '#f59e0b',
|
||||
},
|
||||
[EMOTIONS.SLEEPING]: {
|
||||
eyeScale: 0.3,
|
||||
pupilPos: { x: 0, y: 0 },
|
||||
blink: false,
|
||||
isClosed: true,
|
||||
color: '#6b7280',
|
||||
},
|
||||
[EMOTIONS.EXCITED]: {
|
||||
eyeScale: 1.2,
|
||||
pupilPos: { x: 0, y: 0 },
|
||||
blink: true,
|
||||
blinkRate: 800,
|
||||
bounce: true,
|
||||
color: '#22c55e',
|
||||
},
|
||||
[EMOTIONS.EMERGENCY]: {
|
||||
eyeScale: 1.4,
|
||||
pupilPos: { x: 0, y: -4 },
|
||||
blink: false,
|
||||
color: '#dc2626',
|
||||
flash: true,
|
||||
},
|
||||
[EMOTIONS.LISTENING]: {
|
||||
eyeScale: 1.0,
|
||||
pupilPos: { x: 0, y: -2 },
|
||||
blink: true,
|
||||
blinkRate: 2000,
|
||||
color: '#0ea5e9',
|
||||
},
|
||||
[EMOTIONS.TALKING]: {
|
||||
eyeScale: 1.0,
|
||||
pupilPos: { x: 0, y: 0 },
|
||||
blink: true,
|
||||
blinkRate: 2500,
|
||||
color: '#06b6d4',
|
||||
},
|
||||
};
|
||||
|
||||
// Mouth states for talking
|
||||
const MOUTH_FRAMES = [
|
||||
{ open: 0.0, shape: 'closed' }, // Closed
|
||||
{ open: 0.3, shape: 'smile-closed' }, // Slight smile
|
||||
{ open: 0.5, shape: 'smile-open' }, // Smile open
|
||||
{ open: 0.7, shape: 'oh' }, // "Oh" sound
|
||||
{ open: 0.9, shape: 'ah' }, // "Ah" sound
|
||||
{ open: 0.7, shape: 'ee' }, // "Ee" sound
|
||||
];
|
||||
|
||||
// Canvas-based face renderer
|
||||
function FaceCanvas({ emotion, isTalking, audioLevel, showOverlay, botState }) {
|
||||
const canvasRef = useRef(null);
|
||||
const animationRef = useRef(null);
|
||||
const [eyeWanderOffset, setEyeWanderOffset] = useState({ x: 0, y: 0 });
|
||||
const [mouthFrame, setMouthFrame] = useState(0);
|
||||
const [isBlinking, setIsBlinking] = useState(false);
|
||||
const talkingCounterRef = useRef(0);
|
||||
|
||||
// Main animation loop
|
||||
useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) return;
|
||||
|
||||
const ctx = canvas.getContext('2d', { alpha: true });
|
||||
const W = canvas.width;
|
||||
const H = canvas.height;
|
||||
let frameCount = 0;
|
||||
|
||||
const config = EMOTION_CONFIG[emotion] || EMOTION_CONFIG[EMOTIONS.HAPPY];
|
||||
|
||||
const drawFace = () => {
|
||||
// Clear canvas
|
||||
ctx.fillStyle = 'rgba(5, 5, 16, 0.95)';
|
||||
ctx.fillRect(0, 0, W, H);
|
||||
|
||||
const centerX = W / 2;
|
||||
const centerY = H / 2.2;
|
||||
const eyeRadius = 40;
|
||||
const eyeSpacing = 80;
|
||||
|
||||
// Eye wandering animation (for confused state)
|
||||
let eyeOffX = 0, eyeOffY = 0;
|
||||
if (config.eyeWander) {
|
||||
eyeOffX = Math.sin(frameCount * 0.02) * 8;
|
||||
eyeOffY = Math.cos(frameCount * 0.015) * 8;
|
||||
}
|
||||
|
||||
// Bounce animation (for excited state)
|
||||
let bounceOffset = 0;
|
||||
if (config.bounce) {
|
||||
bounceOffset = Math.sin(frameCount * 0.08) * 6;
|
||||
}
|
||||
|
||||
// Draw eyes
|
||||
const eyeY = centerY + bounceOffset;
|
||||
drawEye(ctx, centerX - eyeSpacing, eyeY + eyeOffY, eyeRadius, config, isBlinking);
|
||||
drawEye(ctx, centerX + eyeSpacing, eyeY + eyeOffY, eyeRadius, config, isBlinking);
|
||||
|
||||
// Draw mouth (if talking)
|
||||
if (isTalking && !config.isClosed) {
|
||||
drawMouth(ctx, centerX, centerY + 80, 50, MOUTH_FRAMES[mouthFrame]);
|
||||
}
|
||||
|
||||
// Flash animation (emergency state)
|
||||
if (config.flash && Math.sin(frameCount * 0.1) > 0.7) {
|
||||
ctx.fillStyle = 'rgba(220, 38, 38, 0.3)';
|
||||
ctx.fillRect(0, 0, W, H);
|
||||
}
|
||||
|
||||
frameCount++;
|
||||
};
|
||||
|
||||
const animationLoop = () => {
|
||||
drawFace();
|
||||
animationRef.current = requestAnimationFrame(animationLoop);
|
||||
};
|
||||
|
||||
animationLoop();
|
||||
|
||||
return () => {
|
||||
if (animationRef.current) {
|
||||
cancelAnimationFrame(animationRef.current);
|
||||
}
|
||||
};
|
||||
}, [emotion, isTalking, isBlinking, mouthFrame]);
|
||||
|
||||
// Blinking logic
|
||||
useEffect(() => {
|
||||
const config = EMOTION_CONFIG[emotion] || EMOTION_CONFIG[EMOTIONS.HAPPY];
|
||||
if (!config.blink || config.isClosed) return;
|
||||
|
||||
const blinkInterval = setInterval(() => {
|
||||
setIsBlinking(true);
|
||||
setTimeout(() => setIsBlinking(false), 150);
|
||||
}, config.blinkRate);
|
||||
|
||||
return () => clearInterval(blinkInterval);
|
||||
}, [emotion]);
|
||||
|
||||
// Mouth animation for talking
|
||||
useEffect(() => {
|
||||
if (!isTalking) {
|
||||
setMouthFrame(0);
|
||||
return;
|
||||
}
|
||||
|
||||
let frameIndex = 0;
|
||||
const mouthInterval = setInterval(() => {
|
||||
frameIndex = (frameIndex + 1) % MOUTH_FRAMES.length;
|
||||
setMouthFrame(frameIndex);
|
||||
}, 100); // ~10fps mouth animation
|
||||
|
||||
return () => clearInterval(mouthInterval);
|
||||
}, [isTalking]);
|
||||
|
||||
return (
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
width={1024}
|
||||
height={600}
|
||||
className="w-full h-full block"
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
// Draw individual eye
|
||||
function drawEye(ctx, x, y, radius, config, isBlinking) {
|
||||
ctx.fillStyle = '#1f2937';
|
||||
ctx.beginPath();
|
||||
ctx.arc(x, y, radius, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
|
||||
if (isBlinking) {
|
||||
// Closed eye (line)
|
||||
ctx.strokeStyle = config.color;
|
||||
ctx.lineWidth = 3;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(x - radius * 0.7, y);
|
||||
ctx.lineTo(x + radius * 0.7, y);
|
||||
ctx.stroke();
|
||||
} else {
|
||||
// Open eye with pupil
|
||||
const pupilRadius = radius * (config.eyeScale / 2);
|
||||
ctx.fillStyle = config.color;
|
||||
ctx.beginPath();
|
||||
ctx.arc(x + config.pupilPos.x, y + config.pupilPos.y, pupilRadius, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
|
||||
// Highlight reflection
|
||||
ctx.fillStyle = 'rgba(255, 255, 255, 0.6)';
|
||||
ctx.beginPath();
|
||||
ctx.arc(x + config.pupilPos.x + pupilRadius * 0.4, y + config.pupilPos.y - pupilRadius * 0.4, pupilRadius * 0.3, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
|
||||
// Draw mouth for talking
|
||||
function drawMouth(ctx, x, y, width, frame) {
|
||||
ctx.strokeStyle = '#f59e0b';
|
||||
ctx.lineWidth = 3;
|
||||
ctx.lineCap = 'round';
|
||||
|
||||
if (frame.shape === 'closed') {
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(x - width * 0.4, y);
|
||||
ctx.lineTo(x + width * 0.4, y);
|
||||
ctx.stroke();
|
||||
} else if (frame.shape === 'smile-open' || frame.shape === 'smile-closed') {
|
||||
ctx.beginPath();
|
||||
ctx.arc(x, y, width * 0.5, 0, Math.PI, false);
|
||||
ctx.stroke();
|
||||
} else if (frame.shape === 'oh') {
|
||||
ctx.fillStyle = '#f59e0b';
|
||||
ctx.beginPath();
|
||||
ctx.arc(x, y, width * 0.35 * frame.open, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
} else if (frame.shape === 'ah') {
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(x - width * 0.3, y - width * 0.2 * frame.open);
|
||||
ctx.lineTo(x + width * 0.3, y - width * 0.2 * frame.open);
|
||||
ctx.lineTo(x + width * 0.2, y + width * 0.3 * frame.open);
|
||||
ctx.lineTo(x - width * 0.2, y + width * 0.3 * frame.open);
|
||||
ctx.closePath();
|
||||
ctx.stroke();
|
||||
}
|
||||
}
|
||||
|
||||
// Status HUD overlay
|
||||
function StatusOverlay({ botState, visible }) {
|
||||
if (!visible) return null;
|
||||
|
||||
return (
|
||||
<div className="absolute inset-0 flex flex-col justify-between p-4 text-xs font-mono pointer-events-none">
|
||||
{/* Top-left: Battery & Status */}
|
||||
<div className="flex flex-col gap-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-amber-400">⚡</span>
|
||||
<span className="text-gray-300">{botState?.battery ?? '--'}%</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-cyan-400">●</span>
|
||||
<span className="text-gray-300">{botState?.state ?? 'IDLE'}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Bottom-left: Distance & Health */}
|
||||
<div className="flex flex-col gap-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-green-400">●</span>
|
||||
<span className="text-gray-300">{botState?.distance?.toFixed(1) ?? '--'}m</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className={botState?.health > 75 ? 'text-green-400' : botState?.health > 50 ? 'text-yellow-400' : 'text-red-400'}>◇</span>
|
||||
<span className="text-gray-300">{botState?.health ?? '--'}%</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Top-right: Speed */}
|
||||
<div className="absolute top-4 right-4 flex flex-col items-end gap-2">
|
||||
<div className="text-cyan-400">{botState?.speed?.toFixed(1) ?? '--'} m/s</div>
|
||||
<div className="text-gray-500 text-xs">[tap to hide]</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Main component
|
||||
export function SaltyFace({ subscribe }) {
|
||||
const [emotion, setEmotion] = useState(EMOTIONS.HAPPY);
|
||||
const [isTalking, setIsTalking] = useState(false);
|
||||
const [audioLevel, setAudioLevel] = useState(0);
|
||||
const [showOverlay, setShowOverlay] = useState(true);
|
||||
const [botState, setBotState] = useState({
|
||||
battery: 85,
|
||||
state: 'IDLE',
|
||||
distance: 0,
|
||||
speed: 0,
|
||||
health: 90,
|
||||
hasTarget: false,
|
||||
obstacles: 0,
|
||||
});
|
||||
|
||||
// Subscribe to robot state
|
||||
useEffect(() => {
|
||||
if (!subscribe) return;
|
||||
|
||||
const unsub1 = subscribe('/saltybot/state', 'std_msgs/String', (msg) => {
|
||||
try {
|
||||
const data = JSON.parse(msg.data);
|
||||
setBotState((prev) => ({ ...prev, state: data.state || 'IDLE' }));
|
||||
|
||||
// Update emotion based on state
|
||||
if (data.state === 'EMERGENCY') {
|
||||
setEmotion(EMOTIONS.EMERGENCY);
|
||||
} else if (data.state === 'TRACKING') {
|
||||
setEmotion(EMOTIONS.HAPPY);
|
||||
} else if (data.state === 'SEARCHING') {
|
||||
setEmotion(EMOTIONS.CONFUSED);
|
||||
} else if (data.state === 'IDLE') {
|
||||
setEmotion(EMOTIONS.HAPPY);
|
||||
}
|
||||
} catch (e) {}
|
||||
});
|
||||
|
||||
const unsub2 = subscribe('/saltybot/target_track', 'geometry_msgs/Pose', (msg) => {
|
||||
setBotState((prev) => ({ ...prev, hasTarget: msg ? true : false }));
|
||||
if (msg) setEmotion(EMOTIONS.EXCITED);
|
||||
});
|
||||
|
||||
const unsub3 = subscribe('/saltybot/obstacles', 'sensor_msgs/LaserScan', (msg) => {
|
||||
const obstacleCount = msg?.ranges?.filter((r) => r < 0.5).length ?? 0;
|
||||
setBotState((prev) => ({ ...prev, obstacles: obstacleCount }));
|
||||
if (obstacleCount > 0) setEmotion(EMOTIONS.ALERT);
|
||||
});
|
||||
|
||||
const unsub4 = subscribe('/social/speech/is_speaking', 'std_msgs/Bool', (msg) => {
|
||||
setIsTalking(msg.data ?? false);
|
||||
if (msg.data) setEmotion(EMOTIONS.TALKING);
|
||||
});
|
||||
|
||||
const unsub5 = subscribe('/social/speech/is_listening', 'std_msgs/Bool', (msg) => {
|
||||
if (msg.data) setEmotion(EMOTIONS.LISTENING);
|
||||
});
|
||||
|
||||
const unsub6 = subscribe('/saltybot/battery', 'std_msgs/Float32', (msg) => {
|
||||
setBotState((prev) => ({ ...prev, battery: Math.round(msg.data) }));
|
||||
});
|
||||
|
||||
const unsub7 = subscribe('/saltybot/audio_level', 'std_msgs/Float32', (msg) => {
|
||||
setAudioLevel(msg.data ?? 0);
|
||||
});
|
||||
|
||||
return () => {
|
||||
unsub1?.();
|
||||
unsub2?.();
|
||||
unsub3?.();
|
||||
unsub4?.();
|
||||
unsub5?.();
|
||||
unsub6?.();
|
||||
unsub7?.();
|
||||
};
|
||||
}, [subscribe]);
|
||||
|
||||
return (
|
||||
<div className="relative w-full h-screen bg-gray-950 overflow-hidden" onClick={() => setShowOverlay(!showOverlay)}>
|
||||
<FaceCanvas emotion={emotion} isTalking={isTalking} audioLevel={audioLevel} showOverlay={showOverlay} botState={botState} />
|
||||
<StatusOverlay botState={botState} visible={showOverlay} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user