/** * SaltyFace.jsx — Animated facial expression UI for MageDok 7" display * * Features: * - 8 emotional states (happy, alert, confused, sleeping, excited, emergency, listening, talking) * - GPU-accelerated Canvas/SVG rendering (target 30fps on Orin Nano) * - ROS2 integration: /saltybot/state, /saltybot/target_track, /saltybot/obstacles * - Mouth animation synchronized with TTS audio * - HUD overlay: battery, speed, distance, sensor health * - Tap-to-toggle status overlay * - Inspired by Cozmo/Vector minimalist design */ import React, { useState, useEffect, useRef, useCallback } from 'react'; // Emotion states const EMOTIONS = { HAPPY: 'happy', // Default, normal operation ALERT: 'alert', // Obstacles detected CONFUSED: 'confused', // Target lost, searching SLEEPING: 'sleeping', // Prolonged inactivity EXCITED: 'excited', // Target reacquired EMERGENCY: 'emergency', // E-stop activated LISTENING: 'listening', // Microphone active TALKING: 'talking', // Text-to-speech output }; // Eye characteristics per emotion const EMOTION_CONFIG = { [EMOTIONS.HAPPY]: { eyeScale: 1.0, pupilPos: { x: 0, y: 0 }, blink: true, blinkRate: 3000, color: '#10b981', }, [EMOTIONS.ALERT]: { eyeScale: 1.3, pupilPos: { x: 0, y: -3 }, blink: false, blinkRate: 0, color: '#ef4444', }, [EMOTIONS.CONFUSED]: { eyeScale: 1.1, pupilPos: { x: 0, y: 0 }, blink: true, blinkRate: 1500, eyeWander: true, color: '#f59e0b', }, [EMOTIONS.SLEEPING]: { eyeScale: 0.3, pupilPos: { x: 0, y: 0 }, blink: false, isClosed: true, color: '#6b7280', }, [EMOTIONS.EXCITED]: { eyeScale: 1.2, pupilPos: { x: 0, y: 0 }, blink: true, blinkRate: 800, bounce: true, color: '#22c55e', }, [EMOTIONS.EMERGENCY]: { eyeScale: 1.4, pupilPos: { x: 0, y: -4 }, blink: false, color: '#dc2626', flash: true, }, [EMOTIONS.LISTENING]: { eyeScale: 1.0, pupilPos: { x: 0, y: -2 }, blink: true, blinkRate: 2000, color: '#0ea5e9', }, [EMOTIONS.TALKING]: { eyeScale: 1.0, pupilPos: { x: 0, y: 0 }, blink: true, blinkRate: 2500, color: '#06b6d4', }, }; // Mouth states for talking const MOUTH_FRAMES = [ { open: 0.0, shape: 'closed' }, // Closed { open: 0.3, shape: 'smile-closed' }, // Slight smile { open: 0.5, shape: 'smile-open' }, // Smile open { open: 0.7, shape: 'oh' }, // "Oh" sound { open: 0.9, shape: 'ah' }, // "Ah" sound { open: 0.7, shape: 'ee' }, // "Ee" sound ]; // Canvas-based face renderer function FaceCanvas({ emotion, isTalking, audioLevel, showOverlay, botState }) { const canvasRef = useRef(null); const animationRef = useRef(null); const [eyeWanderOffset, setEyeWanderOffset] = useState({ x: 0, y: 0 }); const [mouthFrame, setMouthFrame] = useState(0); const [isBlinking, setIsBlinking] = useState(false); const talkingCounterRef = useRef(0); // Main animation loop useEffect(() => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext('2d', { alpha: true }); const W = canvas.width; const H = canvas.height; let frameCount = 0; const config = EMOTION_CONFIG[emotion] || EMOTION_CONFIG[EMOTIONS.HAPPY]; const drawFace = () => { // Clear canvas ctx.fillStyle = 'rgba(5, 5, 16, 0.95)'; ctx.fillRect(0, 0, W, H); const centerX = W / 2; const centerY = H / 2.2; const eyeRadius = 40; const eyeSpacing = 80; // Eye wandering animation (for confused state) let eyeOffX = 0, eyeOffY = 0; if (config.eyeWander) { eyeOffX = Math.sin(frameCount * 0.02) * 8; eyeOffY = Math.cos(frameCount * 0.015) * 8; } // Bounce animation (for excited state) let bounceOffset = 0; if (config.bounce) { bounceOffset = Math.sin(frameCount * 0.08) * 6; } // Draw eyes const eyeY = centerY + bounceOffset; drawEye(ctx, centerX - eyeSpacing, eyeY + eyeOffY, eyeRadius, config, isBlinking); drawEye(ctx, centerX + eyeSpacing, eyeY + eyeOffY, eyeRadius, config, isBlinking); // Draw mouth (if talking) if (isTalking && !config.isClosed) { drawMouth(ctx, centerX, centerY + 80, 50, MOUTH_FRAMES[mouthFrame]); } // Flash animation (emergency state) if (config.flash && Math.sin(frameCount * 0.1) > 0.7) { ctx.fillStyle = 'rgba(220, 38, 38, 0.3)'; ctx.fillRect(0, 0, W, H); } frameCount++; }; const animationLoop = () => { drawFace(); animationRef.current = requestAnimationFrame(animationLoop); }; animationLoop(); return () => { if (animationRef.current) { cancelAnimationFrame(animationRef.current); } }; }, [emotion, isTalking, isBlinking, mouthFrame]); // Blinking logic useEffect(() => { const config = EMOTION_CONFIG[emotion] || EMOTION_CONFIG[EMOTIONS.HAPPY]; if (!config.blink || config.isClosed) return; const blinkInterval = setInterval(() => { setIsBlinking(true); setTimeout(() => setIsBlinking(false), 150); }, config.blinkRate); return () => clearInterval(blinkInterval); }, [emotion]); // Mouth animation for talking useEffect(() => { if (!isTalking) { setMouthFrame(0); return; } let frameIndex = 0; const mouthInterval = setInterval(() => { frameIndex = (frameIndex + 1) % MOUTH_FRAMES.length; setMouthFrame(frameIndex); }, 100); // ~10fps mouth animation return () => clearInterval(mouthInterval); }, [isTalking]); return ( ); } // Draw individual eye function drawEye(ctx, x, y, radius, config, isBlinking) { ctx.fillStyle = '#1f2937'; ctx.beginPath(); ctx.arc(x, y, radius, 0, Math.PI * 2); ctx.fill(); if (isBlinking) { // Closed eye (line) ctx.strokeStyle = config.color; ctx.lineWidth = 3; ctx.beginPath(); ctx.moveTo(x - radius * 0.7, y); ctx.lineTo(x + radius * 0.7, y); ctx.stroke(); } else { // Open eye with pupil const pupilRadius = radius * (config.eyeScale / 2); ctx.fillStyle = config.color; ctx.beginPath(); ctx.arc(x + config.pupilPos.x, y + config.pupilPos.y, pupilRadius, 0, Math.PI * 2); ctx.fill(); // Highlight reflection ctx.fillStyle = 'rgba(255, 255, 255, 0.6)'; ctx.beginPath(); ctx.arc(x + config.pupilPos.x + pupilRadius * 0.4, y + config.pupilPos.y - pupilRadius * 0.4, pupilRadius * 0.3, 0, Math.PI * 2); ctx.fill(); } } // Draw mouth for talking function drawMouth(ctx, x, y, width, frame) { ctx.strokeStyle = '#f59e0b'; ctx.lineWidth = 3; ctx.lineCap = 'round'; if (frame.shape === 'closed') { ctx.beginPath(); ctx.moveTo(x - width * 0.4, y); ctx.lineTo(x + width * 0.4, y); ctx.stroke(); } else if (frame.shape === 'smile-open' || frame.shape === 'smile-closed') { ctx.beginPath(); ctx.arc(x, y, width * 0.5, 0, Math.PI, false); ctx.stroke(); } else if (frame.shape === 'oh') { ctx.fillStyle = '#f59e0b'; ctx.beginPath(); ctx.arc(x, y, width * 0.35 * frame.open, 0, Math.PI * 2); ctx.fill(); } else if (frame.shape === 'ah') { ctx.beginPath(); ctx.moveTo(x - width * 0.3, y - width * 0.2 * frame.open); ctx.lineTo(x + width * 0.3, y - width * 0.2 * frame.open); ctx.lineTo(x + width * 0.2, y + width * 0.3 * frame.open); ctx.lineTo(x - width * 0.2, y + width * 0.3 * frame.open); ctx.closePath(); ctx.stroke(); } } // Status HUD overlay function StatusOverlay({ botState, visible }) { if (!visible) return null; return (
{/* Top-left: Battery & Status */}
{botState?.battery ?? '--'}%
{botState?.state ?? 'IDLE'}
{/* Bottom-left: Distance & Health */}
{botState?.distance?.toFixed(1) ?? '--'}m
75 ? 'text-green-400' : botState?.health > 50 ? 'text-yellow-400' : 'text-red-400'}>◇ {botState?.health ?? '--'}%
{/* Top-right: Speed */}
{botState?.speed?.toFixed(1) ?? '--'} m/s
[tap to hide]
); } // Main component export function SaltyFace({ subscribe }) { const [emotion, setEmotion] = useState(EMOTIONS.HAPPY); const [isTalking, setIsTalking] = useState(false); const [audioLevel, setAudioLevel] = useState(0); const [showOverlay, setShowOverlay] = useState(true); const [botState, setBotState] = useState({ battery: 85, state: 'IDLE', distance: 0, speed: 0, health: 90, hasTarget: false, obstacles: 0, }); // Subscribe to robot state useEffect(() => { if (!subscribe) return; const unsub1 = subscribe('/saltybot/state', 'std_msgs/String', (msg) => { try { const data = JSON.parse(msg.data); setBotState((prev) => ({ ...prev, state: data.state || 'IDLE' })); // Update emotion based on state if (data.state === 'EMERGENCY') { setEmotion(EMOTIONS.EMERGENCY); } else if (data.state === 'TRACKING') { setEmotion(EMOTIONS.HAPPY); } else if (data.state === 'SEARCHING') { setEmotion(EMOTIONS.CONFUSED); } else if (data.state === 'IDLE') { setEmotion(EMOTIONS.HAPPY); } } catch (e) {} }); const unsub2 = subscribe('/saltybot/target_track', 'geometry_msgs/Pose', (msg) => { setBotState((prev) => ({ ...prev, hasTarget: msg ? true : false })); if (msg) setEmotion(EMOTIONS.EXCITED); }); const unsub3 = subscribe('/saltybot/obstacles', 'sensor_msgs/LaserScan', (msg) => { const obstacleCount = msg?.ranges?.filter((r) => r < 0.5).length ?? 0; setBotState((prev) => ({ ...prev, obstacles: obstacleCount })); if (obstacleCount > 0) setEmotion(EMOTIONS.ALERT); }); const unsub4 = subscribe('/social/speech/is_speaking', 'std_msgs/Bool', (msg) => { setIsTalking(msg.data ?? false); if (msg.data) setEmotion(EMOTIONS.TALKING); }); const unsub5 = subscribe('/social/speech/is_listening', 'std_msgs/Bool', (msg) => { if (msg.data) setEmotion(EMOTIONS.LISTENING); }); const unsub6 = subscribe('/saltybot/battery', 'std_msgs/Float32', (msg) => { setBotState((prev) => ({ ...prev, battery: Math.round(msg.data) })); }); const unsub7 = subscribe('/saltybot/audio_level', 'std_msgs/Float32', (msg) => { setAudioLevel(msg.data ?? 0); }); return () => { unsub1?.(); unsub2?.(); unsub3?.(); unsub4?.(); unsub5?.(); unsub6?.(); unsub7?.(); }; }, [subscribe]); return (
setShowOverlay(!showOverlay)}>
); }