diff --git a/ui/social-bot/src/App.jsx b/ui/social-bot/src/App.jsx index 3151f60..a24dda1 100644 --- a/ui/social-bot/src/App.jsx +++ b/ui/social-bot/src/App.jsx @@ -294,6 +294,7 @@ export default function App() { {activeTab === 'personality' && } {activeTab === 'navigation' && } {activeTab === 'audio' && } + {activeTab === 'accessibility' && } {activeTab === 'imu' && } {activeTab === 'battery' && } diff --git a/ui/social-bot/src/components/AccessibilityComm.jsx b/ui/social-bot/src/components/AccessibilityComm.jsx new file mode 100644 index 0000000..9b59803 --- /dev/null +++ b/ui/social-bot/src/components/AccessibilityComm.jsx @@ -0,0 +1,367 @@ +/** + * AccessibilityComm.jsx — Deaf/accessibility communication interface + * + * P1 Issue #371: Speech-to-text display (large font), on-screen touch keyboard, + * text-to-speech responses, and split-screen conversation mode. + * + * Subscribes: + * /social/speech/transcript (SpeechTranscript) — user's transcribed speech + * /social/conversation/response (ConversationResponse) — Salty's responses + * + * Features: + * - Top panel: Large user message display (cyan) + * - Bottom panel: Large Salty response display (teal) + * - On-screen touch keyboard for manual text input + * - Auto-scroll to latest messages + * - Clear conversation history + * - TTS button for Salty's responses + * - Optimized for 7-inch touchscreen (MageDok) + */ + +import { useEffect, useRef, useState } from 'react'; + +const MAX_ENTRIES = 100; +const MIN_FONT_SIZE = 24; // px + +function formatTime(ts) { + return new Date(ts).toLocaleTimeString([], { + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); +} + +// On-screen touch keyboard component +function TouchKeyboard({ onInput, onSend, onClear, value }) { + const rows = [ + ['Q', 'W', 'E', 'R', 'T', 'Y', 'U', 'I', 'O', 'P'], + ['A', 'S', 'D', 'F', 'G', 'H', 'J', 'K', 'L'], + ['Z', 'X', 'C', 'V', 'B', 'N', 'M'], + ]; + + const handleKeyClick = (key) => { + onInput(value + key); + }; + + const handleBackspace = () => { + onInput(value.slice(0, -1)); + }; + + const handleSpace = () => { + onInput(value + ' '); + }; + + return ( +
+ {/* Text input display */} +
+ {value || Type your message…} +
+ + {/* Keyboard rows */} +
+ {rows.map((row, ri) => ( +
+ {row.map((key) => ( + + ))} +
+ ))} + + {/* Bottom row: special keys */} +
+ + + +
+
+ + {/* Send button */} + +
+ ); +} + +// Large message bubble for user +function UserMessageBubble({ entry }) { + return ( +
+
+ {formatTime(entry.ts)} — {entry.speaker || 'You'} {entry.partial && (transcribing…)} +
+
+

+ {entry.text} +

+
+
+ ); +} + +// Large message bubble for Salty +function SaltyMessageBubble({ entry, onSpeak }) { + return ( +
+
+
+ {formatTime(entry.ts)} — Salty {entry.partial && (typing…)} +
+ +
+
+

+ {entry.text} +

+
+
+ ); +} + +export function AccessibilityComm({ subscribe, publish }) { + const [userMessages, setUserMessages] = useState([]); + const [saltyMessages, setSaltyMessages] = useState([]); + const [keyboardInput, setKeyboardInput] = useState(''); + const [autoScroll, setAutoScroll] = useState(true); + const userBottomRef = useRef(null); + const saltyBottomRef = useRef(null); + + // Subscribe to user speech transcripts + useEffect(() => { + const unsub = subscribe( + '/social/speech/transcript', + 'saltybot_social_msgs/SpeechTranscript', + (msg) => { + setUserMessages((prev) => { + const entry = { + id: `h-${msg.header?.stamp?.sec ?? Date.now()}-${msg.turn_id ?? Math.random()}`, + text: msg.text, + speaker: msg.speaker_id, + confidence: msg.confidence, + partial: msg.is_partial, + ts: Date.now(), + }; + + // Replace partial with same turn + if (msg.is_partial) { + const idx = prev.findLastIndex((e) => e.partial && e.speaker === msg.speaker_id); + if (idx !== -1) { + const updated = [...prev]; + updated[idx] = entry; + return updated; + } + } else { + // Replace trailing partial for same speaker + const idx = prev.findLastIndex((e) => e.partial && e.speaker === msg.speaker_id); + if (idx !== -1) { + const updated = [...prev]; + updated[idx] = { ...entry, id: prev[idx].id }; + return updated.slice(-MAX_ENTRIES); + } + } + + return [...prev, entry].slice(-MAX_ENTRIES); + }); + } + ); + return unsub; + }, [subscribe]); + + // Subscribe to Salty responses + useEffect(() => { + const unsub = subscribe( + '/social/conversation/response', + 'saltybot_social_msgs/ConversationResponse', + (msg) => { + setSaltyMessages((prev) => { + const entry = { + id: `b-${msg.turn_id ?? Math.random()}`, + text: msg.text, + speaker: 'Salty', + partial: msg.is_partial, + turnId: msg.turn_id, + ts: Date.now(), + }; + + // Replace streaming partial with same turn_id + if (msg.turn_id != null) { + const idx = prev.findLastIndex((e) => e.turnId === msg.turn_id); + if (idx !== -1) { + const updated = [...prev]; + updated[idx] = entry; + return updated; + } + } + + return [...prev, entry].slice(-MAX_ENTRIES); + }); + } + ); + return unsub; + }, [subscribe]); + + // Auto-scroll to latest messages + useEffect(() => { + if (autoScroll && userBottomRef.current) { + userBottomRef.current.scrollIntoView({ behavior: 'smooth' }); + } + }, [userMessages, autoScroll]); + + useEffect(() => { + if (autoScroll && saltyBottomRef.current) { + saltyBottomRef.current.scrollIntoView({ behavior: 'smooth' }); + } + }, [saltyMessages, autoScroll]); + + // Send manual keyboard input + const handleSendMessage = () => { + if (!keyboardInput.trim()) return; + + // Publish as a user text input message + if (publish) { + publish('/social/text_input', 'std_msgs/String', { + data: keyboardInput, + }); + } + + // Clear input + setKeyboardInput(''); + }; + + // Text-to-speech for Salty's message + const handleSpeak = (text) => { + if ('speechSynthesis' in window) { + window.speechSynthesis.cancel(); // Cancel any ongoing speech + const utterance = new SpeechSynthesisUtterance(text); + utterance.rate = 0.9; // Slightly slower for clarity + utterance.pitch = 1.0; + utterance.volume = 1.0; + window.speechSynthesis.speak(utterance); + } + }; + + const handleClear = () => { + setUserMessages([]); + setSaltyMessages([]); + setKeyboardInput(''); + }; + + return ( +
+ {/* Title & Controls */} +
+
DEAF/ACCESSIBILITY MODE — Issue #371
+
+ + +
+
+ + {/* Split-screen conversation */} +
+ {/* User Messages Panel */} +
+
+
YOUR MESSAGES
+
{userMessages.length} messages
+
+
+ {userMessages.length === 0 ? ( +
+

Waiting for speech…

+

Or use the keyboard below to type

+
+ ) : ( + userMessages.map((entry) => ) + )} +
+
+
+ + {/* Salty Response Panel */} +
+
+
SALTY'S RESPONSES
+
{saltyMessages.length} responses
+
+
+ {saltyMessages.length === 0 ? ( +
+

Waiting for Salty's response…

+

Salty will display her messages here

+
+ ) : ( + saltyMessages.map((entry) => ( + + )) + )} +
+
+
+
+ + {/* On-screen touch keyboard */} +
+ setKeyboardInput('')} + /> +
+ + {/* Footer with instructions */} +
+

💡 Tips: Speak clearly for speech-to-text, or use the keyboard below. Click 🔊 to hear Salty's responses aloud.

+
+
+ ); +}