/** * AccessibilityComm.jsx — Deaf/accessibility communication interface * * P1 Issue #371: Speech-to-text display (large font), on-screen touch keyboard, * text-to-speech responses, and split-screen conversation mode. * * Subscribes: * /social/speech/transcript (SpeechTranscript) — user's transcribed speech * /social/conversation/response (ConversationResponse) — Salty's responses * * Features: * - Top panel: Large user message display (cyan) * - Bottom panel: Large Salty response display (teal) * - On-screen touch keyboard for manual text input * - Auto-scroll to latest messages * - Clear conversation history * - TTS button for Salty's responses * - Optimized for 7-inch touchscreen (MageDok) */ import { useEffect, useRef, useState } from 'react'; const MAX_ENTRIES = 100; const MIN_FONT_SIZE = 24; // px function formatTime(ts) { return new Date(ts).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit', }); } // On-screen touch keyboard component function TouchKeyboard({ onInput, onSend, onClear, value }) { const rows = [ ['Q', 'W', 'E', 'R', 'T', 'Y', 'U', 'I', 'O', 'P'], ['A', 'S', 'D', 'F', 'G', 'H', 'J', 'K', 'L'], ['Z', 'X', 'C', 'V', 'B', 'N', 'M'], ]; const handleKeyClick = (key) => { onInput(value + key); }; const handleBackspace = () => { onInput(value.slice(0, -1)); }; const handleSpace = () => { onInput(value + ' '); }; return (
{/* Text input display */}
{value || Type your message…}
{/* Keyboard rows */}
{rows.map((row, ri) => (
{row.map((key) => ( ))}
))} {/* Bottom row: special keys */}
{/* Send button */}
); } // Large message bubble for user function UserMessageBubble({ entry }) { return (
{formatTime(entry.ts)} — {entry.speaker || 'You'} {entry.partial && (transcribing…)}

{entry.text}

); } // Large message bubble for Salty function SaltyMessageBubble({ entry, onSpeak }) { return (
{formatTime(entry.ts)} — Salty {entry.partial && (typing…)}

{entry.text}

); } export function AccessibilityComm({ subscribe, publish }) { const [userMessages, setUserMessages] = useState([]); const [saltyMessages, setSaltyMessages] = useState([]); const [keyboardInput, setKeyboardInput] = useState(''); const [autoScroll, setAutoScroll] = useState(true); const userBottomRef = useRef(null); const saltyBottomRef = useRef(null); // Subscribe to user speech transcripts useEffect(() => { const unsub = subscribe( '/social/speech/transcript', 'saltybot_social_msgs/SpeechTranscript', (msg) => { setUserMessages((prev) => { const entry = { id: `h-${msg.header?.stamp?.sec ?? Date.now()}-${msg.turn_id ?? Math.random()}`, text: msg.text, speaker: msg.speaker_id, confidence: msg.confidence, partial: msg.is_partial, ts: Date.now(), }; // Replace partial with same turn if (msg.is_partial) { const idx = prev.findLastIndex((e) => e.partial && e.speaker === msg.speaker_id); if (idx !== -1) { const updated = [...prev]; updated[idx] = entry; return updated; } } else { // Replace trailing partial for same speaker const idx = prev.findLastIndex((e) => e.partial && e.speaker === msg.speaker_id); if (idx !== -1) { const updated = [...prev]; updated[idx] = { ...entry, id: prev[idx].id }; return updated.slice(-MAX_ENTRIES); } } return [...prev, entry].slice(-MAX_ENTRIES); }); } ); return unsub; }, [subscribe]); // Subscribe to Salty responses useEffect(() => { const unsub = subscribe( '/social/conversation/response', 'saltybot_social_msgs/ConversationResponse', (msg) => { setSaltyMessages((prev) => { const entry = { id: `b-${msg.turn_id ?? Math.random()}`, text: msg.text, speaker: 'Salty', partial: msg.is_partial, turnId: msg.turn_id, ts: Date.now(), }; // Replace streaming partial with same turn_id if (msg.turn_id != null) { const idx = prev.findLastIndex((e) => e.turnId === msg.turn_id); if (idx !== -1) { const updated = [...prev]; updated[idx] = entry; return updated; } } return [...prev, entry].slice(-MAX_ENTRIES); }); } ); return unsub; }, [subscribe]); // Auto-scroll to latest messages useEffect(() => { if (autoScroll && userBottomRef.current) { userBottomRef.current.scrollIntoView({ behavior: 'smooth' }); } }, [userMessages, autoScroll]); useEffect(() => { if (autoScroll && saltyBottomRef.current) { saltyBottomRef.current.scrollIntoView({ behavior: 'smooth' }); } }, [saltyMessages, autoScroll]); // Send manual keyboard input const handleSendMessage = () => { if (!keyboardInput.trim()) return; // Publish as a user text input message if (publish) { publish('/social/text_input', 'std_msgs/String', { data: keyboardInput, }); } // Clear input setKeyboardInput(''); }; // Text-to-speech for Salty's message const handleSpeak = (text) => { if ('speechSynthesis' in window) { window.speechSynthesis.cancel(); // Cancel any ongoing speech const utterance = new SpeechSynthesisUtterance(text); utterance.rate = 0.9; // Slightly slower for clarity utterance.pitch = 1.0; utterance.volume = 1.0; window.speechSynthesis.speak(utterance); } }; const handleClear = () => { setUserMessages([]); setSaltyMessages([]); setKeyboardInput(''); }; return (
{/* Title & Controls */}
DEAF/ACCESSIBILITY MODE — Issue #371
{/* Split-screen conversation */}
{/* User Messages Panel */}
YOUR MESSAGES
{userMessages.length} messages
{userMessages.length === 0 ? (

Waiting for speech…

Or use the keyboard below to type

) : ( userMessages.map((entry) => ) )}
{/* Salty Response Panel */}
SALTY'S RESPONSES
{saltyMessages.length} responses
{saltyMessages.length === 0 ? (

Waiting for Salty's response…

Salty will display her messages here

) : ( saltyMessages.map((entry) => ( )) )}
{/* On-screen touch keyboard */}
setKeyboardInput('')} />
{/* Footer with instructions */}

💡 Tips: Speak clearly for speech-to-text, or use the keyboard below. Click 🔊 to hear Salty's responses aloud.

); }