/**
* AccessibilityComm.jsx — Deaf/accessibility communication interface
*
* P1 Issue #371: Speech-to-text display (large font), on-screen touch keyboard,
* text-to-speech responses, and split-screen conversation mode.
*
* Subscribes:
* /social/speech/transcript (SpeechTranscript) — user's transcribed speech
* /social/conversation/response (ConversationResponse) — Salty's responses
*
* Features:
* - Top panel: Large user message display (cyan)
* - Bottom panel: Large Salty response display (teal)
* - On-screen touch keyboard for manual text input
* - Auto-scroll to latest messages
* - Clear conversation history
* - TTS button for Salty's responses
* - Optimized for 7-inch touchscreen (MageDok)
*/
import { useEffect, useRef, useState } from 'react';
const MAX_ENTRIES = 100;
const MIN_FONT_SIZE = 24; // px
function formatTime(ts) {
return new Date(ts).toLocaleTimeString([], {
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
});
}
// On-screen touch keyboard component
function TouchKeyboard({ onInput, onSend, onClear, value }) {
const rows = [
['Q', 'W', 'E', 'R', 'T', 'Y', 'U', 'I', 'O', 'P'],
['A', 'S', 'D', 'F', 'G', 'H', 'J', 'K', 'L'],
['Z', 'X', 'C', 'V', 'B', 'N', 'M'],
];
const handleKeyClick = (key) => {
onInput(value + key);
};
const handleBackspace = () => {
onInput(value.slice(0, -1));
};
const handleSpace = () => {
onInput(value + ' ');
};
return (
{/* Text input display */}
{value || Type your message…}
{/* Keyboard rows */}
{rows.map((row, ri) => (
{row.map((key) => (
))}
))}
{/* Bottom row: special keys */}
{/* Send button */}
);
}
// Large message bubble for user
function UserMessageBubble({ entry }) {
return (
{formatTime(entry.ts)} — {entry.speaker || 'You'} {entry.partial && (transcribing…)}
);
}
// Large message bubble for Salty
function SaltyMessageBubble({ entry, onSpeak }) {
return (
{formatTime(entry.ts)} — Salty {entry.partial && (typing…)}
);
}
export function AccessibilityComm({ subscribe, publish }) {
const [userMessages, setUserMessages] = useState([]);
const [saltyMessages, setSaltyMessages] = useState([]);
const [keyboardInput, setKeyboardInput] = useState('');
const [autoScroll, setAutoScroll] = useState(true);
const userBottomRef = useRef(null);
const saltyBottomRef = useRef(null);
// Subscribe to user speech transcripts
useEffect(() => {
const unsub = subscribe(
'/social/speech/transcript',
'saltybot_social_msgs/SpeechTranscript',
(msg) => {
setUserMessages((prev) => {
const entry = {
id: `h-${msg.header?.stamp?.sec ?? Date.now()}-${msg.turn_id ?? Math.random()}`,
text: msg.text,
speaker: msg.speaker_id,
confidence: msg.confidence,
partial: msg.is_partial,
ts: Date.now(),
};
// Replace partial with same turn
if (msg.is_partial) {
const idx = prev.findLastIndex((e) => e.partial && e.speaker === msg.speaker_id);
if (idx !== -1) {
const updated = [...prev];
updated[idx] = entry;
return updated;
}
} else {
// Replace trailing partial for same speaker
const idx = prev.findLastIndex((e) => e.partial && e.speaker === msg.speaker_id);
if (idx !== -1) {
const updated = [...prev];
updated[idx] = { ...entry, id: prev[idx].id };
return updated.slice(-MAX_ENTRIES);
}
}
return [...prev, entry].slice(-MAX_ENTRIES);
});
}
);
return unsub;
}, [subscribe]);
// Subscribe to Salty responses
useEffect(() => {
const unsub = subscribe(
'/social/conversation/response',
'saltybot_social_msgs/ConversationResponse',
(msg) => {
setSaltyMessages((prev) => {
const entry = {
id: `b-${msg.turn_id ?? Math.random()}`,
text: msg.text,
speaker: 'Salty',
partial: msg.is_partial,
turnId: msg.turn_id,
ts: Date.now(),
};
// Replace streaming partial with same turn_id
if (msg.turn_id != null) {
const idx = prev.findLastIndex((e) => e.turnId === msg.turn_id);
if (idx !== -1) {
const updated = [...prev];
updated[idx] = entry;
return updated;
}
}
return [...prev, entry].slice(-MAX_ENTRIES);
});
}
);
return unsub;
}, [subscribe]);
// Auto-scroll to latest messages
useEffect(() => {
if (autoScroll && userBottomRef.current) {
userBottomRef.current.scrollIntoView({ behavior: 'smooth' });
}
}, [userMessages, autoScroll]);
useEffect(() => {
if (autoScroll && saltyBottomRef.current) {
saltyBottomRef.current.scrollIntoView({ behavior: 'smooth' });
}
}, [saltyMessages, autoScroll]);
// Send manual keyboard input
const handleSendMessage = () => {
if (!keyboardInput.trim()) return;
// Publish as a user text input message
if (publish) {
publish('/social/text_input', 'std_msgs/String', {
data: keyboardInput,
});
}
// Clear input
setKeyboardInput('');
};
// Text-to-speech for Salty's message
const handleSpeak = (text) => {
if ('speechSynthesis' in window) {
window.speechSynthesis.cancel(); // Cancel any ongoing speech
const utterance = new SpeechSynthesisUtterance(text);
utterance.rate = 0.9; // Slightly slower for clarity
utterance.pitch = 1.0;
utterance.volume = 1.0;
window.speechSynthesis.speak(utterance);
}
};
const handleClear = () => {
setUserMessages([]);
setSaltyMessages([]);
setKeyboardInput('');
};
return (
{/* Title & Controls */}
{/* Split-screen conversation */}
{/* User Messages Panel */}
YOUR MESSAGES
{userMessages.length} messages
{userMessages.length === 0 ? (
Waiting for speech…
Or use the keyboard below to type
) : (
userMessages.map((entry) =>
)
)}
{/* Salty Response Panel */}
SALTY'S RESPONSES
{saltyMessages.length} responses
{saltyMessages.length === 0 ? (
Waiting for Salty's response…
Salty will display her messages here
) : (
saltyMessages.map((entry) => (
))
)}
{/* On-screen touch keyboard */}
setKeyboardInput('')}
/>
{/* Footer with instructions */}
💡 Tips: Speak clearly for speech-to-text, or use the keyboard below. Click 🔊 to hear Salty's responses aloud.
);
}