feat: accessibility communication UI (Issue #371) #391
@ -294,6 +294,7 @@ export default function App() {
|
||||
{activeTab === 'personality' && <PersonalityTuner subscribe={subscribe} setParam={setParam} />}
|
||||
{activeTab === 'navigation' && <NavModeSelector subscribe={subscribe} publish={publishFn} />}
|
||||
{activeTab === 'audio' && <AudioMeter subscribe={subscribe} />}
|
||||
{activeTab === 'accessibility' && <AccessibilityComm subscribe={subscribe} publish={publishFn} />}
|
||||
|
||||
{activeTab === 'imu' && <PoseViewer subscribe={subscribe} />}
|
||||
{activeTab === 'battery' && <BatteryPanel subscribe={subscribe} />}
|
||||
|
||||
367
ui/social-bot/src/components/AccessibilityComm.jsx
Normal file
367
ui/social-bot/src/components/AccessibilityComm.jsx
Normal file
@ -0,0 +1,367 @@
|
||||
/**
|
||||
* AccessibilityComm.jsx — Deaf/accessibility communication interface
|
||||
*
|
||||
* P1 Issue #371: Speech-to-text display (large font), on-screen touch keyboard,
|
||||
* text-to-speech responses, and split-screen conversation mode.
|
||||
*
|
||||
* Subscribes:
|
||||
* /social/speech/transcript (SpeechTranscript) — user's transcribed speech
|
||||
* /social/conversation/response (ConversationResponse) — Salty's responses
|
||||
*
|
||||
* Features:
|
||||
* - Top panel: Large user message display (cyan)
|
||||
* - Bottom panel: Large Salty response display (teal)
|
||||
* - On-screen touch keyboard for manual text input
|
||||
* - Auto-scroll to latest messages
|
||||
* - Clear conversation history
|
||||
* - TTS button for Salty's responses
|
||||
* - Optimized for 7-inch touchscreen (MageDok)
|
||||
*/
|
||||
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
|
||||
const MAX_ENTRIES = 100;
|
||||
const MIN_FONT_SIZE = 24; // px
|
||||
|
||||
function formatTime(ts) {
|
||||
return new Date(ts).toLocaleTimeString([], {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
second: '2-digit',
|
||||
});
|
||||
}
|
||||
|
||||
// On-screen touch keyboard component
|
||||
function TouchKeyboard({ onInput, onSend, onClear, value }) {
|
||||
const rows = [
|
||||
['Q', 'W', 'E', 'R', 'T', 'Y', 'U', 'I', 'O', 'P'],
|
||||
['A', 'S', 'D', 'F', 'G', 'H', 'J', 'K', 'L'],
|
||||
['Z', 'X', 'C', 'V', 'B', 'N', 'M'],
|
||||
];
|
||||
|
||||
const handleKeyClick = (key) => {
|
||||
onInput(value + key);
|
||||
};
|
||||
|
||||
const handleBackspace = () => {
|
||||
onInput(value.slice(0, -1));
|
||||
};
|
||||
|
||||
const handleSpace = () => {
|
||||
onInput(value + ' ');
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-1.5 bg-gray-950 border border-gray-800 rounded-lg p-2">
|
||||
{/* Text input display */}
|
||||
<div className="bg-gray-900 border border-gray-700 rounded px-3 py-2 text-lg text-white min-h-12 word-break">
|
||||
{value || <span className="text-gray-600">Type your message…</span>}
|
||||
</div>
|
||||
|
||||
{/* Keyboard rows */}
|
||||
<div className="flex flex-col gap-1">
|
||||
{rows.map((row, ri) => (
|
||||
<div key={ri} className="flex gap-1 justify-center">
|
||||
{row.map((key) => (
|
||||
<button
|
||||
key={key}
|
||||
onClick={() => handleKeyClick(key)}
|
||||
className="bg-cyan-700 hover:bg-cyan-600 active:bg-cyan-500 text-white font-bold rounded text-sm flex-1 py-1.5 max-w-12 transition-colors"
|
||||
>
|
||||
{key}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
))}
|
||||
|
||||
{/* Bottom row: special keys */}
|
||||
<div className="flex gap-1 justify-center">
|
||||
<button
|
||||
onClick={handleBackspace}
|
||||
className="bg-red-700 hover:bg-red-600 active:bg-red-500 text-white font-bold rounded px-3 py-1.5 text-sm transition-colors flex-shrink-0"
|
||||
>
|
||||
← Back
|
||||
</button>
|
||||
<button
|
||||
onClick={handleSpace}
|
||||
className="bg-blue-700 hover:bg-blue-600 active:bg-blue-500 text-white font-bold rounded px-4 py-1.5 flex-1 transition-colors"
|
||||
>
|
||||
SPACE
|
||||
</button>
|
||||
<button
|
||||
onClick={onClear}
|
||||
className="bg-orange-700 hover:bg-orange-600 active:bg-orange-500 text-white font-bold rounded px-3 py-1.5 text-sm transition-colors flex-shrink-0"
|
||||
>
|
||||
Clear
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Send button */}
|
||||
<button
|
||||
onClick={onSend}
|
||||
disabled={!value.trim()}
|
||||
className="w-full bg-green-600 hover:bg-green-500 disabled:bg-gray-600 disabled:cursor-not-allowed active:bg-green-400 text-white font-bold rounded py-2 text-lg transition-colors"
|
||||
>
|
||||
SEND MESSAGE
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Large message bubble for user
|
||||
function UserMessageBubble({ entry }) {
|
||||
return (
|
||||
<div className="flex flex-col gap-1 mb-2">
|
||||
<div className="text-xs text-gray-500 px-2">
|
||||
{formatTime(entry.ts)} — {entry.speaker || 'You'} {entry.partial && <span className="text-amber-600">(transcribing…)</span>}
|
||||
</div>
|
||||
<div className="bg-cyan-950 border-2 border-cyan-600 rounded-lg px-4 py-3 text-cyan-100">
|
||||
<p style={{ fontSize: `${MIN_FONT_SIZE}px` }} className="leading-tight break-words whitespace-pre-wrap">
|
||||
{entry.text}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Large message bubble for Salty
|
||||
function SaltyMessageBubble({ entry, onSpeak }) {
|
||||
return (
|
||||
<div className="flex flex-col gap-1 mb-2">
|
||||
<div className="flex items-center justify-between px-2">
|
||||
<div className="text-xs text-gray-500">
|
||||
{formatTime(entry.ts)} — Salty {entry.partial && <span className="text-amber-600 ml-1">(typing…)</span>}
|
||||
</div>
|
||||
<button
|
||||
onClick={() => onSpeak(entry.text)}
|
||||
className="text-xs bg-teal-700 hover:bg-teal-600 active:bg-teal-500 text-white px-2 py-0.5 rounded transition-colors"
|
||||
>
|
||||
🔊 Speak
|
||||
</button>
|
||||
</div>
|
||||
<div className="bg-teal-950 border-2 border-teal-600 rounded-lg px-4 py-3 text-teal-100">
|
||||
<p style={{ fontSize: `${MIN_FONT_SIZE}px` }} className="leading-tight break-words whitespace-pre-wrap">
|
||||
{entry.text}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function AccessibilityComm({ subscribe, publish }) {
|
||||
const [userMessages, setUserMessages] = useState([]);
|
||||
const [saltyMessages, setSaltyMessages] = useState([]);
|
||||
const [keyboardInput, setKeyboardInput] = useState('');
|
||||
const [autoScroll, setAutoScroll] = useState(true);
|
||||
const userBottomRef = useRef(null);
|
||||
const saltyBottomRef = useRef(null);
|
||||
|
||||
// Subscribe to user speech transcripts
|
||||
useEffect(() => {
|
||||
const unsub = subscribe(
|
||||
'/social/speech/transcript',
|
||||
'saltybot_social_msgs/SpeechTranscript',
|
||||
(msg) => {
|
||||
setUserMessages((prev) => {
|
||||
const entry = {
|
||||
id: `h-${msg.header?.stamp?.sec ?? Date.now()}-${msg.turn_id ?? Math.random()}`,
|
||||
text: msg.text,
|
||||
speaker: msg.speaker_id,
|
||||
confidence: msg.confidence,
|
||||
partial: msg.is_partial,
|
||||
ts: Date.now(),
|
||||
};
|
||||
|
||||
// Replace partial with same turn
|
||||
if (msg.is_partial) {
|
||||
const idx = prev.findLastIndex((e) => e.partial && e.speaker === msg.speaker_id);
|
||||
if (idx !== -1) {
|
||||
const updated = [...prev];
|
||||
updated[idx] = entry;
|
||||
return updated;
|
||||
}
|
||||
} else {
|
||||
// Replace trailing partial for same speaker
|
||||
const idx = prev.findLastIndex((e) => e.partial && e.speaker === msg.speaker_id);
|
||||
if (idx !== -1) {
|
||||
const updated = [...prev];
|
||||
updated[idx] = { ...entry, id: prev[idx].id };
|
||||
return updated.slice(-MAX_ENTRIES);
|
||||
}
|
||||
}
|
||||
|
||||
return [...prev, entry].slice(-MAX_ENTRIES);
|
||||
});
|
||||
}
|
||||
);
|
||||
return unsub;
|
||||
}, [subscribe]);
|
||||
|
||||
// Subscribe to Salty responses
|
||||
useEffect(() => {
|
||||
const unsub = subscribe(
|
||||
'/social/conversation/response',
|
||||
'saltybot_social_msgs/ConversationResponse',
|
||||
(msg) => {
|
||||
setSaltyMessages((prev) => {
|
||||
const entry = {
|
||||
id: `b-${msg.turn_id ?? Math.random()}`,
|
||||
text: msg.text,
|
||||
speaker: 'Salty',
|
||||
partial: msg.is_partial,
|
||||
turnId: msg.turn_id,
|
||||
ts: Date.now(),
|
||||
};
|
||||
|
||||
// Replace streaming partial with same turn_id
|
||||
if (msg.turn_id != null) {
|
||||
const idx = prev.findLastIndex((e) => e.turnId === msg.turn_id);
|
||||
if (idx !== -1) {
|
||||
const updated = [...prev];
|
||||
updated[idx] = entry;
|
||||
return updated;
|
||||
}
|
||||
}
|
||||
|
||||
return [...prev, entry].slice(-MAX_ENTRIES);
|
||||
});
|
||||
}
|
||||
);
|
||||
return unsub;
|
||||
}, [subscribe]);
|
||||
|
||||
// Auto-scroll to latest messages
|
||||
useEffect(() => {
|
||||
if (autoScroll && userBottomRef.current) {
|
||||
userBottomRef.current.scrollIntoView({ behavior: 'smooth' });
|
||||
}
|
||||
}, [userMessages, autoScroll]);
|
||||
|
||||
useEffect(() => {
|
||||
if (autoScroll && saltyBottomRef.current) {
|
||||
saltyBottomRef.current.scrollIntoView({ behavior: 'smooth' });
|
||||
}
|
||||
}, [saltyMessages, autoScroll]);
|
||||
|
||||
// Send manual keyboard input
|
||||
const handleSendMessage = () => {
|
||||
if (!keyboardInput.trim()) return;
|
||||
|
||||
// Publish as a user text input message
|
||||
if (publish) {
|
||||
publish('/social/text_input', 'std_msgs/String', {
|
||||
data: keyboardInput,
|
||||
});
|
||||
}
|
||||
|
||||
// Clear input
|
||||
setKeyboardInput('');
|
||||
};
|
||||
|
||||
// Text-to-speech for Salty's message
|
||||
const handleSpeak = (text) => {
|
||||
if ('speechSynthesis' in window) {
|
||||
window.speechSynthesis.cancel(); // Cancel any ongoing speech
|
||||
const utterance = new SpeechSynthesisUtterance(text);
|
||||
utterance.rate = 0.9; // Slightly slower for clarity
|
||||
utterance.pitch = 1.0;
|
||||
utterance.volume = 1.0;
|
||||
window.speechSynthesis.speak(utterance);
|
||||
}
|
||||
};
|
||||
|
||||
const handleClear = () => {
|
||||
setUserMessages([]);
|
||||
setSaltyMessages([]);
|
||||
setKeyboardInput('');
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex flex-col h-full gap-4 p-4 bg-gray-900 rounded-lg" style={{ minHeight: '100vh' }}>
|
||||
{/* Title & Controls */}
|
||||
<div className="flex items-center justify-between bg-gray-950 border border-gray-800 rounded-lg p-3 shrink-0">
|
||||
<div className="text-sm font-bold text-cyan-400 tracking-widest">DEAF/ACCESSIBILITY MODE — Issue #371</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<label className="flex items-center gap-2 cursor-pointer text-xs text-gray-400">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={autoScroll}
|
||||
onChange={(e) => setAutoScroll(e.target.checked)}
|
||||
className="accent-cyan-500 w-3 h-3"
|
||||
/>
|
||||
Auto-scroll
|
||||
</label>
|
||||
<button
|
||||
onClick={handleClear}
|
||||
className="px-3 py-1 rounded bg-red-900 hover:bg-red-800 text-red-200 text-xs font-bold transition-colors"
|
||||
>
|
||||
🗑️ Clear All
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Split-screen conversation */}
|
||||
<div className="flex-1 gap-4 grid grid-cols-1 lg:grid-cols-2 overflow-hidden">
|
||||
{/* User Messages Panel */}
|
||||
<div className="flex flex-col overflow-hidden border border-cyan-900 rounded-lg bg-cyan-950 bg-opacity-30">
|
||||
<div className="px-3 py-2 bg-cyan-950 border-b border-cyan-900 shrink-0">
|
||||
<div className="text-xs font-bold text-cyan-400 tracking-widest">YOUR MESSAGES</div>
|
||||
<div className="text-xs text-cyan-700">{userMessages.length} messages</div>
|
||||
</div>
|
||||
<div className="flex-1 overflow-y-auto p-3 space-y-2">
|
||||
{userMessages.length === 0 ? (
|
||||
<div className="text-gray-600 text-center py-8 border border-dashed border-gray-800 rounded-lg">
|
||||
<p style={{ fontSize: '18px' }}>Waiting for speech…</p>
|
||||
<p className="text-xs text-gray-700 mt-2">Or use the keyboard below to type</p>
|
||||
</div>
|
||||
) : (
|
||||
userMessages.map((entry) => <UserMessageBubble key={entry.id} entry={entry} />)
|
||||
)}
|
||||
<div ref={userBottomRef} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Salty Response Panel */}
|
||||
<div className="flex flex-col overflow-hidden border border-teal-900 rounded-lg bg-teal-950 bg-opacity-30">
|
||||
<div className="px-3 py-2 bg-teal-950 border-b border-teal-900 shrink-0">
|
||||
<div className="text-xs font-bold text-teal-400 tracking-widest">SALTY'S RESPONSES</div>
|
||||
<div className="text-xs text-teal-700">{saltyMessages.length} responses</div>
|
||||
</div>
|
||||
<div className="flex-1 overflow-y-auto p-3 space-y-2">
|
||||
{saltyMessages.length === 0 ? (
|
||||
<div className="text-gray-600 text-center py-8 border border-dashed border-gray-800 rounded-lg">
|
||||
<p style={{ fontSize: '18px' }}>Waiting for Salty's response…</p>
|
||||
<p className="text-xs text-gray-700 mt-2">Salty will display her messages here</p>
|
||||
</div>
|
||||
) : (
|
||||
saltyMessages.map((entry) => (
|
||||
<SaltyMessageBubble
|
||||
key={entry.id}
|
||||
entry={entry}
|
||||
onSpeak={handleSpeak}
|
||||
/>
|
||||
))
|
||||
)}
|
||||
<div ref={saltyBottomRef} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* On-screen touch keyboard */}
|
||||
<div className="shrink-0 border border-gray-800 rounded-lg bg-gray-950">
|
||||
<TouchKeyboard
|
||||
value={keyboardInput}
|
||||
onInput={setKeyboardInput}
|
||||
onSend={handleSendMessage}
|
||||
onClear={() => setKeyboardInput('')}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Footer with instructions */}
|
||||
<div className="text-xs text-gray-600 border-t border-gray-800 pt-2 shrink-0">
|
||||
<p>💡 <strong>Tips:</strong> Speak clearly for speech-to-text, or use the keyboard below. Click 🔊 to hear Salty's responses aloud.</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user