feat: deaf/accessibility communication UI (Issue #371)
This commit is contained in:
parent
3c93a72d01
commit
9fc3e9894e
@ -294,6 +294,7 @@ export default function App() {
|
|||||||
{activeTab === 'personality' && <PersonalityTuner subscribe={subscribe} setParam={setParam} />}
|
{activeTab === 'personality' && <PersonalityTuner subscribe={subscribe} setParam={setParam} />}
|
||||||
{activeTab === 'navigation' && <NavModeSelector subscribe={subscribe} publish={publishFn} />}
|
{activeTab === 'navigation' && <NavModeSelector subscribe={subscribe} publish={publishFn} />}
|
||||||
{activeTab === 'audio' && <AudioMeter subscribe={subscribe} />}
|
{activeTab === 'audio' && <AudioMeter subscribe={subscribe} />}
|
||||||
|
{activeTab === 'accessibility' && <AccessibilityComm subscribe={subscribe} publish={publishFn} />}
|
||||||
|
|
||||||
{activeTab === 'imu' && <PoseViewer subscribe={subscribe} />}
|
{activeTab === 'imu' && <PoseViewer subscribe={subscribe} />}
|
||||||
{activeTab === 'battery' && <BatteryPanel subscribe={subscribe} />}
|
{activeTab === 'battery' && <BatteryPanel subscribe={subscribe} />}
|
||||||
|
|||||||
367
ui/social-bot/src/components/AccessibilityComm.jsx
Normal file
367
ui/social-bot/src/components/AccessibilityComm.jsx
Normal file
@ -0,0 +1,367 @@
|
|||||||
|
/**
|
||||||
|
* AccessibilityComm.jsx — Deaf/accessibility communication interface
|
||||||
|
*
|
||||||
|
* P1 Issue #371: Speech-to-text display (large font), on-screen touch keyboard,
|
||||||
|
* text-to-speech responses, and split-screen conversation mode.
|
||||||
|
*
|
||||||
|
* Subscribes:
|
||||||
|
* /social/speech/transcript (SpeechTranscript) — user's transcribed speech
|
||||||
|
* /social/conversation/response (ConversationResponse) — Salty's responses
|
||||||
|
*
|
||||||
|
* Features:
|
||||||
|
* - Top panel: Large user message display (cyan)
|
||||||
|
* - Bottom panel: Large Salty response display (teal)
|
||||||
|
* - On-screen touch keyboard for manual text input
|
||||||
|
* - Auto-scroll to latest messages
|
||||||
|
* - Clear conversation history
|
||||||
|
* - TTS button for Salty's responses
|
||||||
|
* - Optimized for 7-inch touchscreen (MageDok)
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { useEffect, useRef, useState } from 'react';
|
||||||
|
|
||||||
|
const MAX_ENTRIES = 100;
|
||||||
|
const MIN_FONT_SIZE = 24; // px
|
||||||
|
|
||||||
|
function formatTime(ts) {
|
||||||
|
return new Date(ts).toLocaleTimeString([], {
|
||||||
|
hour: '2-digit',
|
||||||
|
minute: '2-digit',
|
||||||
|
second: '2-digit',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// On-screen touch keyboard component
|
||||||
|
function TouchKeyboard({ onInput, onSend, onClear, value }) {
|
||||||
|
const rows = [
|
||||||
|
['Q', 'W', 'E', 'R', 'T', 'Y', 'U', 'I', 'O', 'P'],
|
||||||
|
['A', 'S', 'D', 'F', 'G', 'H', 'J', 'K', 'L'],
|
||||||
|
['Z', 'X', 'C', 'V', 'B', 'N', 'M'],
|
||||||
|
];
|
||||||
|
|
||||||
|
const handleKeyClick = (key) => {
|
||||||
|
onInput(value + key);
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleBackspace = () => {
|
||||||
|
onInput(value.slice(0, -1));
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleSpace = () => {
|
||||||
|
onInput(value + ' ');
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col gap-1.5 bg-gray-950 border border-gray-800 rounded-lg p-2">
|
||||||
|
{/* Text input display */}
|
||||||
|
<div className="bg-gray-900 border border-gray-700 rounded px-3 py-2 text-lg text-white min-h-12 word-break">
|
||||||
|
{value || <span className="text-gray-600">Type your message…</span>}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Keyboard rows */}
|
||||||
|
<div className="flex flex-col gap-1">
|
||||||
|
{rows.map((row, ri) => (
|
||||||
|
<div key={ri} className="flex gap-1 justify-center">
|
||||||
|
{row.map((key) => (
|
||||||
|
<button
|
||||||
|
key={key}
|
||||||
|
onClick={() => handleKeyClick(key)}
|
||||||
|
className="bg-cyan-700 hover:bg-cyan-600 active:bg-cyan-500 text-white font-bold rounded text-sm flex-1 py-1.5 max-w-12 transition-colors"
|
||||||
|
>
|
||||||
|
{key}
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
|
||||||
|
{/* Bottom row: special keys */}
|
||||||
|
<div className="flex gap-1 justify-center">
|
||||||
|
<button
|
||||||
|
onClick={handleBackspace}
|
||||||
|
className="bg-red-700 hover:bg-red-600 active:bg-red-500 text-white font-bold rounded px-3 py-1.5 text-sm transition-colors flex-shrink-0"
|
||||||
|
>
|
||||||
|
← Back
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={handleSpace}
|
||||||
|
className="bg-blue-700 hover:bg-blue-600 active:bg-blue-500 text-white font-bold rounded px-4 py-1.5 flex-1 transition-colors"
|
||||||
|
>
|
||||||
|
SPACE
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={onClear}
|
||||||
|
className="bg-orange-700 hover:bg-orange-600 active:bg-orange-500 text-white font-bold rounded px-3 py-1.5 text-sm transition-colors flex-shrink-0"
|
||||||
|
>
|
||||||
|
Clear
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Send button */}
|
||||||
|
<button
|
||||||
|
onClick={onSend}
|
||||||
|
disabled={!value.trim()}
|
||||||
|
className="w-full bg-green-600 hover:bg-green-500 disabled:bg-gray-600 disabled:cursor-not-allowed active:bg-green-400 text-white font-bold rounded py-2 text-lg transition-colors"
|
||||||
|
>
|
||||||
|
SEND MESSAGE
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Large message bubble for user
|
||||||
|
function UserMessageBubble({ entry }) {
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col gap-1 mb-2">
|
||||||
|
<div className="text-xs text-gray-500 px-2">
|
||||||
|
{formatTime(entry.ts)} — {entry.speaker || 'You'} {entry.partial && <span className="text-amber-600">(transcribing…)</span>}
|
||||||
|
</div>
|
||||||
|
<div className="bg-cyan-950 border-2 border-cyan-600 rounded-lg px-4 py-3 text-cyan-100">
|
||||||
|
<p style={{ fontSize: `${MIN_FONT_SIZE}px` }} className="leading-tight break-words whitespace-pre-wrap">
|
||||||
|
{entry.text}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Large message bubble for Salty
|
||||||
|
function SaltyMessageBubble({ entry, onSpeak }) {
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col gap-1 mb-2">
|
||||||
|
<div className="flex items-center justify-between px-2">
|
||||||
|
<div className="text-xs text-gray-500">
|
||||||
|
{formatTime(entry.ts)} — Salty {entry.partial && <span className="text-amber-600 ml-1">(typing…)</span>}
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
onClick={() => onSpeak(entry.text)}
|
||||||
|
className="text-xs bg-teal-700 hover:bg-teal-600 active:bg-teal-500 text-white px-2 py-0.5 rounded transition-colors"
|
||||||
|
>
|
||||||
|
🔊 Speak
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div className="bg-teal-950 border-2 border-teal-600 rounded-lg px-4 py-3 text-teal-100">
|
||||||
|
<p style={{ fontSize: `${MIN_FONT_SIZE}px` }} className="leading-tight break-words whitespace-pre-wrap">
|
||||||
|
{entry.text}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AccessibilityComm({ subscribe, publish }) {
|
||||||
|
const [userMessages, setUserMessages] = useState([]);
|
||||||
|
const [saltyMessages, setSaltyMessages] = useState([]);
|
||||||
|
const [keyboardInput, setKeyboardInput] = useState('');
|
||||||
|
const [autoScroll, setAutoScroll] = useState(true);
|
||||||
|
const userBottomRef = useRef(null);
|
||||||
|
const saltyBottomRef = useRef(null);
|
||||||
|
|
||||||
|
// Subscribe to user speech transcripts
|
||||||
|
useEffect(() => {
|
||||||
|
const unsub = subscribe(
|
||||||
|
'/social/speech/transcript',
|
||||||
|
'saltybot_social_msgs/SpeechTranscript',
|
||||||
|
(msg) => {
|
||||||
|
setUserMessages((prev) => {
|
||||||
|
const entry = {
|
||||||
|
id: `h-${msg.header?.stamp?.sec ?? Date.now()}-${msg.turn_id ?? Math.random()}`,
|
||||||
|
text: msg.text,
|
||||||
|
speaker: msg.speaker_id,
|
||||||
|
confidence: msg.confidence,
|
||||||
|
partial: msg.is_partial,
|
||||||
|
ts: Date.now(),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Replace partial with same turn
|
||||||
|
if (msg.is_partial) {
|
||||||
|
const idx = prev.findLastIndex((e) => e.partial && e.speaker === msg.speaker_id);
|
||||||
|
if (idx !== -1) {
|
||||||
|
const updated = [...prev];
|
||||||
|
updated[idx] = entry;
|
||||||
|
return updated;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Replace trailing partial for same speaker
|
||||||
|
const idx = prev.findLastIndex((e) => e.partial && e.speaker === msg.speaker_id);
|
||||||
|
if (idx !== -1) {
|
||||||
|
const updated = [...prev];
|
||||||
|
updated[idx] = { ...entry, id: prev[idx].id };
|
||||||
|
return updated.slice(-MAX_ENTRIES);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...prev, entry].slice(-MAX_ENTRIES);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
);
|
||||||
|
return unsub;
|
||||||
|
}, [subscribe]);
|
||||||
|
|
||||||
|
// Subscribe to Salty responses
|
||||||
|
useEffect(() => {
|
||||||
|
const unsub = subscribe(
|
||||||
|
'/social/conversation/response',
|
||||||
|
'saltybot_social_msgs/ConversationResponse',
|
||||||
|
(msg) => {
|
||||||
|
setSaltyMessages((prev) => {
|
||||||
|
const entry = {
|
||||||
|
id: `b-${msg.turn_id ?? Math.random()}`,
|
||||||
|
text: msg.text,
|
||||||
|
speaker: 'Salty',
|
||||||
|
partial: msg.is_partial,
|
||||||
|
turnId: msg.turn_id,
|
||||||
|
ts: Date.now(),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Replace streaming partial with same turn_id
|
||||||
|
if (msg.turn_id != null) {
|
||||||
|
const idx = prev.findLastIndex((e) => e.turnId === msg.turn_id);
|
||||||
|
if (idx !== -1) {
|
||||||
|
const updated = [...prev];
|
||||||
|
updated[idx] = entry;
|
||||||
|
return updated;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...prev, entry].slice(-MAX_ENTRIES);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
);
|
||||||
|
return unsub;
|
||||||
|
}, [subscribe]);
|
||||||
|
|
||||||
|
// Auto-scroll to latest messages
|
||||||
|
useEffect(() => {
|
||||||
|
if (autoScroll && userBottomRef.current) {
|
||||||
|
userBottomRef.current.scrollIntoView({ behavior: 'smooth' });
|
||||||
|
}
|
||||||
|
}, [userMessages, autoScroll]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (autoScroll && saltyBottomRef.current) {
|
||||||
|
saltyBottomRef.current.scrollIntoView({ behavior: 'smooth' });
|
||||||
|
}
|
||||||
|
}, [saltyMessages, autoScroll]);
|
||||||
|
|
||||||
|
// Send manual keyboard input
|
||||||
|
const handleSendMessage = () => {
|
||||||
|
if (!keyboardInput.trim()) return;
|
||||||
|
|
||||||
|
// Publish as a user text input message
|
||||||
|
if (publish) {
|
||||||
|
publish('/social/text_input', 'std_msgs/String', {
|
||||||
|
data: keyboardInput,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear input
|
||||||
|
setKeyboardInput('');
|
||||||
|
};
|
||||||
|
|
||||||
|
// Text-to-speech for Salty's message
|
||||||
|
const handleSpeak = (text) => {
|
||||||
|
if ('speechSynthesis' in window) {
|
||||||
|
window.speechSynthesis.cancel(); // Cancel any ongoing speech
|
||||||
|
const utterance = new SpeechSynthesisUtterance(text);
|
||||||
|
utterance.rate = 0.9; // Slightly slower for clarity
|
||||||
|
utterance.pitch = 1.0;
|
||||||
|
utterance.volume = 1.0;
|
||||||
|
window.speechSynthesis.speak(utterance);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleClear = () => {
|
||||||
|
setUserMessages([]);
|
||||||
|
setSaltyMessages([]);
|
||||||
|
setKeyboardInput('');
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col h-full gap-4 p-4 bg-gray-900 rounded-lg" style={{ minHeight: '100vh' }}>
|
||||||
|
{/* Title & Controls */}
|
||||||
|
<div className="flex items-center justify-between bg-gray-950 border border-gray-800 rounded-lg p-3 shrink-0">
|
||||||
|
<div className="text-sm font-bold text-cyan-400 tracking-widest">DEAF/ACCESSIBILITY MODE — Issue #371</div>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<label className="flex items-center gap-2 cursor-pointer text-xs text-gray-400">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={autoScroll}
|
||||||
|
onChange={(e) => setAutoScroll(e.target.checked)}
|
||||||
|
className="accent-cyan-500 w-3 h-3"
|
||||||
|
/>
|
||||||
|
Auto-scroll
|
||||||
|
</label>
|
||||||
|
<button
|
||||||
|
onClick={handleClear}
|
||||||
|
className="px-3 py-1 rounded bg-red-900 hover:bg-red-800 text-red-200 text-xs font-bold transition-colors"
|
||||||
|
>
|
||||||
|
🗑️ Clear All
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Split-screen conversation */}
|
||||||
|
<div className="flex-1 gap-4 grid grid-cols-1 lg:grid-cols-2 overflow-hidden">
|
||||||
|
{/* User Messages Panel */}
|
||||||
|
<div className="flex flex-col overflow-hidden border border-cyan-900 rounded-lg bg-cyan-950 bg-opacity-30">
|
||||||
|
<div className="px-3 py-2 bg-cyan-950 border-b border-cyan-900 shrink-0">
|
||||||
|
<div className="text-xs font-bold text-cyan-400 tracking-widest">YOUR MESSAGES</div>
|
||||||
|
<div className="text-xs text-cyan-700">{userMessages.length} messages</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex-1 overflow-y-auto p-3 space-y-2">
|
||||||
|
{userMessages.length === 0 ? (
|
||||||
|
<div className="text-gray-600 text-center py-8 border border-dashed border-gray-800 rounded-lg">
|
||||||
|
<p style={{ fontSize: '18px' }}>Waiting for speech…</p>
|
||||||
|
<p className="text-xs text-gray-700 mt-2">Or use the keyboard below to type</p>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
userMessages.map((entry) => <UserMessageBubble key={entry.id} entry={entry} />)
|
||||||
|
)}
|
||||||
|
<div ref={userBottomRef} />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Salty Response Panel */}
|
||||||
|
<div className="flex flex-col overflow-hidden border border-teal-900 rounded-lg bg-teal-950 bg-opacity-30">
|
||||||
|
<div className="px-3 py-2 bg-teal-950 border-b border-teal-900 shrink-0">
|
||||||
|
<div className="text-xs font-bold text-teal-400 tracking-widest">SALTY'S RESPONSES</div>
|
||||||
|
<div className="text-xs text-teal-700">{saltyMessages.length} responses</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex-1 overflow-y-auto p-3 space-y-2">
|
||||||
|
{saltyMessages.length === 0 ? (
|
||||||
|
<div className="text-gray-600 text-center py-8 border border-dashed border-gray-800 rounded-lg">
|
||||||
|
<p style={{ fontSize: '18px' }}>Waiting for Salty's response…</p>
|
||||||
|
<p className="text-xs text-gray-700 mt-2">Salty will display her messages here</p>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
saltyMessages.map((entry) => (
|
||||||
|
<SaltyMessageBubble
|
||||||
|
key={entry.id}
|
||||||
|
entry={entry}
|
||||||
|
onSpeak={handleSpeak}
|
||||||
|
/>
|
||||||
|
))
|
||||||
|
)}
|
||||||
|
<div ref={saltyBottomRef} />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* On-screen touch keyboard */}
|
||||||
|
<div className="shrink-0 border border-gray-800 rounded-lg bg-gray-950">
|
||||||
|
<TouchKeyboard
|
||||||
|
value={keyboardInput}
|
||||||
|
onInput={setKeyboardInput}
|
||||||
|
onSend={handleSendMessage}
|
||||||
|
onClear={() => setKeyboardInput('')}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Footer with instructions */}
|
||||||
|
<div className="text-xs text-gray-600 border-t border-gray-800 pt-2 shrink-0">
|
||||||
|
<p>💡 <strong>Tips:</strong> Speak clearly for speech-to-text, or use the keyboard below. Click 🔊 to hear Salty's responses aloud.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user