368 lines
13 KiB
JavaScript
368 lines
13 KiB
JavaScript
/**
|
||
* AccessibilityComm.jsx — Deaf/accessibility communication interface
|
||
*
|
||
* P1 Issue #371: Speech-to-text display (large font), on-screen touch keyboard,
|
||
* text-to-speech responses, and split-screen conversation mode.
|
||
*
|
||
* Subscribes:
|
||
* /social/speech/transcript (SpeechTranscript) — user's transcribed speech
|
||
* /social/conversation/response (ConversationResponse) — Salty's responses
|
||
*
|
||
* Features:
|
||
* - Top panel: Large user message display (cyan)
|
||
* - Bottom panel: Large Salty response display (teal)
|
||
* - On-screen touch keyboard for manual text input
|
||
* - Auto-scroll to latest messages
|
||
* - Clear conversation history
|
||
* - TTS button for Salty's responses
|
||
* - Optimized for 7-inch touchscreen (MageDok)
|
||
*/
|
||
|
||
import { useEffect, useRef, useState } from 'react';
|
||
|
||
const MAX_ENTRIES = 100;
|
||
const MIN_FONT_SIZE = 24; // px
|
||
|
||
function formatTime(ts) {
|
||
return new Date(ts).toLocaleTimeString([], {
|
||
hour: '2-digit',
|
||
minute: '2-digit',
|
||
second: '2-digit',
|
||
});
|
||
}
|
||
|
||
// On-screen touch keyboard component
|
||
function TouchKeyboard({ onInput, onSend, onClear, value }) {
|
||
const rows = [
|
||
['Q', 'W', 'E', 'R', 'T', 'Y', 'U', 'I', 'O', 'P'],
|
||
['A', 'S', 'D', 'F', 'G', 'H', 'J', 'K', 'L'],
|
||
['Z', 'X', 'C', 'V', 'B', 'N', 'M'],
|
||
];
|
||
|
||
const handleKeyClick = (key) => {
|
||
onInput(value + key);
|
||
};
|
||
|
||
const handleBackspace = () => {
|
||
onInput(value.slice(0, -1));
|
||
};
|
||
|
||
const handleSpace = () => {
|
||
onInput(value + ' ');
|
||
};
|
||
|
||
return (
|
||
<div className="flex flex-col gap-1.5 bg-gray-950 border border-gray-800 rounded-lg p-2">
|
||
{/* Text input display */}
|
||
<div className="bg-gray-900 border border-gray-700 rounded px-3 py-2 text-lg text-white min-h-12 word-break">
|
||
{value || <span className="text-gray-600">Type your message…</span>}
|
||
</div>
|
||
|
||
{/* Keyboard rows */}
|
||
<div className="flex flex-col gap-1">
|
||
{rows.map((row, ri) => (
|
||
<div key={ri} className="flex gap-1 justify-center">
|
||
{row.map((key) => (
|
||
<button
|
||
key={key}
|
||
onClick={() => handleKeyClick(key)}
|
||
className="bg-cyan-700 hover:bg-cyan-600 active:bg-cyan-500 text-white font-bold rounded text-sm flex-1 py-1.5 max-w-12 transition-colors"
|
||
>
|
||
{key}
|
||
</button>
|
||
))}
|
||
</div>
|
||
))}
|
||
|
||
{/* Bottom row: special keys */}
|
||
<div className="flex gap-1 justify-center">
|
||
<button
|
||
onClick={handleBackspace}
|
||
className="bg-red-700 hover:bg-red-600 active:bg-red-500 text-white font-bold rounded px-3 py-1.5 text-sm transition-colors flex-shrink-0"
|
||
>
|
||
← Back
|
||
</button>
|
||
<button
|
||
onClick={handleSpace}
|
||
className="bg-blue-700 hover:bg-blue-600 active:bg-blue-500 text-white font-bold rounded px-4 py-1.5 flex-1 transition-colors"
|
||
>
|
||
SPACE
|
||
</button>
|
||
<button
|
||
onClick={onClear}
|
||
className="bg-orange-700 hover:bg-orange-600 active:bg-orange-500 text-white font-bold rounded px-3 py-1.5 text-sm transition-colors flex-shrink-0"
|
||
>
|
||
Clear
|
||
</button>
|
||
</div>
|
||
</div>
|
||
|
||
{/* Send button */}
|
||
<button
|
||
onClick={onSend}
|
||
disabled={!value.trim()}
|
||
className="w-full bg-green-600 hover:bg-green-500 disabled:bg-gray-600 disabled:cursor-not-allowed active:bg-green-400 text-white font-bold rounded py-2 text-lg transition-colors"
|
||
>
|
||
SEND MESSAGE
|
||
</button>
|
||
</div>
|
||
);
|
||
}
|
||
|
||
// Large message bubble for user
|
||
function UserMessageBubble({ entry }) {
|
||
return (
|
||
<div className="flex flex-col gap-1 mb-2">
|
||
<div className="text-xs text-gray-500 px-2">
|
||
{formatTime(entry.ts)} — {entry.speaker || 'You'} {entry.partial && <span className="text-amber-600">(transcribing…)</span>}
|
||
</div>
|
||
<div className="bg-cyan-950 border-2 border-cyan-600 rounded-lg px-4 py-3 text-cyan-100">
|
||
<p style={{ fontSize: `${MIN_FONT_SIZE}px` }} className="leading-tight break-words whitespace-pre-wrap">
|
||
{entry.text}
|
||
</p>
|
||
</div>
|
||
</div>
|
||
);
|
||
}
|
||
|
||
// Large message bubble for Salty
|
||
function SaltyMessageBubble({ entry, onSpeak }) {
|
||
return (
|
||
<div className="flex flex-col gap-1 mb-2">
|
||
<div className="flex items-center justify-between px-2">
|
||
<div className="text-xs text-gray-500">
|
||
{formatTime(entry.ts)} — Salty {entry.partial && <span className="text-amber-600 ml-1">(typing…)</span>}
|
||
</div>
|
||
<button
|
||
onClick={() => onSpeak(entry.text)}
|
||
className="text-xs bg-teal-700 hover:bg-teal-600 active:bg-teal-500 text-white px-2 py-0.5 rounded transition-colors"
|
||
>
|
||
🔊 Speak
|
||
</button>
|
||
</div>
|
||
<div className="bg-teal-950 border-2 border-teal-600 rounded-lg px-4 py-3 text-teal-100">
|
||
<p style={{ fontSize: `${MIN_FONT_SIZE}px` }} className="leading-tight break-words whitespace-pre-wrap">
|
||
{entry.text}
|
||
</p>
|
||
</div>
|
||
</div>
|
||
);
|
||
}
|
||
|
||
export function AccessibilityComm({ subscribe, publish }) {
|
||
const [userMessages, setUserMessages] = useState([]);
|
||
const [saltyMessages, setSaltyMessages] = useState([]);
|
||
const [keyboardInput, setKeyboardInput] = useState('');
|
||
const [autoScroll, setAutoScroll] = useState(true);
|
||
const userBottomRef = useRef(null);
|
||
const saltyBottomRef = useRef(null);
|
||
|
||
// Subscribe to user speech transcripts
|
||
useEffect(() => {
|
||
const unsub = subscribe(
|
||
'/social/speech/transcript',
|
||
'saltybot_social_msgs/SpeechTranscript',
|
||
(msg) => {
|
||
setUserMessages((prev) => {
|
||
const entry = {
|
||
id: `h-${msg.header?.stamp?.sec ?? Date.now()}-${msg.turn_id ?? Math.random()}`,
|
||
text: msg.text,
|
||
speaker: msg.speaker_id,
|
||
confidence: msg.confidence,
|
||
partial: msg.is_partial,
|
||
ts: Date.now(),
|
||
};
|
||
|
||
// Replace partial with same turn
|
||
if (msg.is_partial) {
|
||
const idx = prev.findLastIndex((e) => e.partial && e.speaker === msg.speaker_id);
|
||
if (idx !== -1) {
|
||
const updated = [...prev];
|
||
updated[idx] = entry;
|
||
return updated;
|
||
}
|
||
} else {
|
||
// Replace trailing partial for same speaker
|
||
const idx = prev.findLastIndex((e) => e.partial && e.speaker === msg.speaker_id);
|
||
if (idx !== -1) {
|
||
const updated = [...prev];
|
||
updated[idx] = { ...entry, id: prev[idx].id };
|
||
return updated.slice(-MAX_ENTRIES);
|
||
}
|
||
}
|
||
|
||
return [...prev, entry].slice(-MAX_ENTRIES);
|
||
});
|
||
}
|
||
);
|
||
return unsub;
|
||
}, [subscribe]);
|
||
|
||
// Subscribe to Salty responses
|
||
useEffect(() => {
|
||
const unsub = subscribe(
|
||
'/social/conversation/response',
|
||
'saltybot_social_msgs/ConversationResponse',
|
||
(msg) => {
|
||
setSaltyMessages((prev) => {
|
||
const entry = {
|
||
id: `b-${msg.turn_id ?? Math.random()}`,
|
||
text: msg.text,
|
||
speaker: 'Salty',
|
||
partial: msg.is_partial,
|
||
turnId: msg.turn_id,
|
||
ts: Date.now(),
|
||
};
|
||
|
||
// Replace streaming partial with same turn_id
|
||
if (msg.turn_id != null) {
|
||
const idx = prev.findLastIndex((e) => e.turnId === msg.turn_id);
|
||
if (idx !== -1) {
|
||
const updated = [...prev];
|
||
updated[idx] = entry;
|
||
return updated;
|
||
}
|
||
}
|
||
|
||
return [...prev, entry].slice(-MAX_ENTRIES);
|
||
});
|
||
}
|
||
);
|
||
return unsub;
|
||
}, [subscribe]);
|
||
|
||
// Auto-scroll to latest messages
|
||
useEffect(() => {
|
||
if (autoScroll && userBottomRef.current) {
|
||
userBottomRef.current.scrollIntoView({ behavior: 'smooth' });
|
||
}
|
||
}, [userMessages, autoScroll]);
|
||
|
||
useEffect(() => {
|
||
if (autoScroll && saltyBottomRef.current) {
|
||
saltyBottomRef.current.scrollIntoView({ behavior: 'smooth' });
|
||
}
|
||
}, [saltyMessages, autoScroll]);
|
||
|
||
// Send manual keyboard input
|
||
const handleSendMessage = () => {
|
||
if (!keyboardInput.trim()) return;
|
||
|
||
// Publish as a user text input message
|
||
if (publish) {
|
||
publish('/social/text_input', 'std_msgs/String', {
|
||
data: keyboardInput,
|
||
});
|
||
}
|
||
|
||
// Clear input
|
||
setKeyboardInput('');
|
||
};
|
||
|
||
// Text-to-speech for Salty's message
|
||
const handleSpeak = (text) => {
|
||
if ('speechSynthesis' in window) {
|
||
window.speechSynthesis.cancel(); // Cancel any ongoing speech
|
||
const utterance = new SpeechSynthesisUtterance(text);
|
||
utterance.rate = 0.9; // Slightly slower for clarity
|
||
utterance.pitch = 1.0;
|
||
utterance.volume = 1.0;
|
||
window.speechSynthesis.speak(utterance);
|
||
}
|
||
};
|
||
|
||
const handleClear = () => {
|
||
setUserMessages([]);
|
||
setSaltyMessages([]);
|
||
setKeyboardInput('');
|
||
};
|
||
|
||
return (
|
||
<div className="flex flex-col h-full gap-4 p-4 bg-gray-900 rounded-lg" style={{ minHeight: '100vh' }}>
|
||
{/* Title & Controls */}
|
||
<div className="flex items-center justify-between bg-gray-950 border border-gray-800 rounded-lg p-3 shrink-0">
|
||
<div className="text-sm font-bold text-cyan-400 tracking-widest">DEAF/ACCESSIBILITY MODE — Issue #371</div>
|
||
<div className="flex items-center gap-2">
|
||
<label className="flex items-center gap-2 cursor-pointer text-xs text-gray-400">
|
||
<input
|
||
type="checkbox"
|
||
checked={autoScroll}
|
||
onChange={(e) => setAutoScroll(e.target.checked)}
|
||
className="accent-cyan-500 w-3 h-3"
|
||
/>
|
||
Auto-scroll
|
||
</label>
|
||
<button
|
||
onClick={handleClear}
|
||
className="px-3 py-1 rounded bg-red-900 hover:bg-red-800 text-red-200 text-xs font-bold transition-colors"
|
||
>
|
||
🗑️ Clear All
|
||
</button>
|
||
</div>
|
||
</div>
|
||
|
||
{/* Split-screen conversation */}
|
||
<div className="flex-1 gap-4 grid grid-cols-1 lg:grid-cols-2 overflow-hidden">
|
||
{/* User Messages Panel */}
|
||
<div className="flex flex-col overflow-hidden border border-cyan-900 rounded-lg bg-cyan-950 bg-opacity-30">
|
||
<div className="px-3 py-2 bg-cyan-950 border-b border-cyan-900 shrink-0">
|
||
<div className="text-xs font-bold text-cyan-400 tracking-widest">YOUR MESSAGES</div>
|
||
<div className="text-xs text-cyan-700">{userMessages.length} messages</div>
|
||
</div>
|
||
<div className="flex-1 overflow-y-auto p-3 space-y-2">
|
||
{userMessages.length === 0 ? (
|
||
<div className="text-gray-600 text-center py-8 border border-dashed border-gray-800 rounded-lg">
|
||
<p style={{ fontSize: '18px' }}>Waiting for speech…</p>
|
||
<p className="text-xs text-gray-700 mt-2">Or use the keyboard below to type</p>
|
||
</div>
|
||
) : (
|
||
userMessages.map((entry) => <UserMessageBubble key={entry.id} entry={entry} />)
|
||
)}
|
||
<div ref={userBottomRef} />
|
||
</div>
|
||
</div>
|
||
|
||
{/* Salty Response Panel */}
|
||
<div className="flex flex-col overflow-hidden border border-teal-900 rounded-lg bg-teal-950 bg-opacity-30">
|
||
<div className="px-3 py-2 bg-teal-950 border-b border-teal-900 shrink-0">
|
||
<div className="text-xs font-bold text-teal-400 tracking-widest">SALTY'S RESPONSES</div>
|
||
<div className="text-xs text-teal-700">{saltyMessages.length} responses</div>
|
||
</div>
|
||
<div className="flex-1 overflow-y-auto p-3 space-y-2">
|
||
{saltyMessages.length === 0 ? (
|
||
<div className="text-gray-600 text-center py-8 border border-dashed border-gray-800 rounded-lg">
|
||
<p style={{ fontSize: '18px' }}>Waiting for Salty's response…</p>
|
||
<p className="text-xs text-gray-700 mt-2">Salty will display her messages here</p>
|
||
</div>
|
||
) : (
|
||
saltyMessages.map((entry) => (
|
||
<SaltyMessageBubble
|
||
key={entry.id}
|
||
entry={entry}
|
||
onSpeak={handleSpeak}
|
||
/>
|
||
))
|
||
)}
|
||
<div ref={saltyBottomRef} />
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
{/* On-screen touch keyboard */}
|
||
<div className="shrink-0 border border-gray-800 rounded-lg bg-gray-950">
|
||
<TouchKeyboard
|
||
value={keyboardInput}
|
||
onInput={setKeyboardInput}
|
||
onSend={handleSendMessage}
|
||
onClear={() => setKeyboardInput('')}
|
||
/>
|
||
</div>
|
||
|
||
{/* Footer with instructions */}
|
||
<div className="text-xs text-gray-600 border-t border-gray-800 pt-2 shrink-0">
|
||
<p>💡 <strong>Tips:</strong> Speak clearly for speech-to-text, or use the keyboard below. Click 🔊 to hear Salty's responses aloud.</p>
|
||
</div>
|
||
</div>
|
||
);
|
||
}
|