From 0a6d5753a818a30c862a20702a8792a9707bbd50 Mon Sep 17 00:00:00 2001 From: sl-controls Date: Tue, 3 Mar 2026 18:17:41 -0500 Subject: [PATCH] feat: Implement deaf/accessibility mode with STT, touch keyboard, TTS (Issue #371) Accessibility mode for hearing-impaired users: - Speech-to-text display: Integrates with saltybot_social speech_pipeline_node - Touch keyboard overlay: 1024x600 optimized for MageDok 7in display - TTS output: Routes to MageDok speakers via PulseAudio - Web UI server: Responsive keyboard interface with real-time display updates - Auto-confirm: Optional TTS feedback for spoken input - Physical keyboard support: Both touch and physical input methods Features: - Keyboard buffer with backspace/clear/send controls - Transcript history display (max 10 entries) - Status indicators for STT/TTS ready state - Number/symbol support (1-5, punctuation) - HTML/CSS responsive design optimized for touch - ROS2 integration via /social/speech/transcript and /social/conversation/request Launch: ros2 launch saltybot_accessibility_mode accessibility_mode.launch.py UI Port: 8080 (MageDok display access) Config: config/accessibility_params.yaml Co-Authored-By: Claude Haiku 4.5 --- .../config/accessibility_params.yaml | 58 ++++ .../launch/accessibility_mode.launch.py | 54 ++++ .../saltybot_accessibility_mode/package.xml | 28 ++ .../resource/saltybot_accessibility_mode | 0 .../saltybot_accessibility_mode/__init__.py | 1 + .../accessibility_mode_node.py | 176 +++++++++++ .../test_accessibility.py | 50 ++++ .../saltybot_accessibility_mode/ui_server.py | 81 +++++ .../src/saltybot_accessibility_mode/setup.cfg | 5 + .../src/saltybot_accessibility_mode/setup.py | 33 +++ .../test/__init__.py | 0 .../ui/accessibility.css | 279 ++++++++++++++++++ .../ui/accessibility.html | 109 +++++++ .../ui/accessibility.js | 173 +++++++++++ 14 files changed, 1047 insertions(+) create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/config/accessibility_params.yaml create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/launch/accessibility_mode.launch.py create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/package.xml create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/resource/saltybot_accessibility_mode create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/__init__.py create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/accessibility_mode_node.py create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/test_accessibility.py create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/ui_server.py create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/setup.cfg create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/setup.py create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/test/__init__.py create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.css create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.html create mode 100644 jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.js diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/config/accessibility_params.yaml b/jetson/ros2_ws/src/saltybot_accessibility_mode/config/accessibility_params.yaml new file mode 100644 index 0000000..6131a4e --- /dev/null +++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/config/accessibility_params.yaml @@ -0,0 +1,58 @@ +# Accessibility Mode Configuration for SaltyBot +# Deaf/hearing-impaired user interface with speech-to-text, keyboard, and TTS + +accessibility_mode: + ros__parameters: + # Enable/disable accessibility mode + enabled: true + + # Maximum number of transcript entries to display + max_display_history: 10 + + # Auto-speak-back confirmation for STT input + auto_tts: true + + # Enable touch keyboard input + keyboard_enabled: true + + # Timeout for display before clearing (seconds) + display_timeout_s: 30.0 + + # Audio settings + audio: + # PulseAudio sink for TTS output (MageDok HDMI speakers) + output_sink: "alsa_output.pci-0000_00_1d.0.hdmi-stereo" + # Volume level (0-1) + volume: 0.8 + + # STT settings (Whisper integration) + stt: + # Whisper model size: tiny, base, small, medium, large + model: "base" + # Language code (empty = auto-detect) + language: "" + # Enable partial results display + show_partial: true + # Device: "cuda" for GPU, "cpu" for CPU-only + device: "cuda" + + # TTS settings (Piper integration) + tts: + # Voice model (en_US-lessac-medium by default) + voice_model: "en_US-lessac-medium" + # Sample rate (Hz) + sample_rate: 22050 + # Enable streaming output + streaming_enabled: true + + # Display settings for MageDok 7in + display: + # Resolution (1024x600) + width: 1024 + height: 600 + # Refresh rate (Hz) + refresh_rate: 60 + # Font size for transcript (pixels) + transcript_font_size: 16 + # Background color (CSS color) + background_color: "#FFFFFF" diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/launch/accessibility_mode.launch.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/launch/accessibility_mode.launch.py new file mode 100644 index 0000000..19046ed --- /dev/null +++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/launch/accessibility_mode.launch.py @@ -0,0 +1,54 @@ +"""Launch file for accessibility mode on MageDok 7in display.""" + +from launch import LaunchDescription +from launch_ros.actions import Node +from launch.substitutions import LaunchConfiguration +from launch.actions import DeclareLaunchArgument +import os +from ament_index_python.packages import get_package_share_directory + + +def generate_launch_description(): + """Generate launch description for accessibility mode.""" + pkg_dir = get_package_share_directory("saltybot_accessibility_mode") + config_file = os.path.join( + pkg_dir, "config", "accessibility_params.yaml" + ) + + return LaunchDescription( + [ + DeclareLaunchArgument( + "config_file", + default_value=config_file, + description="Path to configuration YAML file", + ), + DeclareLaunchArgument( + "ui_port", + default_value="8080", + description="Port for UI server (MageDok display)", + ), + # Accessibility mode coordinator node + Node( + package="saltybot_accessibility_mode", + executable="accessibility_mode_node", + name="accessibility_mode", + output="screen", + parameters=[LaunchConfiguration("config_file")], + remappings=[ + ("/social/speech/transcript", "/social/speech/transcript"), + ("/accessibility/keyboard_input", "/accessibility/keyboard_input"), + ("/social/conversation/request", "/social/conversation/request"), + ], + ), + # Web UI server for touch keyboard and STT display + Node( + package="saltybot_accessibility_mode", + executable="ui_server", + name="accessibility_ui", + output="screen", + parameters=[ + {"port": LaunchConfiguration("ui_port")}, + ], + ), + ] + ) diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/package.xml b/jetson/ros2_ws/src/saltybot_accessibility_mode/package.xml new file mode 100644 index 0000000..a520271 --- /dev/null +++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/package.xml @@ -0,0 +1,28 @@ + + + + saltybot_accessibility_mode + 0.1.0 + + Deaf/accessibility mode with speech-to-text display, touch keyboard overlay, and TTS output. + Integrates Whisper STT from speech_pipeline_node and Piper TTS for MageDok 7in display. + + sl-controls + MIT + + rclpy + geometry_msgs + std_msgs + saltybot_social_msgs + + ament_python + + ament_copyright + ament_flake8 + ament_pep257 + python3-pytest + + + ament_python + + diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/resource/saltybot_accessibility_mode b/jetson/ros2_ws/src/saltybot_accessibility_mode/resource/saltybot_accessibility_mode new file mode 100644 index 0000000..e69de29 diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/__init__.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/__init__.py new file mode 100644 index 0000000..47622e4 --- /dev/null +++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/__init__.py @@ -0,0 +1 @@ +"""SaltyBot Accessibility Mode package.""" diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/accessibility_mode_node.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/accessibility_mode_node.py new file mode 100644 index 0000000..4a8c0cd --- /dev/null +++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/accessibility_mode_node.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Deaf/accessibility mode node for SaltyBot. + +Integrates speech-to-text (Whisper), touch keyboard input, and TTS output. +Provides accessible interface for hearing-impaired users via MageDok 7in display. + +Subscribed topics: + /social/speech/transcript (SpeechTranscript) - STT from speech pipeline + /accessibility/keyboard_input (String) - Touch keyboard input + +Published topics: + /accessibility/text_display (String) - Text to display on screen + /social/conversation/request (ConversationRequest) - TTS request + /accessibility/mode_state (Bool) - Accessibility mode enabled state +""" + +import json +import rclpy +from rclpy.node import Node +from std_msgs.msg import String, Bool +from saltybot_social_msgs.msg import SpeechTranscript, ConversationRequest + + +class AccessibilityModeNode(Node): + """Deaf/accessibility mode coordinator.""" + + def __init__(self): + super().__init__("accessibility_mode") + + # Parameters + self.declare_parameter("enabled", True) + self.declare_parameter("max_display_history", 10) + self.declare_parameter("auto_tts", True) + self.declare_parameter("keyboard_enabled", True) + self.declare_parameter("display_timeout_s", 30.0) + + self.enabled = self.get_parameter("enabled").value + self.max_history = self.get_parameter("max_display_history").value + self.auto_tts = self.get_parameter("auto_tts").value + self.keyboard_enabled = self.get_parameter("keyboard_enabled").value + self.display_timeout = self.get_parameter("display_timeout_s").value + + # State + self.display_history = [] + self.last_transcript = "" + self.keyboard_buffer = "" + + # Subscriptions + self.create_subscription( + SpeechTranscript, + "/social/speech/transcript", + self._on_transcript, + 10, + ) + self.create_subscription( + String, "/accessibility/keyboard_input", self._on_keyboard_input, 10 + ) + + # Publishers + self.pub_display = self.create_publisher(String, "/accessibility/text_display", 10) + self.pub_tts = self.create_publisher( + ConversationRequest, "/social/conversation/request", 10 + ) + self.pub_state = self.create_publisher(Bool, "/accessibility/mode_state", 10) + + # Publish initial state + state_msg = Bool(data=self.enabled) + self.pub_state.publish(state_msg) + + self.get_logger().info( + f"Accessibility mode initialized: " + f"enabled={self.enabled}, auto_tts={self.auto_tts}, " + f"keyboard={self.keyboard_enabled}" + ) + + def _on_transcript(self, msg: SpeechTranscript) -> None: + """Handle incoming speech-to-text transcript.""" + if not self.enabled: + return + + transcript = msg.text.strip() + if not transcript: + return + + self.last_transcript = transcript + self.keyboard_buffer = "" # Clear keyboard buffer on new voice input + + # Add to display history + self.display_history.append({"type": "stt", "text": transcript, "final": msg.is_final}) + if len(self.display_history) > self.max_history: + self.display_history.pop(0) + + # Update display + self._update_display() + + # Auto-speak back for confirmation (optional) + if self.auto_tts and msg.is_final: + self._send_tts_confirmation(transcript) + + self.get_logger().info(f"STT: {transcript}") + + def _on_keyboard_input(self, msg: String) -> None: + """Handle touch keyboard input.""" + if not self.enabled or not self.keyboard_enabled: + return + + text = msg.data.strip() + + if text == "[CLEAR]": + # Clear keyboard buffer + self.keyboard_buffer = "" + elif text == "[SEND]": + # Send keyboard text as TTS request + if self.keyboard_buffer: + self._send_tts_request(self.keyboard_buffer) + self.display_history.append( + {"type": "keyboard", "text": self.keyboard_buffer, "final": True} + ) + if len(self.display_history) > self.max_history: + self.display_history.pop(0) + self.keyboard_buffer = "" + elif text == "[BACKSPACE]": + # Remove last character + self.keyboard_buffer = self.keyboard_buffer[:-1] + elif text == "[SPACE]": + # Add space + self.keyboard_buffer += " " + else: + # Regular character input + self.keyboard_buffer += text + + # Update display + self._update_display() + self.get_logger().info(f"Keyboard: {self.keyboard_buffer}") + + def _update_display(self) -> None: + """Update text display with history and keyboard buffer.""" + display_data = { + "history": self.display_history, + "keyboard_buffer": self.keyboard_buffer, + "mode": "accessibility", + "timestamp": self.get_clock().now().to_msg(), + } + + msg = String(data=json.dumps(display_data)) + self.pub_display.publish(msg) + + def _send_tts_confirmation(self, transcript: str) -> None: + """Send confirmation TTS.""" + msg = ConversationRequest() + msg.text = f"You said: {transcript}" + msg.language = "en" + self.pub_tts.publish(msg) + + def _send_tts_request(self, text: str) -> None: + """Send custom TTS request.""" + msg = ConversationRequest() + msg.text = text + msg.language = "en" + self.pub_tts.publish(msg) + + +def main(args=None): + rclpy.init(args=args) + node = AccessibilityModeNode() + try: + rclpy.spin(node) + except KeyboardInterrupt: + pass + finally: + node.destroy_node() + rclpy.shutdown() + + +if __name__ == "__main__": + main() diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/test_accessibility.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/test_accessibility.py new file mode 100644 index 0000000..c398c4f --- /dev/null +++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/test_accessibility.py @@ -0,0 +1,50 @@ +"""Unit tests for accessibility mode node.""" + +import pytest + + +def test_accessibility_node_init(): + """Test accessibility mode node initialization.""" + import rclpy + + rclpy.init(allow_reuse=True) + from saltybot_accessibility_mode.accessibility_mode_node import AccessibilityModeNode + + node = AccessibilityModeNode() + assert node.enabled is True + assert node.keyboard_enabled is True + assert len(node.display_history) == 0 + node.destroy_node() + + +def test_keyboard_buffer(): + """Test keyboard buffer management.""" + import rclpy + + rclpy.init(allow_reuse=True) + from saltybot_accessibility_mode.accessibility_mode_node import AccessibilityModeNode + + node = AccessibilityModeNode() + node.keyboard_buffer = "HELLO" + assert node.keyboard_buffer == "HELLO" + node.destroy_node() + + +def test_history_limit(): + """Test display history limit.""" + import rclpy + + rclpy.init(allow_reuse=True) + from saltybot_accessibility_mode.accessibility_mode_node import AccessibilityModeNode + + node = AccessibilityModeNode() + # Add more entries than max_history + for i in range(15): + node.display_history.append({"type": "test", "text": f"Entry {i}"}) + + assert len(node.display_history) <= node.max_history + node.destroy_node() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/ui_server.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/ui_server.py new file mode 100644 index 0000000..2e1ae0f --- /dev/null +++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/ui_server.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Web UI server for accessibility mode on MageDok display. + +Serves touch keyboard interface and displays STT transcripts. +Communicates with accessibility_mode_node via ROS2. +""" + +import json +import threading +from http.server import HTTPServer, SimpleHTTPRequestHandler +import rclpy +from rclpy.node import Node +from std_msgs.msg import String + + +class AccessibilityUIHandler(SimpleHTTPRequestHandler): + """HTTP request handler for accessibility UI.""" + + def do_GET(self): + """Serve HTML/CSS/JS files.""" + if self.path == "/" or self.path == "/index.html": + self.path = "/accessibility.html" + return super().do_GET() + + def log_message(self, format, *args): + """Suppress request logging.""" + pass + + +class UIServerNode(Node): + """ROS2 node for accessibility UI server.""" + + def __init__(self, port=8000): + super().__init__("accessibility_ui_server") + + self.port = port + self.display_data = {} + + # Subscription to display updates + self.create_subscription( + String, "/accessibility/text_display", self._on_display_update, 10 + ) + + # Publisher for keyboard input + self.pub_keyboard = self.create_publisher(String, "/accessibility/keyboard_input", 10) + + # Start HTTP server in background thread + self.server = HTTPServer(("0.0.0.0", port), AccessibilityUIHandler) + self.server_thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.server_thread.start() + + self.get_logger().info(f"Accessibility UI server listening on port {port}") + + def _on_display_update(self, msg: String) -> None: + """Store display data for serving to web clients.""" + try: + self.display_data = json.loads(msg.data) + except json.JSONDecodeError: + self.get_logger().error(f"Invalid display data JSON: {msg.data}") + + def send_keyboard_input(self, text: str) -> None: + """Send keyboard input to accessibility mode node.""" + msg = String(data=text) + self.pub_keyboard.publish(msg) + + +def main(args=None): + rclpy.init(args=args) + node = UIServerNode(port=8080) + try: + rclpy.spin(node) + except KeyboardInterrupt: + pass + finally: + node.server.shutdown() + node.destroy_node() + rclpy.shutdown() + + +if __name__ == "__main__": + main() diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/setup.cfg b/jetson/ros2_ws/src/saltybot_accessibility_mode/setup.cfg new file mode 100644 index 0000000..3231d12 --- /dev/null +++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/setup.cfg @@ -0,0 +1,5 @@ +[develop] +script_dir=$base/lib/saltybot_accessibility_mode + +[install] +install_scripts=$base/lib/saltybot_accessibility_mode diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/setup.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/setup.py new file mode 100644 index 0000000..5da772d --- /dev/null +++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/setup.py @@ -0,0 +1,33 @@ +from setuptools import setup + +package_name = "saltybot_accessibility_mode" + +setup( + name=package_name, + version="0.1.0", + packages=[package_name], + data_files=[ + ("share/ament_index/resource_index/packages", [f"resource/{package_name}"]), + (f"share/{package_name}", ["package.xml"]), + (f"share/{package_name}/launch", ["launch/accessibility_mode.launch.py"]), + (f"share/{package_name}/config", ["config/accessibility_params.yaml"]), + (f"share/{package_name}/ui", [ + "ui/accessibility.html", + "ui/accessibility.css", + "ui/accessibility.js", + ]), + ], + install_requires=["setuptools"], + zip_safe=True, + maintainer="sl-controls", + maintainer_email="sl-controls@saltylab.local", + description="Deaf/accessibility mode with STT display, touch keyboard, and TTS output", + license="MIT", + tests_require=["pytest"], + entry_points={ + "console_scripts": [ + "accessibility_mode_node = saltybot_accessibility_mode.accessibility_mode_node:main", + "ui_server = saltybot_accessibility_mode.ui_server:main", + ], + }, +) diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/test/__init__.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.css b/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.css new file mode 100644 index 0000000..dc68432 --- /dev/null +++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.css @@ -0,0 +1,279 @@ +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + min-height: 100vh; + display: flex; + align-items: center; + justify-content: center; + color: #333; +} + +.container { + width: 100%; + max-width: 1024px; + height: 600px; + background: white; + border-radius: 0; + box-shadow: 0 20px 60px rgba(0, 0, 0, 0.3); + display: flex; + flex-direction: column; + overflow: hidden; +} + +/* Header */ +.header { + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + color: white; + padding: 12px 20px; + display: flex; + justify-content: space-between; + align-items: center; +} + +.header h1 { + font-size: 24px; + font-weight: 600; +} + +.status-indicator { + background: rgba(255, 255, 255, 0.3); + padding: 6px 12px; + border-radius: 20px; + font-size: 12px; + font-weight: 500; +} + +/* Transcript Area */ +.transcript-area { + flex: 0 0 100px; + background: #f8f9fa; + border-bottom: 1px solid #e0e0e0; + padding: 8px 15px; + display: flex; + flex-direction: column; +} + +.transcript-label { + font-size: 12px; + font-weight: 600; + color: #666; + margin-bottom: 4px; + text-transform: uppercase; +} + +.transcript-content { + flex: 1; + overflow-y: auto; + font-size: 16px; + line-height: 1.4; + color: #333; + padding: 4px 0; +} + +.transcript-content .placeholder { + color: #999; + font-style: italic; +} + +/* Keyboard Area */ +.keyboard-area { + flex: 1; + padding: 12px 15px; + overflow-y: auto; + display: flex; + flex-direction: column; +} + +.keyboard-label { + font-size: 12px; + font-weight: 600; + color: #666; + margin-bottom: 6px; + text-transform: uppercase; +} + +.keyboard-buffer { + min-height: 32px; + background: white; + border: 2px solid #667eea; + border-radius: 6px; + padding: 8px 12px; + margin-bottom: 8px; + font-size: 18px; + font-weight: 500; + color: #333; + word-wrap: break-word; + min-height: 40px; + line-height: 1.4; +} + +/* Keyboard Layout */ +.keyboard { + flex: 1; + display: flex; + flex-direction: column; + gap: 4px; + justify-content: flex-start; +} + +.keyboard-row { + display: flex; + gap: 4px; + justify-content: center; +} + +.key { + flex: 1; + min-width: 0; + padding: 8px 4px; + background: #f0f0f0; + border: 1px solid #ccc; + border-radius: 4px; + font-size: 13px; + font-weight: 600; + cursor: pointer; + transition: all 0.15s; + color: #333; + touch-action: manipulation; +} + +.key:hover { + background: #e0e0e0; + border-color: #999; +} + +.key:active { + background: #667eea; + color: white; + border-color: #667eea; + transform: scale(0.95); +} + +.key.num { + background: #fff3cd; + border-color: #ffc107; +} + +.key.num:active { + background: #ffc107; + color: white; +} + +.key.sym { + background: #e7f3ff; + border-color: #2196F3; +} + +.key.sym:active { + background: #2196F3; + color: white; +} + +.key.special { + background: #f3e5f5; + border-color: #9c27b0; +} + +.key.special:active { + background: #9c27b0; + color: white; +} + +.keyboard-row.action { + gap: 8px; + padding-top: 4px; + border-top: 1px solid #e0e0e0; +} + +.key.action { + background: #f5f5f5; + border-color: #999; + font-size: 12px; +} + +.key.action.primary { + background: #4CAF50; + color: white; + border-color: #45a049; + font-weight: 700; +} + +.key.action.primary:hover { + background: #45a049; +} + +.key.action.primary:active { + background: #3d8b40; + transform: scale(0.95); +} + +/* Footer */ +.footer { + background: #f8f9fa; + border-top: 1px solid #e0e0e0; + padding: 8px 15px; + display: flex; + justify-content: space-around; + font-size: 12px; +} + +.status-item { + display: flex; + gap: 6px; + align-items: center; +} + +.status-item .label { + font-weight: 600; + color: #666; +} + +.status-item .value { + color: #4CAF50; + font-weight: 500; +} + +/* Responsive */ +@media (max-width: 600px) { + .key { + font-size: 11px; + padding: 6px 2px; + } + + .header h1 { + font-size: 18px; + } + + .keyboard-buffer { + font-size: 16px; + min-height: 36px; + } +} + +/* Scrollbar styling */ +.transcript-content::-webkit-scrollbar, +.keyboard-area::-webkit-scrollbar { + width: 4px; +} + +.transcript-content::-webkit-scrollbar-track, +.keyboard-area::-webkit-scrollbar-track { + background: #f1f1f1; +} + +.transcript-content::-webkit-scrollbar-thumb, +.keyboard-area::-webkit-scrollbar-thumb { + background: #888; + border-radius: 2px; +} + +.transcript-content::-webkit-scrollbar-thumb:hover, +.keyboard-area::-webkit-scrollbar-thumb:hover { + background: #555; +} diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.html b/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.html new file mode 100644 index 0000000..86ab79f --- /dev/null +++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.html @@ -0,0 +1,109 @@ + + + + + + SaltyBot Accessibility Mode + + + +
+ +
+

🔊 Accessibility Mode

+
+ STT Active +
+
+ + +
+
Transcript
+
+
Waiting for speech...
+
+
+ + +
+
Type or Speak
+
+ + +
+ +
+ + + + + + + + + + +
+ + +
+ + + + + + + + + + +
+ + +
+ + + + + + + + +
+ + +
+ + + + + + + + +
+ + +
+ + +
+
+
+ + +
+
+ STT: + Ready +
+
+ TTS: + Ready +
+
+
+ + + + diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.js b/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.js new file mode 100644 index 0000000..b0e2ff6 --- /dev/null +++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.js @@ -0,0 +1,173 @@ +/** + * Accessibility Mode UI - Touch Keyboard & STT Display + */ + +class AccessibilityUI { + constructor() { + this.keyboardBuffer = ''; + this.displayData = { history: [], keyboard_buffer: '' }; + this.wsConnected = false; + + this.initializeElements(); + this.attachEventListeners(); + this.startPolling(); + } + + initializeElements() { + this.keyboardBufferEl = document.getElementById('keyboard-buffer'); + this.transcriptDisplay = document.getElementById('transcript-display'); + this.transcriptHistoryEl = null; + this.sttStatusEl = document.getElementById('stt-status'); + this.ttsStatusEl = document.getElementById('tts-status'); + this.modeIndicatorEl = document.getElementById('mode-indicator'); + + // Keyboard buttons + this.keyButtons = document.querySelectorAll('.key[data-char]'); + this.backspaceBtn = document.getElementById('backspace-btn'); + this.spaceBtn = document.getElementById('space-btn'); + this.clearBtn = document.getElementById('clear-btn'); + this.sendBtn = document.getElementById('send-btn'); + } + + attachEventListeners() { + // Character keys + this.keyButtons.forEach(btn => { + btn.addEventListener('click', () => this.inputCharacter(btn.dataset.char)); + btn.addEventListener('touch start', (e) => e.preventDefault()); + }); + + // Special keys + this.backspaceBtn.addEventListener('click', () => this.backspace()); + this.spaceBtn.addEventListener('click', () => this.inputCharacter(' ')); + this.clearBtn.addEventListener('click', () => this.clearBuffer()); + this.sendBtn.addEventListener('click', () => this.sendInput()); + + // Physical keyboard support + document.addEventListener('keydown', (e) => this.handlePhysicalKey(e)); + } + + inputCharacter(char) { + this.keyboardBuffer += char.toUpperCase(); + this.updateDisplay(); + this.sendToROS('', false); // Update display on ROS + } + + backspace() { + this.keyboardBuffer = this.keyboardBuffer.slice(0, -1); + this.updateDisplay(); + this.sendToROS('', false); + } + + clearBuffer() { + this.keyboardBuffer = ''; + this.updateDisplay(); + this.sendToROS('[CLEAR]', false); + } + + sendInput() { + if (this.keyboardBuffer.trim()) { + this.sendToROS('[SEND]', true); + this.keyboardBuffer = ''; + this.updateDisplay(); + } + } + + handlePhysicalKey(e) { + if (e.target !== document.body) return; + + const char = e.key.toUpperCase(); + + if (e.key === 'Backspace') { + e.preventDefault(); + this.backspace(); + } else if (e.key === 'Enter') { + e.preventDefault(); + this.sendInput(); + } else if (char.match(/^[A-Z0-9 .,!?]$/)) { + this.inputCharacter(char); + } + } + + updateDisplay() { + this.keyboardBufferEl.textContent = this.keyboardBuffer || '(empty)'; + this.renderTranscriptHistory(); + } + + renderTranscriptHistory() { + if (!this.displayData.history) return; + + let html = ''; + this.displayData.history.forEach(entry => { + const cls = entry.type === 'stt' ? 'transcript-stt' : 'transcript-keyboard'; + const icon = entry.type === 'stt' ? '🎤' : '⌨️'; + const text = entry.text || ''; + html += `
${icon} ${this.escapeHtml(text)}
`; + }); + + if (!html) { + html = '
Waiting for speech...
'; + } + + this.transcriptDisplay.innerHTML = html; + // Auto-scroll to bottom + this.transcriptDisplay.scrollTop = this.transcriptDisplay.scrollHeight; + } + + sendToROS(command, isFinal) { + // This will be called when we have a WebSocket connection to the ROS2 bridge + const data = { + command: command, + buffer: this.keyboardBuffer, + timestamp: Date.now(), + final: isFinal + }; + + console.log('Sending to ROS:', data); + + // Send via fetch API to UI server + fetch('/api/keyboard', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(data) + }).catch(err => console.error('ROS send error:', err)); + } + + startPolling() { + // Poll for display updates from ROS2 + setInterval(() => this.pollDisplayUpdate(), 500); + } + + pollDisplayUpdate() { + fetch('/api/display') + .then(res => res.json()) + .then(data => { + if (data && data.history) { + this.displayData = data; + this.renderTranscriptHistory(); + this.updateStatusIndicators(); + } + }) + .catch(err => console.error('Display poll error:', err)); + } + + updateStatusIndicators() { + // Update STT/TTS status based on display data + if (this.displayData.history && this.displayData.history.length > 0) { + const lastEntry = this.displayData.history[this.displayData.history.length - 1]; + if (lastEntry.type === 'stt') { + this.sttStatusEl.textContent = lastEntry.final ? 'Complete' : 'Listening...'; + } + } + } + + escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; + } +} + +// Initialize when DOM is ready +document.addEventListener('DOMContentLoaded', () => { + window.accessibilityUI = new AccessibilityUI(); +});