diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/config/accessibility_params.yaml b/jetson/ros2_ws/src/saltybot_accessibility_mode/config/accessibility_params.yaml
new file mode 100644
index 0000000..6131a4e
--- /dev/null
+++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/config/accessibility_params.yaml
@@ -0,0 +1,58 @@
+# Accessibility Mode Configuration for SaltyBot
+# Deaf/hearing-impaired user interface with speech-to-text, keyboard, and TTS
+
+accessibility_mode:
+ ros__parameters:
+ # Enable/disable accessibility mode
+ enabled: true
+
+ # Maximum number of transcript entries to display
+ max_display_history: 10
+
+ # Auto-speak-back confirmation for STT input
+ auto_tts: true
+
+ # Enable touch keyboard input
+ keyboard_enabled: true
+
+ # Timeout for display before clearing (seconds)
+ display_timeout_s: 30.0
+
+ # Audio settings
+ audio:
+ # PulseAudio sink for TTS output (MageDok HDMI speakers)
+ output_sink: "alsa_output.pci-0000_00_1d.0.hdmi-stereo"
+ # Volume level (0-1)
+ volume: 0.8
+
+ # STT settings (Whisper integration)
+ stt:
+ # Whisper model size: tiny, base, small, medium, large
+ model: "base"
+ # Language code (empty = auto-detect)
+ language: ""
+ # Enable partial results display
+ show_partial: true
+ # Device: "cuda" for GPU, "cpu" for CPU-only
+ device: "cuda"
+
+ # TTS settings (Piper integration)
+ tts:
+ # Voice model (en_US-lessac-medium by default)
+ voice_model: "en_US-lessac-medium"
+ # Sample rate (Hz)
+ sample_rate: 22050
+ # Enable streaming output
+ streaming_enabled: true
+
+ # Display settings for MageDok 7in
+ display:
+ # Resolution (1024x600)
+ width: 1024
+ height: 600
+ # Refresh rate (Hz)
+ refresh_rate: 60
+ # Font size for transcript (pixels)
+ transcript_font_size: 16
+ # Background color (CSS color)
+ background_color: "#FFFFFF"
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/launch/accessibility_mode.launch.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/launch/accessibility_mode.launch.py
new file mode 100644
index 0000000..19046ed
--- /dev/null
+++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/launch/accessibility_mode.launch.py
@@ -0,0 +1,54 @@
+"""Launch file for accessibility mode on MageDok 7in display."""
+
+from launch import LaunchDescription
+from launch_ros.actions import Node
+from launch.substitutions import LaunchConfiguration
+from launch.actions import DeclareLaunchArgument
+import os
+from ament_index_python.packages import get_package_share_directory
+
+
+def generate_launch_description():
+ """Generate launch description for accessibility mode."""
+ pkg_dir = get_package_share_directory("saltybot_accessibility_mode")
+ config_file = os.path.join(
+ pkg_dir, "config", "accessibility_params.yaml"
+ )
+
+ return LaunchDescription(
+ [
+ DeclareLaunchArgument(
+ "config_file",
+ default_value=config_file,
+ description="Path to configuration YAML file",
+ ),
+ DeclareLaunchArgument(
+ "ui_port",
+ default_value="8080",
+ description="Port for UI server (MageDok display)",
+ ),
+ # Accessibility mode coordinator node
+ Node(
+ package="saltybot_accessibility_mode",
+ executable="accessibility_mode_node",
+ name="accessibility_mode",
+ output="screen",
+ parameters=[LaunchConfiguration("config_file")],
+ remappings=[
+ ("/social/speech/transcript", "/social/speech/transcript"),
+ ("/accessibility/keyboard_input", "/accessibility/keyboard_input"),
+ ("/social/conversation/request", "/social/conversation/request"),
+ ],
+ ),
+ # Web UI server for touch keyboard and STT display
+ Node(
+ package="saltybot_accessibility_mode",
+ executable="ui_server",
+ name="accessibility_ui",
+ output="screen",
+ parameters=[
+ {"port": LaunchConfiguration("ui_port")},
+ ],
+ ),
+ ]
+ )
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/package.xml b/jetson/ros2_ws/src/saltybot_accessibility_mode/package.xml
new file mode 100644
index 0000000..a520271
--- /dev/null
+++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/package.xml
@@ -0,0 +1,28 @@
+
+
+
+ saltybot_accessibility_mode
+ 0.1.0
+
+ Deaf/accessibility mode with speech-to-text display, touch keyboard overlay, and TTS output.
+ Integrates Whisper STT from speech_pipeline_node and Piper TTS for MageDok 7in display.
+
+ sl-controls
+ MIT
+
+ rclpy
+ geometry_msgs
+ std_msgs
+ saltybot_social_msgs
+
+ ament_python
+
+ ament_copyright
+ ament_flake8
+ ament_pep257
+ python3-pytest
+
+
+ ament_python
+
+
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/resource/saltybot_accessibility_mode b/jetson/ros2_ws/src/saltybot_accessibility_mode/resource/saltybot_accessibility_mode
new file mode 100644
index 0000000..e69de29
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/__init__.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/__init__.py
new file mode 100644
index 0000000..47622e4
--- /dev/null
+++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/__init__.py
@@ -0,0 +1 @@
+"""SaltyBot Accessibility Mode package."""
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/accessibility_mode_node.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/accessibility_mode_node.py
new file mode 100644
index 0000000..4a8c0cd
--- /dev/null
+++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/accessibility_mode_node.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+"""Deaf/accessibility mode node for SaltyBot.
+
+Integrates speech-to-text (Whisper), touch keyboard input, and TTS output.
+Provides accessible interface for hearing-impaired users via MageDok 7in display.
+
+Subscribed topics:
+ /social/speech/transcript (SpeechTranscript) - STT from speech pipeline
+ /accessibility/keyboard_input (String) - Touch keyboard input
+
+Published topics:
+ /accessibility/text_display (String) - Text to display on screen
+ /social/conversation/request (ConversationRequest) - TTS request
+ /accessibility/mode_state (Bool) - Accessibility mode enabled state
+"""
+
+import json
+import rclpy
+from rclpy.node import Node
+from std_msgs.msg import String, Bool
+from saltybot_social_msgs.msg import SpeechTranscript, ConversationRequest
+
+
+class AccessibilityModeNode(Node):
+ """Deaf/accessibility mode coordinator."""
+
+ def __init__(self):
+ super().__init__("accessibility_mode")
+
+ # Parameters
+ self.declare_parameter("enabled", True)
+ self.declare_parameter("max_display_history", 10)
+ self.declare_parameter("auto_tts", True)
+ self.declare_parameter("keyboard_enabled", True)
+ self.declare_parameter("display_timeout_s", 30.0)
+
+ self.enabled = self.get_parameter("enabled").value
+ self.max_history = self.get_parameter("max_display_history").value
+ self.auto_tts = self.get_parameter("auto_tts").value
+ self.keyboard_enabled = self.get_parameter("keyboard_enabled").value
+ self.display_timeout = self.get_parameter("display_timeout_s").value
+
+ # State
+ self.display_history = []
+ self.last_transcript = ""
+ self.keyboard_buffer = ""
+
+ # Subscriptions
+ self.create_subscription(
+ SpeechTranscript,
+ "/social/speech/transcript",
+ self._on_transcript,
+ 10,
+ )
+ self.create_subscription(
+ String, "/accessibility/keyboard_input", self._on_keyboard_input, 10
+ )
+
+ # Publishers
+ self.pub_display = self.create_publisher(String, "/accessibility/text_display", 10)
+ self.pub_tts = self.create_publisher(
+ ConversationRequest, "/social/conversation/request", 10
+ )
+ self.pub_state = self.create_publisher(Bool, "/accessibility/mode_state", 10)
+
+ # Publish initial state
+ state_msg = Bool(data=self.enabled)
+ self.pub_state.publish(state_msg)
+
+ self.get_logger().info(
+ f"Accessibility mode initialized: "
+ f"enabled={self.enabled}, auto_tts={self.auto_tts}, "
+ f"keyboard={self.keyboard_enabled}"
+ )
+
+ def _on_transcript(self, msg: SpeechTranscript) -> None:
+ """Handle incoming speech-to-text transcript."""
+ if not self.enabled:
+ return
+
+ transcript = msg.text.strip()
+ if not transcript:
+ return
+
+ self.last_transcript = transcript
+ self.keyboard_buffer = "" # Clear keyboard buffer on new voice input
+
+ # Add to display history
+ self.display_history.append({"type": "stt", "text": transcript, "final": msg.is_final})
+ if len(self.display_history) > self.max_history:
+ self.display_history.pop(0)
+
+ # Update display
+ self._update_display()
+
+ # Auto-speak back for confirmation (optional)
+ if self.auto_tts and msg.is_final:
+ self._send_tts_confirmation(transcript)
+
+ self.get_logger().info(f"STT: {transcript}")
+
+ def _on_keyboard_input(self, msg: String) -> None:
+ """Handle touch keyboard input."""
+ if not self.enabled or not self.keyboard_enabled:
+ return
+
+ text = msg.data.strip()
+
+ if text == "[CLEAR]":
+ # Clear keyboard buffer
+ self.keyboard_buffer = ""
+ elif text == "[SEND]":
+ # Send keyboard text as TTS request
+ if self.keyboard_buffer:
+ self._send_tts_request(self.keyboard_buffer)
+ self.display_history.append(
+ {"type": "keyboard", "text": self.keyboard_buffer, "final": True}
+ )
+ if len(self.display_history) > self.max_history:
+ self.display_history.pop(0)
+ self.keyboard_buffer = ""
+ elif text == "[BACKSPACE]":
+ # Remove last character
+ self.keyboard_buffer = self.keyboard_buffer[:-1]
+ elif text == "[SPACE]":
+ # Add space
+ self.keyboard_buffer += " "
+ else:
+ # Regular character input
+ self.keyboard_buffer += text
+
+ # Update display
+ self._update_display()
+ self.get_logger().info(f"Keyboard: {self.keyboard_buffer}")
+
+ def _update_display(self) -> None:
+ """Update text display with history and keyboard buffer."""
+ display_data = {
+ "history": self.display_history,
+ "keyboard_buffer": self.keyboard_buffer,
+ "mode": "accessibility",
+ "timestamp": self.get_clock().now().to_msg(),
+ }
+
+ msg = String(data=json.dumps(display_data))
+ self.pub_display.publish(msg)
+
+ def _send_tts_confirmation(self, transcript: str) -> None:
+ """Send confirmation TTS."""
+ msg = ConversationRequest()
+ msg.text = f"You said: {transcript}"
+ msg.language = "en"
+ self.pub_tts.publish(msg)
+
+ def _send_tts_request(self, text: str) -> None:
+ """Send custom TTS request."""
+ msg = ConversationRequest()
+ msg.text = text
+ msg.language = "en"
+ self.pub_tts.publish(msg)
+
+
+def main(args=None):
+ rclpy.init(args=args)
+ node = AccessibilityModeNode()
+ try:
+ rclpy.spin(node)
+ except KeyboardInterrupt:
+ pass
+ finally:
+ node.destroy_node()
+ rclpy.shutdown()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/test_accessibility.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/test_accessibility.py
new file mode 100644
index 0000000..c398c4f
--- /dev/null
+++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/test_accessibility.py
@@ -0,0 +1,50 @@
+"""Unit tests for accessibility mode node."""
+
+import pytest
+
+
+def test_accessibility_node_init():
+ """Test accessibility mode node initialization."""
+ import rclpy
+
+ rclpy.init(allow_reuse=True)
+ from saltybot_accessibility_mode.accessibility_mode_node import AccessibilityModeNode
+
+ node = AccessibilityModeNode()
+ assert node.enabled is True
+ assert node.keyboard_enabled is True
+ assert len(node.display_history) == 0
+ node.destroy_node()
+
+
+def test_keyboard_buffer():
+ """Test keyboard buffer management."""
+ import rclpy
+
+ rclpy.init(allow_reuse=True)
+ from saltybot_accessibility_mode.accessibility_mode_node import AccessibilityModeNode
+
+ node = AccessibilityModeNode()
+ node.keyboard_buffer = "HELLO"
+ assert node.keyboard_buffer == "HELLO"
+ node.destroy_node()
+
+
+def test_history_limit():
+ """Test display history limit."""
+ import rclpy
+
+ rclpy.init(allow_reuse=True)
+ from saltybot_accessibility_mode.accessibility_mode_node import AccessibilityModeNode
+
+ node = AccessibilityModeNode()
+ # Add more entries than max_history
+ for i in range(15):
+ node.display_history.append({"type": "test", "text": f"Entry {i}"})
+
+ assert len(node.display_history) <= node.max_history
+ node.destroy_node()
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/ui_server.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/ui_server.py
new file mode 100644
index 0000000..2e1ae0f
--- /dev/null
+++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/saltybot_accessibility_mode/ui_server.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""Web UI server for accessibility mode on MageDok display.
+
+Serves touch keyboard interface and displays STT transcripts.
+Communicates with accessibility_mode_node via ROS2.
+"""
+
+import json
+import threading
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+import rclpy
+from rclpy.node import Node
+from std_msgs.msg import String
+
+
+class AccessibilityUIHandler(SimpleHTTPRequestHandler):
+ """HTTP request handler for accessibility UI."""
+
+ def do_GET(self):
+ """Serve HTML/CSS/JS files."""
+ if self.path == "/" or self.path == "/index.html":
+ self.path = "/accessibility.html"
+ return super().do_GET()
+
+ def log_message(self, format, *args):
+ """Suppress request logging."""
+ pass
+
+
+class UIServerNode(Node):
+ """ROS2 node for accessibility UI server."""
+
+ def __init__(self, port=8000):
+ super().__init__("accessibility_ui_server")
+
+ self.port = port
+ self.display_data = {}
+
+ # Subscription to display updates
+ self.create_subscription(
+ String, "/accessibility/text_display", self._on_display_update, 10
+ )
+
+ # Publisher for keyboard input
+ self.pub_keyboard = self.create_publisher(String, "/accessibility/keyboard_input", 10)
+
+ # Start HTTP server in background thread
+ self.server = HTTPServer(("0.0.0.0", port), AccessibilityUIHandler)
+ self.server_thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+ self.server_thread.start()
+
+ self.get_logger().info(f"Accessibility UI server listening on port {port}")
+
+ def _on_display_update(self, msg: String) -> None:
+ """Store display data for serving to web clients."""
+ try:
+ self.display_data = json.loads(msg.data)
+ except json.JSONDecodeError:
+ self.get_logger().error(f"Invalid display data JSON: {msg.data}")
+
+ def send_keyboard_input(self, text: str) -> None:
+ """Send keyboard input to accessibility mode node."""
+ msg = String(data=text)
+ self.pub_keyboard.publish(msg)
+
+
+def main(args=None):
+ rclpy.init(args=args)
+ node = UIServerNode(port=8080)
+ try:
+ rclpy.spin(node)
+ except KeyboardInterrupt:
+ pass
+ finally:
+ node.server.shutdown()
+ node.destroy_node()
+ rclpy.shutdown()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/setup.cfg b/jetson/ros2_ws/src/saltybot_accessibility_mode/setup.cfg
new file mode 100644
index 0000000..3231d12
--- /dev/null
+++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/setup.cfg
@@ -0,0 +1,5 @@
+[develop]
+script_dir=$base/lib/saltybot_accessibility_mode
+
+[install]
+install_scripts=$base/lib/saltybot_accessibility_mode
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/setup.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/setup.py
new file mode 100644
index 0000000..5da772d
--- /dev/null
+++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/setup.py
@@ -0,0 +1,33 @@
+from setuptools import setup
+
+package_name = "saltybot_accessibility_mode"
+
+setup(
+ name=package_name,
+ version="0.1.0",
+ packages=[package_name],
+ data_files=[
+ ("share/ament_index/resource_index/packages", [f"resource/{package_name}"]),
+ (f"share/{package_name}", ["package.xml"]),
+ (f"share/{package_name}/launch", ["launch/accessibility_mode.launch.py"]),
+ (f"share/{package_name}/config", ["config/accessibility_params.yaml"]),
+ (f"share/{package_name}/ui", [
+ "ui/accessibility.html",
+ "ui/accessibility.css",
+ "ui/accessibility.js",
+ ]),
+ ],
+ install_requires=["setuptools"],
+ zip_safe=True,
+ maintainer="sl-controls",
+ maintainer_email="sl-controls@saltylab.local",
+ description="Deaf/accessibility mode with STT display, touch keyboard, and TTS output",
+ license="MIT",
+ tests_require=["pytest"],
+ entry_points={
+ "console_scripts": [
+ "accessibility_mode_node = saltybot_accessibility_mode.accessibility_mode_node:main",
+ "ui_server = saltybot_accessibility_mode.ui_server:main",
+ ],
+ },
+)
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/test/__init__.py b/jetson/ros2_ws/src/saltybot_accessibility_mode/test/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.css b/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.css
new file mode 100644
index 0000000..dc68432
--- /dev/null
+++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.css
@@ -0,0 +1,279 @@
+* {
+ margin: 0;
+ padding: 0;
+ box-sizing: border-box;
+}
+
+body {
+ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+ min-height: 100vh;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ color: #333;
+}
+
+.container {
+ width: 100%;
+ max-width: 1024px;
+ height: 600px;
+ background: white;
+ border-radius: 0;
+ box-shadow: 0 20px 60px rgba(0, 0, 0, 0.3);
+ display: flex;
+ flex-direction: column;
+ overflow: hidden;
+}
+
+/* Header */
+.header {
+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+ color: white;
+ padding: 12px 20px;
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+}
+
+.header h1 {
+ font-size: 24px;
+ font-weight: 600;
+}
+
+.status-indicator {
+ background: rgba(255, 255, 255, 0.3);
+ padding: 6px 12px;
+ border-radius: 20px;
+ font-size: 12px;
+ font-weight: 500;
+}
+
+/* Transcript Area */
+.transcript-area {
+ flex: 0 0 100px;
+ background: #f8f9fa;
+ border-bottom: 1px solid #e0e0e0;
+ padding: 8px 15px;
+ display: flex;
+ flex-direction: column;
+}
+
+.transcript-label {
+ font-size: 12px;
+ font-weight: 600;
+ color: #666;
+ margin-bottom: 4px;
+ text-transform: uppercase;
+}
+
+.transcript-content {
+ flex: 1;
+ overflow-y: auto;
+ font-size: 16px;
+ line-height: 1.4;
+ color: #333;
+ padding: 4px 0;
+}
+
+.transcript-content .placeholder {
+ color: #999;
+ font-style: italic;
+}
+
+/* Keyboard Area */
+.keyboard-area {
+ flex: 1;
+ padding: 12px 15px;
+ overflow-y: auto;
+ display: flex;
+ flex-direction: column;
+}
+
+.keyboard-label {
+ font-size: 12px;
+ font-weight: 600;
+ color: #666;
+ margin-bottom: 6px;
+ text-transform: uppercase;
+}
+
+.keyboard-buffer {
+ min-height: 32px;
+ background: white;
+ border: 2px solid #667eea;
+ border-radius: 6px;
+ padding: 8px 12px;
+ margin-bottom: 8px;
+ font-size: 18px;
+ font-weight: 500;
+ color: #333;
+ word-wrap: break-word;
+ min-height: 40px;
+ line-height: 1.4;
+}
+
+/* Keyboard Layout */
+.keyboard {
+ flex: 1;
+ display: flex;
+ flex-direction: column;
+ gap: 4px;
+ justify-content: flex-start;
+}
+
+.keyboard-row {
+ display: flex;
+ gap: 4px;
+ justify-content: center;
+}
+
+.key {
+ flex: 1;
+ min-width: 0;
+ padding: 8px 4px;
+ background: #f0f0f0;
+ border: 1px solid #ccc;
+ border-radius: 4px;
+ font-size: 13px;
+ font-weight: 600;
+ cursor: pointer;
+ transition: all 0.15s;
+ color: #333;
+ touch-action: manipulation;
+}
+
+.key:hover {
+ background: #e0e0e0;
+ border-color: #999;
+}
+
+.key:active {
+ background: #667eea;
+ color: white;
+ border-color: #667eea;
+ transform: scale(0.95);
+}
+
+.key.num {
+ background: #fff3cd;
+ border-color: #ffc107;
+}
+
+.key.num:active {
+ background: #ffc107;
+ color: white;
+}
+
+.key.sym {
+ background: #e7f3ff;
+ border-color: #2196F3;
+}
+
+.key.sym:active {
+ background: #2196F3;
+ color: white;
+}
+
+.key.special {
+ background: #f3e5f5;
+ border-color: #9c27b0;
+}
+
+.key.special:active {
+ background: #9c27b0;
+ color: white;
+}
+
+.keyboard-row.action {
+ gap: 8px;
+ padding-top: 4px;
+ border-top: 1px solid #e0e0e0;
+}
+
+.key.action {
+ background: #f5f5f5;
+ border-color: #999;
+ font-size: 12px;
+}
+
+.key.action.primary {
+ background: #4CAF50;
+ color: white;
+ border-color: #45a049;
+ font-weight: 700;
+}
+
+.key.action.primary:hover {
+ background: #45a049;
+}
+
+.key.action.primary:active {
+ background: #3d8b40;
+ transform: scale(0.95);
+}
+
+/* Footer */
+.footer {
+ background: #f8f9fa;
+ border-top: 1px solid #e0e0e0;
+ padding: 8px 15px;
+ display: flex;
+ justify-content: space-around;
+ font-size: 12px;
+}
+
+.status-item {
+ display: flex;
+ gap: 6px;
+ align-items: center;
+}
+
+.status-item .label {
+ font-weight: 600;
+ color: #666;
+}
+
+.status-item .value {
+ color: #4CAF50;
+ font-weight: 500;
+}
+
+/* Responsive */
+@media (max-width: 600px) {
+ .key {
+ font-size: 11px;
+ padding: 6px 2px;
+ }
+
+ .header h1 {
+ font-size: 18px;
+ }
+
+ .keyboard-buffer {
+ font-size: 16px;
+ min-height: 36px;
+ }
+}
+
+/* Scrollbar styling */
+.transcript-content::-webkit-scrollbar,
+.keyboard-area::-webkit-scrollbar {
+ width: 4px;
+}
+
+.transcript-content::-webkit-scrollbar-track,
+.keyboard-area::-webkit-scrollbar-track {
+ background: #f1f1f1;
+}
+
+.transcript-content::-webkit-scrollbar-thumb,
+.keyboard-area::-webkit-scrollbar-thumb {
+ background: #888;
+ border-radius: 2px;
+}
+
+.transcript-content::-webkit-scrollbar-thumb:hover,
+.keyboard-area::-webkit-scrollbar-thumb:hover {
+ background: #555;
+}
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.html b/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.html
new file mode 100644
index 0000000..86ab79f
--- /dev/null
+++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.html
@@ -0,0 +1,109 @@
+
+
+
+
+
+ SaltyBot Accessibility Mode
+
+
+
+
+
+
+
+
+
+
Transcript
+
+
Waiting for speech...
+
+
+
+
+
+
Type or Speak
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.js b/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.js
new file mode 100644
index 0000000..b0e2ff6
--- /dev/null
+++ b/jetson/ros2_ws/src/saltybot_accessibility_mode/ui/accessibility.js
@@ -0,0 +1,173 @@
+/**
+ * Accessibility Mode UI - Touch Keyboard & STT Display
+ */
+
+class AccessibilityUI {
+ constructor() {
+ this.keyboardBuffer = '';
+ this.displayData = { history: [], keyboard_buffer: '' };
+ this.wsConnected = false;
+
+ this.initializeElements();
+ this.attachEventListeners();
+ this.startPolling();
+ }
+
+ initializeElements() {
+ this.keyboardBufferEl = document.getElementById('keyboard-buffer');
+ this.transcriptDisplay = document.getElementById('transcript-display');
+ this.transcriptHistoryEl = null;
+ this.sttStatusEl = document.getElementById('stt-status');
+ this.ttsStatusEl = document.getElementById('tts-status');
+ this.modeIndicatorEl = document.getElementById('mode-indicator');
+
+ // Keyboard buttons
+ this.keyButtons = document.querySelectorAll('.key[data-char]');
+ this.backspaceBtn = document.getElementById('backspace-btn');
+ this.spaceBtn = document.getElementById('space-btn');
+ this.clearBtn = document.getElementById('clear-btn');
+ this.sendBtn = document.getElementById('send-btn');
+ }
+
+ attachEventListeners() {
+ // Character keys
+ this.keyButtons.forEach(btn => {
+ btn.addEventListener('click', () => this.inputCharacter(btn.dataset.char));
+ btn.addEventListener('touch start', (e) => e.preventDefault());
+ });
+
+ // Special keys
+ this.backspaceBtn.addEventListener('click', () => this.backspace());
+ this.spaceBtn.addEventListener('click', () => this.inputCharacter(' '));
+ this.clearBtn.addEventListener('click', () => this.clearBuffer());
+ this.sendBtn.addEventListener('click', () => this.sendInput());
+
+ // Physical keyboard support
+ document.addEventListener('keydown', (e) => this.handlePhysicalKey(e));
+ }
+
+ inputCharacter(char) {
+ this.keyboardBuffer += char.toUpperCase();
+ this.updateDisplay();
+ this.sendToROS('', false); // Update display on ROS
+ }
+
+ backspace() {
+ this.keyboardBuffer = this.keyboardBuffer.slice(0, -1);
+ this.updateDisplay();
+ this.sendToROS('', false);
+ }
+
+ clearBuffer() {
+ this.keyboardBuffer = '';
+ this.updateDisplay();
+ this.sendToROS('[CLEAR]', false);
+ }
+
+ sendInput() {
+ if (this.keyboardBuffer.trim()) {
+ this.sendToROS('[SEND]', true);
+ this.keyboardBuffer = '';
+ this.updateDisplay();
+ }
+ }
+
+ handlePhysicalKey(e) {
+ if (e.target !== document.body) return;
+
+ const char = e.key.toUpperCase();
+
+ if (e.key === 'Backspace') {
+ e.preventDefault();
+ this.backspace();
+ } else if (e.key === 'Enter') {
+ e.preventDefault();
+ this.sendInput();
+ } else if (char.match(/^[A-Z0-9 .,!?]$/)) {
+ this.inputCharacter(char);
+ }
+ }
+
+ updateDisplay() {
+ this.keyboardBufferEl.textContent = this.keyboardBuffer || '(empty)';
+ this.renderTranscriptHistory();
+ }
+
+ renderTranscriptHistory() {
+ if (!this.displayData.history) return;
+
+ let html = '';
+ this.displayData.history.forEach(entry => {
+ const cls = entry.type === 'stt' ? 'transcript-stt' : 'transcript-keyboard';
+ const icon = entry.type === 'stt' ? '🎤' : '⌨️';
+ const text = entry.text || '';
+ html += `${icon} ${this.escapeHtml(text)}
`;
+ });
+
+ if (!html) {
+ html = 'Waiting for speech...
';
+ }
+
+ this.transcriptDisplay.innerHTML = html;
+ // Auto-scroll to bottom
+ this.transcriptDisplay.scrollTop = this.transcriptDisplay.scrollHeight;
+ }
+
+ sendToROS(command, isFinal) {
+ // This will be called when we have a WebSocket connection to the ROS2 bridge
+ const data = {
+ command: command,
+ buffer: this.keyboardBuffer,
+ timestamp: Date.now(),
+ final: isFinal
+ };
+
+ console.log('Sending to ROS:', data);
+
+ // Send via fetch API to UI server
+ fetch('/api/keyboard', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(data)
+ }).catch(err => console.error('ROS send error:', err));
+ }
+
+ startPolling() {
+ // Poll for display updates from ROS2
+ setInterval(() => this.pollDisplayUpdate(), 500);
+ }
+
+ pollDisplayUpdate() {
+ fetch('/api/display')
+ .then(res => res.json())
+ .then(data => {
+ if (data && data.history) {
+ this.displayData = data;
+ this.renderTranscriptHistory();
+ this.updateStatusIndicators();
+ }
+ })
+ .catch(err => console.error('Display poll error:', err));
+ }
+
+ updateStatusIndicators() {
+ // Update STT/TTS status based on display data
+ if (this.displayData.history && this.displayData.history.length > 0) {
+ const lastEntry = this.displayData.history[this.displayData.history.length - 1];
+ if (lastEntry.type === 'stt') {
+ this.sttStatusEl.textContent = lastEntry.final ? 'Complete' : 'Listening...';
+ }
+ }
+ }
+
+ escapeHtml(text) {
+ const div = document.createElement('div');
+ div.textContent = text;
+ return div.innerHTML;
+ }
+}
+
+// Initialize when DOM is ready
+document.addEventListener('DOMContentLoaded', () => {
+ window.accessibilityUI = new AccessibilityUI();
+});