seb/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(whatsapp): add WhatsApp Business Cloud API adapter

Browse Source 
Add an official, production-grade WhatsApp integration via Meta's
Business Cloud API as a complement to the existing Baileys bridge.
No bridge subprocess, no QR codes, no account-ban risk — at the cost
of a Meta Business account and a public HTTPS webhook URL.

Setup is fully wizard-driven: 'hermes whatsapp-cloud' walks through
every credential with paste-time validation (catches the #1 trap of
pasting a phone number into the Phone Number ID field), generates a
verify token, and ends with copy-paste instructions for the
cloudflared / Meta-dashboard / Business Manager pieces that can't be
automated. The wizard also points users at Meta's Business Manager
for setting the bot's display name and profile picture.

Feature set:

- Inbound: text, images (with native-vision routing), voice notes
  (STT), documents (small text inlined, larger cached), reply context.
- Outbound: text with WhatsApp-flavored markdown conversion, images,
  videos, documents, opus voice notes via ffmpeg with MP3 fallback.
- Native interactive buttons for clarify, dangerous-command approval,
  and slash-command confirmation flows — matches the Telegram /
  Discord UX, graceful degrades to plain text.
- Read receipts (blue double-checkmarks) and typing indicator,
  using Meta's combined endpoint so they fire in a single API call.
- Webhook security: X-Hub-Signature-256 HMAC verification (raw body,
  constant-time), wamid deduplication, group-shaped-message refusal
  (groups deferred to v2 — Baileys still covers them).
- Full integration with the gateway's session, cron, display-tier,
  prompt-hint, and auth-allowlist systems. Cloud and Baileys can run
  side-by-side against different phone numbers.

Also wires STT (speech-to-text) through Nous's managed audio gateway
for Nous subscribers — previously the default stt.provider=local
required a separate faster-whisper install. New subscribers now get
voice-note transcription out of the box.

Docs: 418-line user guide at website/docs/user-guide/messaging/
whatsapp-cloud.md, sidebar entry, environment-variables reference,
ADDING_A_PLATFORM.md updated with the optional interactive-UX
contract for future adapter authors.

Tests: 100 dedicated tests for the adapter, 32 for the setup wizard,
20 for the Nous subscription STT wiring, plus regression coverage
across display_config, prompt_builder, and the cron scheduler.

Known limitations (deferred until clear demand signal):
- Group chats — use the Baileys bridge if you need them.
- Message templates for 24-hour-window outside-conversation sends —
  reactive chat is unaffected; cron / delegate_task with gaps > 24h
  will fail with a clear error. The agent's system prompt warns the
  model about this so it knows to mention it when scheduling delayed
  messages.
This commit is contained in:
emozilla
2026-05-23 01:07:01 -04:00
parent a7cd254c29
commit 984e6cb5b8
26 changed files with 6368 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hermes_cli/nous_subscription.py
+130 -4
View File
@@ -66,6 +66,10 @@ class NousSubscriptionFeatures:
def tts(self) -> NousFeatureState:
return self.features["tts"]
@property
def stt(self) -> NousFeatureState:
return self.features["stt"]
@property
def browser(self) -> NousFeatureState:
return self.features["browser"]
@@ -75,7 +79,7 @@ class NousSubscriptionFeatures:
return self.features["modal"]
def items(self) -> Iterable[NousFeatureState]:
ordered = ("web", "image_gen", "tts", "browser", "modal")
ordered = ("web", "image_gen", "tts", "stt", "browser", "modal")
for key in ordered:
yield self.features[key]
@@ -159,6 +163,16 @@ def _tts_label(current_provider: str) -> str:
return mapping.get(current_provider or "edge", current_provider or "Edge TTS")
def _stt_label(current_provider: str) -> str:
mapping = {
"openai": "OpenAI Whisper",
"groq": "Groq Whisper",
"mistral": "Mistral Voxtral Transcribe",
"local": "Local faster-whisper",
}
return mapping.get(current_provider or "local", current_provider or "Local faster-whisper")
def _resolve_browser_feature_state(
*,
browser_tool_enabled: bool,
@@ -251,6 +265,7 @@ def get_nous_subscription_features(
web_cfg = config.get("web") if isinstance(config.get("web"), dict) else {}
tts_cfg = config.get("tts") if isinstance(config.get("tts"), dict) else {}
stt_cfg = config.get("stt") if isinstance(config.get("stt"), dict) else {}
browser_cfg = config.get("browser") if isinstance(config.get("browser"), dict) else {}
terminal_cfg = config.get("terminal") if isinstance(config.get("terminal"), dict) else {}
@@ -260,6 +275,11 @@ def get_nous_subscription_features(
web_search_backend = str(web_cfg.get("search_backend") or "").strip().lower()
web_extract_backend = str(web_cfg.get("extract_backend") or "").strip().lower()
tts_provider = str(tts_cfg.get("provider") or "edge").strip().lower()
# STT default is "local" (faster-whisper) per DEFAULT_CONFIG, which
# requires `pip install faster-whisper`. For Nous subscribers we'd
# rather route through the managed OpenAI audio gateway — see
# apply_nous_managed_defaults below.
stt_provider = str(stt_cfg.get("provider") or "local").strip().lower()
browser_provider_explicit = "cloud_provider" in browser_cfg
browser_provider = normalize_browser_cloud_provider(
browser_cfg.get("cloud_provider") if browser_provider_explicit else None
@@ -276,6 +296,7 @@ def get_nous_subscription_features(
# prevent gateway routing.
web_use_gateway = _uses_gateway(web_cfg)
tts_use_gateway = _uses_gateway(tts_cfg)
stt_use_gateway = _uses_gateway(stt_cfg)
browser_use_gateway = _uses_gateway(browser_cfg)
image_gen_cfg = config.get("image_gen") if isinstance(config.get("image_gen"), dict) else {}
image_use_gateway = _uses_gateway(image_gen_cfg)
@@ -293,6 +314,22 @@ def get_nous_subscription_features(
direct_browser_use = bool(get_env_value("BROWSER_USE_API_KEY"))
direct_modal = has_direct_modal_credentials()
# STT direct providers. OpenAI Whisper reuses the same audio key as
# OpenAI TTS — resolve_openai_audio_api_key() reads VOICE_TOOLS_OPENAI_KEY
# and falls back to OPENAI_API_KEY. The local provider's "direct"
# signal is whether faster-whisper is importable; we lazy-import so
# this module stays cheap on the happy path.
direct_openai_stt = bool(resolve_openai_audio_api_key())
direct_groq_stt = bool(get_env_value("GROQ_API_KEY"))
direct_mistral_stt = bool(get_env_value("MISTRAL_API_KEY"))
try:
from tools.transcription_tools import _HAS_FASTER_WHISPER
local_stt_available = bool(_HAS_FASTER_WHISPER) or bool(
get_env_value("HERMES_LOCAL_STT_COMMAND")
)
except Exception:
local_stt_available = bool(get_env_value("HERMES_LOCAL_STT_COMMAND"))
# When use_gateway is set, suppress direct credentials for managed detection
if web_use_gateway:
direct_firecrawl = False
@@ -304,6 +341,11 @@ def get_nous_subscription_features(
if tts_use_gateway:
direct_openai_tts = False
direct_elevenlabs = False
if stt_use_gateway:
direct_openai_stt = False
direct_groq_stt = False
direct_mistral_stt = False
local_stt_available = False
if browser_use_gateway:
direct_browser_use = False
direct_browserbase = False
@@ -311,6 +353,10 @@ def get_nous_subscription_features(
managed_web_available = managed_tools_flag and nous_auth_present and is_managed_tool_gateway_ready("firecrawl")
managed_image_available = managed_tools_flag and nous_auth_present and is_managed_tool_gateway_ready("fal-queue")
managed_tts_available = managed_tools_flag and nous_auth_present and is_managed_tool_gateway_ready("openai-audio")
# STT and TTS share the same managed gateway endpoint ("openai-audio")
# because the OpenAI audio API covers both /audio/speech (TTS) and
# /audio/transcriptions (STT). One probe, used by both.
managed_stt_available = managed_tts_available
managed_browser_available = managed_tools_flag and nous_auth_present and is_managed_tool_gateway_ready("browser-use")
managed_modal_available = managed_tools_flag and nous_auth_present and is_managed_tool_gateway_ready("modal")
modal_state = resolve_modal_backend_state(
@@ -361,6 +407,24 @@ def get_nous_subscription_features(
)
tts_active = bool(tts_tool_enabled and tts_available)
# STT availability per provider. Unlike TTS, STT isn't a model-callable
# tool — the gateway voice middleware calls it on every inbound voice
# message — so toolset_enabled is N/A and we treat stt as always
# "enabled" if a usable provider is configured.
stt_current_provider = stt_provider or "local"
stt_managed = (
stt_current_provider == "openai"
and managed_stt_available
and not direct_openai_stt
)
stt_available = bool(
(stt_current_provider == "local" and local_stt_available)
or (stt_current_provider == "openai" and (managed_stt_available or direct_openai_stt))
or (stt_current_provider == "groq" and direct_groq_stt)
or (stt_current_provider == "mistral" and direct_mistral_stt)
)
stt_active = stt_available
browser_local_available = _has_agent_browser()
(
browser_current_provider,
@@ -415,6 +479,13 @@ def get_nous_subscription_features(
if isinstance(raw_tts_cfg, dict) and "provider" in raw_tts_cfg:
tts_explicit_configured = tts_provider not in {"", "edge"}
# STT considers any non-default provider explicit. "local" is the
# DEFAULT_CONFIG seed, so seeing it doesn't mean the user picked it.
stt_explicit_configured = False
raw_stt_cfg = config.get("stt")
if isinstance(raw_stt_cfg, dict) and "provider" in raw_stt_cfg:
stt_explicit_configured = stt_provider not in {"", "local"}
features = {
"web": NousFeatureState(
key="web",
@@ -452,6 +523,21 @@ def get_nous_subscription_features(
current_provider=_tts_label(tts_current_provider),
explicit_configured=tts_explicit_configured,
),
"stt": NousFeatureState(
key="stt",
label="Speech-to-text",
included_by_default=True,
available=stt_available,
active=stt_active,
managed_by_nous=stt_managed,
direct_override=stt_active and not stt_managed,
# STT isn't toolset-gated (gateway middleware calls it
# unconditionally on inbound voice), so report True so the
# status display doesn't flag it as "tool disabled".
toolset_enabled=True,
current_provider=_stt_label(stt_current_provider),
explicit_configured=stt_explicit_configured,
),
"browser": NousFeatureState(
key="browser",
label="Browser automation",
@@ -514,6 +600,11 @@ def apply_nous_managed_defaults(
tts_cfg = {}
config["tts"] = tts_cfg
stt_cfg = config.get("stt")
if not isinstance(stt_cfg, dict):
stt_cfg = {}
config["stt"] = stt_cfg
browser_cfg = config.get("browser")
if not isinstance(browser_cfg, dict):
browser_cfg = {}
@@ -535,6 +626,18 @@ def apply_nous_managed_defaults(
tts_cfg["provider"] = "openai"
changed.add("tts")
# STT: same pattern as TTS. The DEFAULT_CONFIG seed is "local"
# (requires `pip install faster-whisper`); for Nous subscribers we
# flip it to "openai" so the managed audio gateway handles transcription
# via the same auth as TTS. Skipped when the user has explicitly
# configured STT or has direct credentials for a non-managed provider.
if not features.stt.explicit_configured and not (
get_env_value("GROQ_API_KEY")
or get_env_value("MISTRAL_API_KEY")
):
stt_cfg["provider"] = "openai"
changed.add("stt")
if "browser" in selected_toolsets and not features.browser.explicit_configured and not (
get_env_value("BROWSER_USE_API_KEY")
or get_env_value("BROWSERBASE_API_KEY")
@@ -556,6 +659,7 @@ _GATEWAY_TOOL_LABELS = {
"web": "Web search & extract (Firecrawl)",
"image_gen": "Image generation (FAL)",
"tts": "Text-to-speech (OpenAI TTS)",
"stt": "Speech-to-text (OpenAI Whisper)",
"browser": "Browser automation (Browser Use)",
}
@@ -575,6 +679,15 @@ def _get_gateway_direct_credentials() -> Dict[str, bool]:
resolve_openai_audio_api_key()
or get_env_value("ELEVENLABS_API_KEY")
),
# STT direct credentials. OpenAI Whisper shares the audio key
# with TTS via resolve_openai_audio_api_key() — counting it here
# too is intentional: if the user has an OpenAI audio key they
# don't need the gateway for either.
"stt": bool(
resolve_openai_audio_api_key()
or get_env_value("GROQ_API_KEY")
or get_env_value("MISTRAL_API_KEY")
),
"browser": bool(
get_env_value("BROWSER_USE_API_KEY")
or (get_env_value("BROWSERBASE_API_KEY") and get_env_value("BROWSERBASE_PROJECT_ID"))
@@ -586,10 +699,11 @@ _GATEWAY_DIRECT_LABELS = {
"web": "Firecrawl/Exa/Parallel/Tavily key",
"image_gen": "FAL key",
"tts": "OpenAI/ElevenLabs key",
"stt": "OpenAI/Groq/Mistral key",
"browser": "Browser Use/Browserbase key",
}
_ALL_GATEWAY_KEYS = ("web", "image_gen", "tts", "browser")
_ALL_GATEWAY_KEYS = ("web", "image_gen", "tts", "stt", "browser")
def get_gateway_eligible_tools(
@@ -625,6 +739,7 @@ def get_gateway_eligible_tools(
"web": _uses_gateway(config.get("web")),
"image_gen": _uses_gateway(config.get("image_gen")),
"tts": _uses_gateway(config.get("tts")),
"stt": _uses_gateway(config.get("stt")),
"browser": _uses_gateway(config.get("browser")),
}
@@ -664,6 +779,11 @@ def apply_gateway_defaults(
tts_cfg = {}
config["tts"] = tts_cfg
stt_cfg = config.get("stt")
if not isinstance(stt_cfg, dict):
stt_cfg = {}
config["stt"] = stt_cfg
browser_cfg = config.get("browser")
if not isinstance(browser_cfg, dict):
browser_cfg = {}
@@ -679,6 +799,11 @@ def apply_gateway_defaults(
tts_cfg["use_gateway"] = True
changed.add("tts")
if "stt" in tool_keys:
stt_cfg["provider"] = "openai"
stt_cfg["use_gateway"] = True
changed.add("stt")
if "browser" in tool_keys:
browser_cfg["cloud_provider"] = "browser-use"
browser_cfg["use_gateway"] = True
@@ -717,8 +842,9 @@ def prompt_enable_tool_gateway(config: Dict[str, object]) -> set[str]:
desc_parts: list[str] = [
"",
" The Tool Gateway gives you access to web search, image generation,",
" text-to-speech, and browser automation through your Nous subscription.",
" No need to sign up for separate API keys — just pick the tools you want.",
" text-to-speech, speech-to-text, and browser automation through your",
" Nous subscription. No need to sign up for separate API keys — just",
" pick the tools you want.",
"",
]
if already_managed: