voice_loop.py
Rama master · réplica de lectura del repositorio original.
#!/usr/bin/env python3
"""Async event loop orchestrating Rex's wake-word, STT, LLM and TTS pipeline.
.. deprecated::
This root-level file is kept only for ``AsyncRexAssistant`` backward-compat.
New code should use ``rex.voice_loop`` instead.
RELATIONSHIP NOTE — two voice_loop files exist in this repo:
- ``voice_loop.py`` (this file, repo root): legacy implementation containing
``AsyncRexAssistant``. Kept for backward compatibility with external code
that imports ``AsyncRexAssistant`` from here. Not the active startup path.
- ``rex/voice_loop.py`` (package): canonical implementation. ``rex_loop.py``
imports ``build_voice_loop`` from ``rex.voice_loop`` — that is the
authoritative voice loop executed when Rex starts.
"""
# Compatibility shim. Canonical implementation: rex.voice_loop. Scheduled for removal — see SURFACE-CLASSIFICATION.md.
# ruff: noqa: E402, I001
from __future__ import annotations
import warnings as _warnings
_warnings.warn(
"Importing from root-level 'voice_loop' is deprecated. "
"Use 'from rex.voice_loop import ...' instead. "
"This file will be removed in a future release.",
DeprecationWarning,
stacklevel=2,
)
import asyncio
import contextlib
import os
import platform
import sys
import tempfile
from dataclasses import dataclass
from importlib import import_module
from importlib.util import find_spec
from pathlib import Path
def _import_optional(module_name: str):
module = sys.modules.get(module_name)
if module is not None:
return module
if find_spec(module_name) is None:
return None
return import_module(module_name)
def _lazy_import_numpy():
return _import_optional("numpy")
def _lazy_import_soundfile():
return _import_optional("soundfile")
def _lazy_import_whisper():
return _import_optional("whisper")
def _lazy_import_tts():
# Check TTS availability without triggering internal imports that need
# BeamSearchScorer. The shim MUST run before TTS.api is imported.
if find_spec("TTS") is None:
return None
from rex.compat import ensure_transformers_compatibility
ensure_transformers_compatibility()
tts_api = import_module("TTS.api")
return tts_api.TTS
def _lazy_import_simpleaudio():
return _import_optional("simpleaudio")
def _load_sounddevice():
global sd
if sd is not None:
return sd
sd = _import_optional("sounddevice")
return sd
def _require_sounddevice():
module = _load_sounddevice()
if module is None:
raise WakeWordError("sounddevice is required for audio capture")
return module
def _require_numpy():
if np is None:
raise WakeWordError("numpy is required for wake word detection")
return np
np = _lazy_import_numpy()
sd = None
sa = _lazy_import_simpleaudio()
from rex.assistant_errors import (
SpeechToTextError,
TextToSpeechError,
WakeWordError,
)
from rex.config import AppConfig, load_config
from rex.llm_client import LanguageModel
from rex.logging_utils import get_logger
from rex.memory_utils import (
append_history_entry,
export_transcript,
extract_voice_reference,
load_all_profiles,
load_users_map,
resolve_user_key,
)
from rex.plugins import PluginSpec as _PluginSpec
from rex.plugins import load_plugins as _load_plugins_impl
from rex.tts_utils import chunk_text_for_xtts
from rex.wakeword_utils import detect_wakeword, load_wakeword_model
from rex.wake_acknowledgment import ensure_wake_acknowledgment_sound
from utils.audio_device import (
enumerate_input_devices,
load_audio_config,
resolve_audio_device,
)
logger = get_logger(__name__)
@dataclass
class WakeWordListener:
model: object
threshold: float
sample_rate: int
block_size: int
device: int | None
loop: asyncio.AbstractEventLoop | None = None
def __post_init__(self) -> None:
self._event = asyncio.Event()
self._stream: sd.InputStream | None = None
self._callback_count = 0 # Track number of audio callbacks received
def start(self) -> None:
"""Start audio input stream with robust Windows-friendly fallback strategy."""
sounddevice = _require_sounddevice()
# Get device info for logging and error messages
device_name = "default"
device_hostapi = "default"
if self.device is not None:
try:
devices = enumerate_input_devices()
device_obj = next((d for d in devices if d.index == self.device), None)
if device_obj:
device_name = device_obj.name
device_hostapi = device_obj.hostapi_name
else:
device_name = f"device_{self.device}"
except Exception:
device_name = f"device_{self.device}"
# Get device default samplerate if available
device_default_sr = None
if self.device is not None:
try:
info = sounddevice.query_devices(self.device)
if isinstance(info, dict):
device_default_sr = int(info.get("default_samplerate", 0))
except Exception:
pass
# Define retry strategies: (samplerate, blocksize, latency)
# Try different combinations for Windows WASAPI compatibility
retry_strategies = []
# Start with requested configuration
retry_strategies.append((self.sample_rate, self.block_size, None))
# Try with device's default samplerate
if device_default_sr and device_default_sr != self.sample_rate:
retry_strategies.append((device_default_sr, self.block_size, None))
# Try common samplerates
for sr in [16000, 44100, 48000]:
if sr != self.sample_rate and (not device_default_sr or sr != device_default_sr):
retry_strategies.append((sr, self.block_size, None))
# Try with different blocksizes
retry_strategies.append((self.sample_rate, None, None))
retry_strategies.append((self.sample_rate, 1024, None))
retry_strategies.append((self.sample_rate, 2048, None))
# Try with high latency
retry_strategies.append((self.sample_rate, self.block_size, "high"))
# Attempt to start stream with retry strategies
last_exception = None
attempt_details = []
for attempt_num, (samplerate, blocksize, latency) in enumerate(retry_strategies, 1):
try:
logger.info(
f"Attempt {attempt_num}: Using input device index: {self.device} "
f"name: {device_name} hostapi: {device_hostapi} "
f"samplerate: {samplerate} channels: 1 dtype: float32 "
f"blocksize: {blocksize} latency: {latency or 'default'}"
)
stream_kwargs = {
"channels": 1,
"samplerate": samplerate,
"callback": self._callback,
"device": self.device,
"dtype": "float32",
}
if blocksize is not None:
stream_kwargs["blocksize"] = blocksize
if latency is not None:
stream_kwargs["latency"] = latency
self._stream = sounddevice.InputStream(**stream_kwargs)
self._stream.start()
# Success!
logger.info(
f"Wake-word listener started successfully on device {self.device} ({device_name}) at {samplerate} Hz"
)
# Update sample_rate if we had to use a different one
if samplerate != self.sample_rate:
logger.warning(
f"Using samplerate {samplerate} instead of requested {self.sample_rate}"
)
self.sample_rate = samplerate
return
except Exception as exc:
last_exception = exc
attempt_details.append(
f" Attempt {attempt_num} [sr={samplerate}, bs={blocksize}, lat={latency or 'default'}]: {type(exc).__name__}: {exc}"
)
logger.debug(f"Stream start attempt {attempt_num} failed: {exc}")
# Close stream if it was partially created
if self._stream is not None:
try:
self._stream.close()
except Exception:
pass
self._stream = None
# All attempts failed - build comprehensive error message
error_msg = f"Failed to start input stream on device {self.device} ({device_name}) [{device_hostapi}]\n"
error_msg += f"Tried {len(retry_strategies)} different configurations:\n"
error_msg += "\n".join(attempt_details)
error_msg += f"\n\nFinal error: {last_exception}"
# Add helpful suggestions for common Windows DirectSound errors
error_str = str(last_exception).lower()
if "directsound" in error_str and "-2005401480" in error_str:
error_msg += "\n\n💡 DirectSound Exclusive Access Error - Try these fixes:"
error_msg += (
"\n 1. Close any apps using the microphone (Skype, Teams, Discord, Zoom, OBS)"
)
error_msg += "\n 2. Disable exclusive mode: Sound settings → Recording → Device Properties → Advanced → Uncheck 'Allow exclusive control'"
error_msg += "\n 3. Look for a WASAPI version of this device in the device dropdown (more reliable than DirectSound)"
error_msg += "\n 4. Try a different audio device"
elif "directsound" in error_str or "wasapi" in error_str:
error_msg += "\n\n💡 Windows Audio Error - Suggestions:"
error_msg += "\n 1. Close other audio applications"
error_msg += "\n 2. Try selecting a different device"
error_msg += "\n 3. Check Windows Sound settings → Recording tab"
raise WakeWordError(error_msg)
def stop(self) -> None:
if self._stream is not None:
with contextlib.suppress(Exception):
self._stream.stop()
self._stream.close()
logger.info("Wake-word listener stopped")
self._event.set()
def pause_stream(self) -> None:
"""Pause the audio input stream without destroying listener state.
Call this before recording or playback so that the wake-word
microphone does not compete with sounddevice.rec() or playback,
and so that TTS audio cannot spuriously trigger the wake word.
"""
if self._stream is not None:
with contextlib.suppress(Exception):
self._stream.stop()
logger.debug("Wake-word listener stream paused")
def resume_stream(self) -> None:
"""Resume the audio input stream and discard accumulated events.
Call this after the full interaction (playback) completes so that
the listener re-arms for the next wake word within 1 second.
Any wake-word events that fired during the pause (e.g. TTS audio
picked up by the mic) are discarded by clearing the event.
"""
if self._stream is not None:
with contextlib.suppress(Exception):
self._stream.start()
# Discard spurious detections that accumulated while paused.
self._event.clear()
logger.debug("Wake-word listener stream resumed and event cleared")
def trigger(self) -> None:
self.loop.call_soon_threadsafe(self._event.set)
def _callback(self, indata, frames, time_info, status) -> None:
self._callback_count += 1
np = _require_numpy()
# Log every 100 callbacks (~5-10 seconds) to show we're receiving audio
if self._callback_count % 100 == 1:
audio_level = np.max(np.abs(indata)) if indata.size > 0 else 0.0
logger.info(
f"Audio callback #{self._callback_count}: receiving audio (level: {audio_level:.3f})"
)
if status:
logger.warning("Audio status: %s", status)
audio_data = np.squeeze(indata)
# TEMPORARY DEBUG: Log before calling detect_wakeword
if self._callback_count % 100 == 1:
logger.info(
f">>> ABOUT TO CALL detect_wakeword (callback #{self._callback_count}, threshold={self.threshold})"
)
try:
result = detect_wakeword(self.model, audio_data, threshold=self.threshold)
# TEMPORARY DEBUG: Log after calling detect_wakeword
if self._callback_count % 100 == 1:
logger.info(f"<<< detect_wakeword RETURNED: {result}")
if bool(result):
logger.info(
"!!! WAKE WORD DETECTED IN CALLBACK - calling loop.call_soon_threadsafe to set event"
)
if self.loop is None:
logger.error("!!! ERROR: loop is None - cannot schedule event.set()!")
else:
self.loop.call_soon_threadsafe(self._event.set)
logger.info(f"!!! Event.set() scheduled on event loop: {self.loop}")
except Exception as exc:
logger.error("Wake-word detection failed: %s", exc)
import traceback
logger.error(f"Traceback: {traceback.format_exc()}")
async def wait_for_wake(self) -> None:
logger.info(">>> wait_for_wake() called - waiting for event...")
await self._event.wait()
logger.info("<<< wait_for_wake() - event received! Clearing event...")
self._event.clear()
logger.info("<<< wait_for_wake() - event cleared, returning")
class AsyncRexAssistant:
def __init__(self, config: AppConfig | None = None) -> None:
self.config = config or load_config()
self.language_model = LanguageModel(self.config)
# Also create a higher-level Assistant for tool-routed responses.
try:
from rex.assistant import Assistant
from rex.identity import resolve_entrypoint_user_id
# Deliberate single-user profile selection (issue #303):
# Assistant no longer invents an identity when user_id is omitted.
self._assistant = Assistant(
settings_obj=self.config,
user_id=resolve_entrypoint_user_id(self.config),
)
except Exception as exc:
logger.warning("Failed to create Assistant (tool routing unavailable): %s", exc)
self._assistant = None
if self.config.use_openclaw_voice_backend:
try:
from rex.openclaw.voice_bridge import VoiceBridge
self._assistant = VoiceBridge()
logger.info("Voice loop using OpenClaw VoiceBridge backend")
except Exception as exc:
logger.warning(
"Failed to create VoiceBridge (falling back to default assistant): %s", exc
)
self._wake_sound_path: str | None = None
try:
self._wake_sound_path = ensure_wake_acknowledgment_sound(
path=self.config.wake_sound_path
)
except Exception as exc:
logger.warning("Failed to generate wake acknowledgment sound: %s", exc)
self._wake_model, self._wake_keyword = load_wakeword_model(
keyword=self.config.wakeword_keyword,
backend=self.config.wakeword_backend,
model_path=self.config.wakeword_model_path,
embedding_path=self.config.wakeword_embedding_path,
fallback_to_builtin=self.config.wakeword_fallback_to_builtin,
fallback_keyword=self.config.wakeword_fallback_keyword,
)
self._sample_rate = 16000
# Load audio config and resolve device with validation
audio_config = load_audio_config()
if self.config.audio_input_device is not None:
configured_device = self.config.audio_input_device
else:
configured_device = audio_config.get("input_device_index")
resolved_device, status_msg = resolve_audio_device(configured_device, self._sample_rate)
if resolved_device is None:
logger.warning(f"Audio device resolution failed: {status_msg}")
logger.warning("Will attempt to use default device (None)")
else:
logger.info(status_msg)
# Bind the listener to the actual running loop in run(), where that loop exists.
self._listener = WakeWordListener(
model=self._wake_model,
threshold=self.config.wakeword_threshold,
sample_rate=self._sample_rate,
block_size=int(self._sample_rate * self.config.wakeword_window),
device=resolved_device,
loop=None, # Will be set to the running loop in run()
)
self._tts: object | None = None
self._whisper_model: object | None = None
self.users_map = load_users_map()
self.profiles = load_all_profiles()
resolved = resolve_user_key(
self.config.default_user, self.users_map, profiles=self.profiles
)
self.active_user = resolved or (self.config.default_user or "james")
self.user_voice_refs = {
user: extract_voice_reference(profile, user_key=user)
for user, profile in self.profiles.items()
}
_plugin_specs: list[_PluginSpec] = _load_plugins_impl()
self.plugins: dict[str, _PluginSpec] = {f"plugins.{s.name}": s for s in _plugin_specs}
logger.info("Loaded plugins: %s", ", ".join(self.plugins.keys()) or "none")
# Register plugins as LLM tools (for function calling)
self._register_plugins_as_tools()
self._running = True
self._state = "initialized"
self._stop_requested_by: str | None = None
def _get_tts(self) -> object:
if self._tts is None:
tts_class = _lazy_import_tts()
if tts_class is None:
raise TextToSpeechError("TTS is not installed")
import torch
gpu_available = torch.cuda.is_available()
logger.info(
f"Loading TTS model (XTTS v2) on {'GPU' if gpu_available else 'CPU (WARNING: GPU not available!)'}..."
)
self._tts = tts_class(
model_name="tts_models/multilingual/multi-dataset/xtts_v2",
progress_bar=False,
gpu=gpu_available, # Enable GPU if available, fallback to CPU
)
# Verify the model is actually on GPU
if gpu_available:
try:
# Try to move model to GPU explicitly
if hasattr(self._tts, "synthesizer") and hasattr(
self._tts.synthesizer, "tts_model"
):
device = next(self._tts.synthesizer.tts_model.parameters()).device
logger.info(f"TTS model device: {device}")
if device.type != "cuda":
logger.warning(
f"⚠️ TTS model is on {device.type}, not CUDA! Attempting to move to GPU..."
)
self._tts.synthesizer.tts_model = self._tts.synthesizer.tts_model.cuda()
logger.info("✓ Moved TTS model to GPU")
except Exception as e:
logger.warning(f"Could not verify TTS device: {e}")
logger.info("TTS model loaded successfully")
return self._tts
def _get_whisper(self) -> object:
if self._whisper_model is None:
whisper_module = _lazy_import_whisper()
if whisper_module is None:
raise SpeechToTextError("openai-whisper is not installed")
# Resolve STT device: "auto" selects CUDA when available.
device = getattr(self.config, "whisper_device", "cpu") or "cpu"
if device == "auto":
try:
import torch
device = "cuda" if torch.cuda.is_available() else "cpu"
except ImportError:
device = "cpu"
logger.info(
"[STT] Loading Whisper '%s' on device '%s'", self.config.whisper_model, device
)
try:
self._whisper_model = whisper_module.load_model(
self.config.whisper_model, device=device
)
except Exception as exc:
raise SpeechToTextError(f"Failed to load Whisper model: {exc}")
return self._whisper_model
def _register_plugins_as_tools(self) -> None:
"""Register loaded plugins as LLM tools for function calling."""
# Only register tools if using OpenAI-compatible provider
if self.config.llm_provider.lower() not in ["openai", "ollama"]:
logger.info(
"LLM provider '%s' does not support function calling; skipping tool registration",
self.config.llm_provider,
)
return
# Register web_search plugin if available
if "plugins.web_search" in self.plugins:
try:
from plugins.web_search import search_web
self.language_model.register_tool(
name="web_search",
description="Search the web for current information, news, facts, weather, or anything requiring up-to-date knowledge. Use this when the user asks about current events, recent information, or anything you don't have knowledge about.",
parameters={
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "The search query to look up on the web",
}
},
"required": ["query"],
},
function=search_web,
)
logger.info("Registered web_search tool for LLM function calling")
except Exception as exc:
logger.warning("Failed to register web_search tool: %s", exc)
async def run(self) -> None:
if self._state in {"starting", "running"}:
logger.warning(
"Assistant start requested while already %s; ignoring duplicate start.", self._state
)
return
self._state = "starting"
self._running = True
# Log wake word details right before starting (this is in background thread so GUI will see it)
logger.info(
f"Starting wake word listener for keyword: '{self._wake_keyword}' (threshold: {self.config.wakeword_threshold})"
)
logger.info(f"Wake word model type: {type(self._wake_model).__name__}")
# CRITICAL: Get the running event loop and update listener to use it
# This ensures the audio callback schedules events on the correct loop
running_loop = asyncio.get_running_loop()
self._listener.loop = running_loop
logger.info(f"Updated listener to use running event loop: {running_loop}")
# Pre-load models in background thread for faster first response
logger.info("Pre-loading Whisper and TTS models in background...")
try:
await asyncio.to_thread(self._get_whisper) # Pre-load Whisper
await asyncio.to_thread(self._get_tts) # Pre-load TTS on GPU
logger.info("Models pre-loaded successfully")
except Exception:
logger.exception("Model preloading failed with an exception")
self._running = False
self._state = "stopped"
return
if not self._running:
logger.info(
"Stop requested before wake listener start; aborting startup (requested_by=%s).",
self._stop_requested_by or "unknown",
)
self._state = "stopped"
return
self._listener.start()
self._state = "running"
try:
logger.info(">>> Entering main run loop - waiting for wake word...")
while self._running:
logger.info(">>> Loop iteration - calling wait_for_wake()...")
await self._listener.wait_for_wake()
logger.info(">>> wait_for_wake() returned!")
if not self._running:
logger.info(">>> _running is False, breaking loop")
break
logger.info("Wake word '%s' detected", self._wake_keyword)
logger.info(">>> Calling _handle_interaction()...")
await self._handle_interaction()
logger.info(">>> _handle_interaction() completed")
finally:
self._listener.stop()
self._state = "stopped"
async def _handle_interaction(self) -> None:
import time
start_time = time.time()
logger.info(
">>> _handle_interaction() started - playing wake sound and processing conversation..."
)
# Pause the wake-word mic stream so it does not compete with
# sounddevice.rec() during recording and cannot pick up TTS audio.
self._listener.pause_stream()
try:
# Play wake sound first, then record (don't overlap to avoid audio device conflicts)
await asyncio.to_thread(self._play_wake_sound)
await self._process_conversation()
total_time = time.time() - start_time
logger.info(
f">>> _handle_interaction() completed successfully in {total_time:.2f} seconds"
)
except Exception as exc:
logger.error(f"!!! _handle_interaction() FAILED: {exc}")
import traceback
logger.error(f"Traceback: {traceback.format_exc()}")
finally:
# Re-arm the wake-word listener immediately after TTS playback
# completes (or on error). resume_stream() restarts the mic
# stream and clears any spurious events — the next
# wait_for_wake() call will block until a real wake word fires.
self._listener.resume_stream()
logger.info("Wake-word listener re-armed after interaction")
async def _process_conversation(self) -> None:
np = _require_numpy()
logger.info(">>> _process_conversation() started - recording audio...")
audio = await asyncio.to_thread(self._record_audio)
logger.info(f">>> Audio recording complete - {audio.size} samples")
if audio.size:
min_val = float(np.min(audio))
max_val = float(np.max(audio))
else:
min_val = max_val = 0.0
logger.info(
"Audio shape: %s, dtype: %s, min: %.4f, max: %.4f",
audio.shape,
audio.dtype,
min_val,
max_val,
)
try:
transcript = await self.transcribe(audio, self._sample_rate)
except SpeechToTextError as exc:
logger.error("Transcription failed: %s", exc)
return
if not transcript:
logger.info("No speech detected after wake word")
return
if self._maybe_switch_user(transcript):
logger.info("Switched active user to %s", self.active_user)
return
logger.info("User (%s): %s", self.active_user, transcript)
# Offload synchronous file I/O to a thread so the event loop is not blocked.
await asyncio.to_thread(
append_history_entry,
self.active_user,
{"role": "user", "text": transcript},
)
import time as _time
# Use the Assistant (with tool routing + system context) when available,
# falling back to the bare LanguageModel otherwise.
try:
if self._assistant is not None:
response = await self._assistant.generate_reply(transcript, voice_mode=True)
else:
response = await asyncio.to_thread(self.language_model.generate, transcript)
except Exception as exc:
logger.error("LLM generation failed: %s", exc, exc_info=True)
return
logger.debug(
"[PIPELINE] stage=llm_response_received ts=%.3f text_len=%d",
_time.time(),
len(response),
)
if not response or not response.strip():
logger.warning("LLM returned empty or whitespace-only response; skipping TTS")
return
await asyncio.to_thread(
append_history_entry,
self.active_user,
{"role": "assistant", "text": response},
)
if self.config.conversation_export:
await asyncio.to_thread(
export_transcript,
self.active_user,
[
{"role": "user", "text": transcript},
{"role": "assistant", "text": response},
],
)
try:
await asyncio.to_thread(self._speak_response, response)
except TextToSpeechError as exc:
logger.error("TTS failed: %s", exc, exc_info=True)
def _record_audio(self) -> np.ndarray:
np = _require_numpy()
sounddevice = _require_sounddevice()
soundfile = _lazy_import_soundfile()
sample_rate = self._sample_rate
duration = self.config.command_duration
frames = int(sample_rate * duration)
logger.info("Recording %.1f seconds of audio", duration)
# Use the same device as the listener
device = self._listener.device
audio = sounddevice.rec(
frames,
samplerate=sample_rate,
channels=1,
dtype="float32",
device=device,
)
sounddevice.wait()
audio = np.squeeze(audio)
peak = float(np.max(np.abs(audio))) if audio.size else 0.0
logger.debug(
"[MIC] Recorded %s samples, %.2fs, peak volume %.4f", audio.size, duration, peak
)
debug_output = Path("debug_recording.wav")
if soundfile is None:
logger.warning("[MIC] soundfile not available; skipping debug recording write")
else:
try:
soundfile.write(debug_output, audio, sample_rate)
logger.debug("[MIC] Saved debug audio to %s", debug_output)
except Exception as exc:
logger.warning("[MIC] Failed to save debug recording: %s", exc)
return audio
async def transcribe(self, audio: np.ndarray, sample_rate: int) -> str:
np = _require_numpy()
self._get_whisper()
def _transcribe() -> str:
processed = audio.astype(np.float32) if audio.dtype != np.float32 else audio
if processed.size:
processed = np.clip(processed, -1.0, 1.0)
result = self._whisper_model.transcribe(processed, language="en", fp16=False)
logger.debug(f"[STT] Raw Whisper result: {result}")
return str(result.get("text", "")).strip()
try:
return await asyncio.to_thread(_transcribe)
except Exception as exc:
logger.exception("[STT] Whisper failed:")
raise SpeechToTextError(str(exc)) from exc
def _speak_response(self, text: str) -> None:
import time
np = _require_numpy()
soundfile = _lazy_import_soundfile()
if soundfile is None:
raise TextToSpeechError("soundfile is required for TTS playback")
tts = self._get_tts()
speaker_wav = self.user_voice_refs.get(self.active_user)
tts_speed = getattr(self.config, "tts_speed", 1.08)
try:
# XTTS v2 requires either speaker_wav or speaker parameter
# If no voice reference, use built-in speaker
logger.info(f"[TTS] Generating speech for {len(text)} characters...")
tts_start = time.time()
chunks = chunk_text_for_xtts(text, max_tokens=300)
if not chunks:
return
logger.debug(
"[PIPELINE] stage=tts_input_prepared ts=%.3f text_len=%d chunks=%d",
time.time(),
len(text),
len(chunks),
)
audio_segments = []
sample_rate = None
total_chunks = len(chunks)
for chunk_idx, chunk in enumerate(chunks):
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
chunk_path = tmp.name
try:
logger.debug(
"[PIPELINE] stage=tts_engine_called ts=%.3f chunk=%d/%d chunk_len=%d",
time.time(),
chunk_idx + 1,
total_chunks,
len(chunk),
)
if speaker_wav:
tts.tts_to_file(
text=chunk,
speaker_wav=speaker_wav,
language="en",
file_path=chunk_path,
speed=tts_speed,
)
else:
# Use built-in XTTS speaker when no voice reference is available
tts.tts_to_file(
text=chunk,
speaker="Claribel Dervla", # Default XTTS v2 speaker
language="en",
file_path=chunk_path,
speed=tts_speed,
)
data, rate = soundfile.read(chunk_path, dtype="float32")
logger.debug(
"[PIPELINE] stage=audio_data_returned ts=%.3f chunk=%d/%d samples=%d",
time.time(),
chunk_idx + 1,
total_chunks,
len(data),
)
if sample_rate is None:
sample_rate = rate
audio_segments.append(data)
finally:
try:
os.unlink(chunk_path)
except (OSError, PermissionError) as exc:
logger.warning("Failed to remove temp file %s: %s", chunk_path, exc)
if not audio_segments or sample_rate is None:
return
combined_audio = np.concatenate(audio_segments, axis=0)
soundfile.write("assistant_response.wav", combined_audio, sample_rate)
tts_duration = time.time() - tts_start
logger.info(f"[TTS] Speech generated in {tts_duration:.2f} seconds")
except Exception as exc:
raise TextToSpeechError(f"Failed to synthesise speech: {exc}") from exc
# Playback section — separate from synthesis so errors are clearly attributed
logger.info("[TTS] Playing audio...")
playback_start = time.time()
logger.debug(
"[PIPELINE] stage=audio_playback_initiated ts=%.3f total_samples=%d sample_rate=%d",
time.time(),
len(combined_audio),
sample_rate,
)
try:
output_device = self.config.audio_output_device
sounddevice_mod = _load_sounddevice()
if sounddevice_mod is not None:
# sounddevice supports device selection by index or name
sounddevice_mod.play(
combined_audio, sample_rate, device=output_device, blocking=True
)
sounddevice_mod.stop()
elif platform.system() == "Windows":
import winsound
winsound.PlaySound("assistant_response.wav", winsound.SND_FILENAME)
# Give Windows time to release audio device before next recording
time.sleep(0.5)
elif sa is not None:
wave_obj = sa.WaveObject.from_wave_file("assistant_response.wav")
play_obj = wave_obj.play()
play_obj.wait_done()
else:
logger.warning("No audio playback library available - audio saved but not played")
except Exception as exc:
logger.error("Audio playback failed: %s", exc, exc_info=True)
raise TextToSpeechError(f"Audio playback failed: {exc}") from exc
finally:
try:
os.unlink("assistant_response.wav")
except (OSError, PermissionError) as exc:
logger.warning("Failed to remove temp file %s: %s", "assistant_response.wav", exc)
playback_duration = time.time() - playback_start
logger.info(f"[TTS] Audio playback completed in {playback_duration:.2f} seconds")
logger.debug(
"[PIPELINE] stage=audio_playback_completed ts=%.3f duration_s=%.2f",
time.time(),
playback_duration,
)
def _play_wake_sound(self) -> None:
path = self._wake_sound_path or self.config.wake_sound_path
try:
path = ensure_wake_acknowledgment_sound(path=path)
except Exception as exc:
logger.warning("Wake acknowledgment sound unavailable: %s", exc)
return
# Check if file exists
if not Path(path).exists():
logger.warning("Wake sound file not found after ensure: %s", path)
return
# Use platform-specific playback for reliability
if platform.system() == "Windows":
self._play_wake_sound_windows(path)
else:
self._play_wake_sound_sounddevice(path)
def _play_wake_sound_windows(self, path: str | Path) -> None:
"""Play wake sound on Windows using winsound (more reliable than sounddevice)."""
winsound = import_module("winsound") if find_spec("winsound") is not None else None
if winsound is None:
logger.warning("winsound not available; falling back to sounddevice")
self._play_wake_sound_sounddevice(path)
return
try:
logger.debug(f"Playing wake sound (Windows): {path}")
# First, stop any currently playing sound
winsound.PlaySound(None, winsound.SND_PURGE)
# Play the wake sound synchronously (blocks until complete)
# SND_FILENAME: path is a file
# SND_NODEFAULT: no default beep if file missing
# SND_NOSTOP: don't interrupt if another sound is playing (we already purged)
winsound.PlaySound(str(path), winsound.SND_FILENAME | winsound.SND_NODEFAULT)
# Ensure playback is fully stopped
winsound.PlaySound(None, winsound.SND_PURGE)
logger.info("Wake acknowledgment tone played (Windows)")
except Exception as exc:
logger.warning(
"Failed to play wake sound with winsound: %s, falling back to sounddevice", exc
)
self._play_wake_sound_sounddevice(path)
def _play_wake_sound_sounddevice(self, path: str | Path) -> None:
"""Play wake sound using sounddevice (cross-platform fallback)."""
sounddevice = _require_sounddevice()
soundfile = _lazy_import_soundfile()
if soundfile is None:
logger.warning("soundfile is required to play wake sounds")
return
try:
# Read audio file
data, samplerate = soundfile.read(path)
logger.debug(f"Playing wake sound: {path} ({len(data)} samples at {samplerate} Hz)")
# Forcefully abort any existing streams before playing
try:
sounddevice.stop()
except Exception:
pass
# Use blocking playback to ensure proper cleanup
device = self.config.audio_output_device
sounddevice.play(data, samplerate, device=device, blocking=True)
# Ensure stream is fully stopped
sounddevice.stop()
logger.info("Wake acknowledgment tone played")
except Exception as exc:
logger.warning("Failed to play wake sound %s: %s", path, exc)
# Forcefully stop any stuck playback
try:
sounddevice.stop()
except Exception:
pass
def _maybe_switch_user(self, transcript: str) -> bool:
cleaned = transcript.strip().lower()
if not cleaned.startswith("this is "):
return False
candidate = cleaned.replace("this is ", "", 1).strip().strip(".")
resolved = resolve_user_key(candidate, self.users_map, profiles=self.profiles)
if resolved:
self.active_user = resolved
return True
return False
def stop(self) -> None:
self.stop_requested_by("unknown")
def stop_requested_by(self, requested_by: str) -> None:
self._stop_requested_by = requested_by
logger.info(
"Stop requested by %s (state=%s, running=%s)", requested_by, self._state, self._running
)
self._running = False
self._listener.trigger()
def build_voice_loop(assistant: AsyncRexAssistant | None = None) -> AsyncRexAssistant:
"""Helper function for rex_loop.py to build and run the voice assistant loop."""
return assistant or AsyncRexAssistant()