Stage 4: honest health, explicit error policy, event-bus integrity #4

Merged
r merged 1 commits from feature/stage4-honest-health into feature/stage3-structure 2026-07-10 17:48:52 +00:00
21 changed files with 492 additions and 357 deletions

View File

@@ -12,6 +12,9 @@ DATABASE_URL=postgresql+asyncpg://holdslayer:<db-password>@localhost:5432/holdsl
API_TOKEN= API_TOKEN=
# --- SIP Trunk --- # --- SIP Trunk ---
# The mock engine must be requested explicitly; an unconfigured trunk
# without USE_MOCK_SIP=true refuses to start.
USE_MOCK_SIP=false
SIP_TRUNK_HOST=sip.yourprovider.com SIP_TRUNK_HOST=sip.yourprovider.com
SIP_TRUNK_PORT=5060 SIP_TRUNK_PORT=5060
SIP_TRUNK_USERNAME=your_sip_username SIP_TRUNK_USERNAME=your_sip_username

View File

@@ -94,7 +94,7 @@ async def event_stream(websocket: WebSocket):
# Immediately push current trunk status so the dashboard doesn't start blank # Immediately push current trunk status so the dashboard doesn't start blank
await _send_trunk_status(websocket, gateway) await _send_trunk_status(websocket, gateway)
subscription = gateway.event_bus.subscribe() subscription = gateway.event_bus.subscribe(replay_last=25)
try: try:
async for event in subscription: async for event in subscription:

View File

@@ -11,7 +11,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
class SIPTrunkSettings(BaseSettings): class SIPTrunkSettings(BaseSettings):
"""SIP trunk provider configuration.""" """SIP trunk provider configuration."""
model_config = SettingsConfigDict(env_prefix="SIP_TRUNK_") model_config = SettingsConfigDict(env_prefix="SIP_TRUNK_", env_file=".env", extra="ignore")
host: str = "sip.provider.com" host: str = "sip.provider.com"
port: int = 5060 port: int = 5060
@@ -24,7 +24,7 @@ class SIPTrunkSettings(BaseSettings):
class GatewaySIPSettings(BaseSettings): class GatewaySIPSettings(BaseSettings):
"""Gateway SIP listener for device registration.""" """Gateway SIP listener for device registration."""
model_config = SettingsConfigDict(env_prefix="GATEWAY_SIP_") model_config = SettingsConfigDict(env_prefix="GATEWAY_SIP_", env_file=".env", extra="ignore")
host: str = "0.0.0.0" host: str = "0.0.0.0"
port: int = 5060 port: int = 5060
@@ -34,7 +34,7 @@ class GatewaySIPSettings(BaseSettings):
class SpeachesSettings(BaseSettings): class SpeachesSettings(BaseSettings):
"""Speaches STT service configuration.""" """Speaches STT service configuration."""
model_config = SettingsConfigDict(env_prefix="SPEACHES_") model_config = SettingsConfigDict(env_prefix="SPEACHES_", env_file=".env", extra="ignore")
url: str = "http://localhost:22070" url: str = "http://localhost:22070"
model: str = "whisper-large-v3" model: str = "whisper-large-v3"
@@ -43,7 +43,7 @@ class SpeachesSettings(BaseSettings):
class ClassifierSettings(BaseSettings): class ClassifierSettings(BaseSettings):
"""Audio classifier thresholds.""" """Audio classifier thresholds."""
model_config = SettingsConfigDict(env_prefix="CLASSIFIER_") model_config = SettingsConfigDict(env_prefix="CLASSIFIER_", env_file=".env", extra="ignore")
music_threshold: float = 0.7 music_threshold: float = 0.7
speech_threshold: float = 0.6 speech_threshold: float = 0.6
@@ -54,7 +54,7 @@ class ClassifierSettings(BaseSettings):
class LLMSettings(BaseSettings): class LLMSettings(BaseSettings):
"""LLM service configuration (OpenAI-compatible API).""" """LLM service configuration (OpenAI-compatible API)."""
model_config = SettingsConfigDict(env_prefix="LLM_") model_config = SettingsConfigDict(env_prefix="LLM_", env_file=".env", extra="ignore")
base_url: str = "http://localhost:11434/v1" base_url: str = "http://localhost:11434/v1"
model: str = "llama3" model: str = "llama3"
@@ -67,7 +67,7 @@ class LLMSettings(BaseSettings):
class HoldSlayerSettings(BaseSettings): class HoldSlayerSettings(BaseSettings):
"""Hold Slayer behavior settings.""" """Hold Slayer behavior settings."""
model_config = SettingsConfigDict(env_prefix="HOLD_SLAYER_", env_prefix_allow_empty=True) model_config = SettingsConfigDict(env_prefix="HOLD_SLAYER_", env_prefix_allow_empty=True, env_file=".env", extra="ignore")
default_transfer_device: str = Field( default_transfer_device: str = Field(
default="sip_phone", validation_alias="DEFAULT_TRANSFER_DEVICE" default="sip_phone", validation_alias="DEFAULT_TRANSFER_DEVICE"
@@ -79,7 +79,7 @@ class HoldSlayerSettings(BaseSettings):
class TTSSettings(BaseSettings): class TTSSettings(BaseSettings):
"""Rhema TTS service configuration (OpenAI-compatible /v1/audio/speech).""" """Rhema TTS service configuration (OpenAI-compatible /v1/audio/speech)."""
model_config = SettingsConfigDict(env_prefix="TTS_") model_config = SettingsConfigDict(env_prefix="TTS_", env_file=".env", extra="ignore")
base_url: str = "http://localhost:8000" base_url: str = "http://localhost:8000"
model: str = "speaches-ai/Kokoro-82M-v1.0-ONNX" model: str = "speaches-ai/Kokoro-82M-v1.0-ONNX"
@@ -92,7 +92,7 @@ class TTSSettings(BaseSettings):
class ReceptionistSettings(BaseSettings): class ReceptionistSettings(BaseSettings):
"""AI Receptionist behavior settings.""" """AI Receptionist behavior settings."""
model_config = SettingsConfigDict(env_prefix="RECEPTIONIST_") model_config = SettingsConfigDict(env_prefix="RECEPTIONIST_", env_file=".env", extra="ignore")
enabled: bool = True enabled: bool = True
greeting_template: str = ( greeting_template: str = (
@@ -133,6 +133,11 @@ class Settings(BaseSettings):
# Outbound-call safety cap (REST + MCP make_call) # Outbound-call safety cap (REST + MCP make_call)
max_concurrent_calls: int = 4 max_concurrent_calls: int = 4
# Explicit engine mode — the mock engine must be asked for. An
# unconfigured trunk without this flag fails startup instead of
# silently degrading to a gateway that can't place real calls.
use_mock_sip: bool = False
# Notifications # Notifications
notify_sms_number: str = "" notify_sms_number: str = ""

View File

@@ -20,63 +20,70 @@ class EventBus:
Features: Features:
- Non-blocking publish (put_nowait) - Non-blocking publish (put_nowait)
- Automatic dead-subscriber cleanup (full queues are removed) - Slow subscribers lose their oldest event, never their subscription
- Event history (last N events for late joiners) - Event history (last N events, replayable to late joiners)
- Typed event filtering on subscriptions - Typed event filtering on subscriptions
- Async iteration via EventSubscription - Async iteration via EventSubscription
""" """
def __init__(self, max_history: int = 1000): def __init__(self, max_history: int = 1000):
self._subscribers: list[tuple[asyncio.Queue[GatewayEvent], Optional[set[EventType]]]] = [] self._subscribers: list[EventSubscription] = []
self._history: list[GatewayEvent] = [] self._history: list[GatewayEvent] = []
self._max_history = max_history self._max_history = max_history
async def publish(self, event: GatewayEvent) -> None: async def publish(self, event: GatewayEvent) -> None:
"""Publish an event to all subscribers.""" """Publish an event to all subscribers.
A full subscriber queue drops its oldest event (counted on the
subscription) — a slow dashboard must never be silently
unsubscribed while its socket stays open.
"""
self._history.append(event) self._history.append(event)
if len(self._history) > self._max_history: if len(self._history) > self._max_history:
self._history = self._history[-self._max_history :] self._history = self._history[-self._max_history :]
logger.info(f"📡 Event: {event.type.value} | {event.message or ''}") logger.info(f"📡 Event: {event.type.value} | {event.message or ''}")
dead_queues = [] for sub in self._subscribers:
for queue, type_filter in self._subscribers: if sub.type_filter and event.type not in sub.type_filter:
# Skip if subscriber has a type filter and this event doesn't match
if type_filter and event.type not in type_filter:
continue continue
try: sub.deliver(event)
queue.put_nowait(event)
except asyncio.QueueFull:
dead_queues.append((queue, type_filter))
for entry in dead_queues:
self._subscribers.remove(entry)
def subscribe( def subscribe(
self, self,
max_size: int = 100, max_size: int = 100,
event_types: Optional[set[EventType]] = None, event_types: Optional[set[EventType]] = None,
replay_last: int = 0,
) -> "EventSubscription": ) -> "EventSubscription":
""" """
Create a new subscription. Create a new subscription.
Args: Args:
max_size: Queue depth before subscriber is considered dead. max_size: Queue depth; overflow drops the oldest event.
event_types: Optional filter — only receive these event types. event_types: Optional filter — only receive these event types.
None means receive everything. None means receive everything.
replay_last: Seed the queue with up to N most recent
history events (post-filter) before live ones.
Returns: Returns:
An async iterator of GatewayEvents. An async iterator of GatewayEvents.
""" """
queue: asyncio.Queue[GatewayEvent] = asyncio.Queue(maxsize=max_size) queue: asyncio.Queue[GatewayEvent] = asyncio.Queue(maxsize=max_size)
entry = (queue, event_types) sub = EventSubscription(queue, self, event_types)
self._subscribers.append(entry) if replay_last > 0:
return EventSubscription(queue, self, entry) replayable = [
e for e in self._history
if not event_types or e.type in event_types
]
for event in replayable[-replay_last:]:
sub.deliver(event)
self._subscribers.append(sub)
return sub
def unsubscribe(self, entry: tuple) -> None: def unsubscribe(self, sub: "EventSubscription") -> None:
"""Remove a subscriber.""" """Remove a subscriber."""
if entry in self._subscribers: if sub in self._subscribers:
self._subscribers.remove(entry) self._subscribers.remove(sub)
@property @property
def recent_events(self) -> list[GatewayEvent]: def recent_events(self) -> list[GatewayEvent]:
@@ -95,11 +102,28 @@ class EventSubscription:
self, self,
queue: asyncio.Queue[GatewayEvent], queue: asyncio.Queue[GatewayEvent],
bus: EventBus, bus: EventBus,
entry: tuple, type_filter: Optional[set[EventType]] = None,
): ):
self._queue = queue self._queue = queue
self._bus = bus self._bus = bus
self._entry = entry self.type_filter = type_filter
self.dropped = 0 # events lost to queue overflow
def deliver(self, event: GatewayEvent) -> None:
"""Enqueue an event, dropping the oldest on overflow."""
try:
self._queue.put_nowait(event)
except asyncio.QueueFull:
try:
self._queue.get_nowait()
self._queue.put_nowait(event)
except (asyncio.QueueEmpty, asyncio.QueueFull):
pass
self.dropped += 1
if self.dropped in (1, 10, 100) or self.dropped % 1000 == 0:
logger.warning(
f"📡 Slow subscriber: {self.dropped} events dropped"
)
def __aiter__(self): def __aiter__(self):
return self return self
@@ -108,7 +132,7 @@ class EventSubscription:
try: try:
return await self._queue.get() return await self._queue.get()
except asyncio.CancelledError: except asyncio.CancelledError:
self._bus.unsubscribe(self._entry) self._bus.unsubscribe(self)
raise raise
async def get(self, timeout: Optional[float] = None) -> GatewayEvent: async def get(self, timeout: Optional[float] = None) -> GatewayEvent:
@@ -117,4 +141,4 @@ class EventSubscription:
def close(self): def close(self):
"""Unsubscribe from the event bus.""" """Unsubscribe from the event bus."""
self._bus.unsubscribe(self._entry) self._bus.unsubscribe(self)

View File

@@ -33,13 +33,28 @@ def build_sip_engine(
on_device_registered: Callable, on_device_registered: Callable,
on_incoming_call: Callable, on_incoming_call: Callable,
) -> SIPEngine: ) -> SIPEngine:
"""Build the appropriate SIP engine from config.""" """
Build the SIP engine from config.
The mock engine must be requested explicitly (USE_MOCK_SIP=true).
An unconfigured trunk or a failed SippyEngine construction raises —
the caller fails startup rather than running a gateway that can't
place real calls while reporting healthy.
"""
if settings.use_mock_sip:
logger.warning("🧪 USE_MOCK_SIP=true — SIP engine is a mock, no real calls")
return MockSIPEngine()
trunk = settings.sip_trunk trunk = settings.sip_trunk
gw_sip = settings.gateway_sip gw_sip = settings.gateway_sip
if trunk.host and trunk.host != "sip.provider.com": if not trunk.host or trunk.host in ("sip.provider.com", "sip.yourprovider.com"):
# Real trunk configured — use Sippy B2BUA raise RuntimeError(
try: "SIP trunk is not configured (SIP_TRUNK_HOST is unset or a "
"placeholder). Set SIP_TRUNK_* in .env, or set USE_MOCK_SIP=true "
"for development without a trunk."
)
return SippyEngine( return SippyEngine(
sip_address=gw_sip.host, sip_address=gw_sip.host,
sip_port=gw_sip.port, sip_port=gw_sip.port,
@@ -55,10 +70,6 @@ def build_sip_engine(
on_device_registered=on_device_registered, on_device_registered=on_device_registered,
on_incoming_call=on_incoming_call, on_incoming_call=on_incoming_call,
) )
except Exception as e:
logger.warning(f"Could not create SippyEngine: {e} — using mock")
return MockSIPEngine()
class AIPSTNGateway: class AIPSTNGateway:

View File

@@ -50,7 +50,6 @@ class SipCallLeg:
self.state = "init" # init, trying, ringing, connected, terminated self.state = "init" # init, trying, ringing, connected, terminated
self.media_port: Optional[int] = None # PJSUA2 conf bridge port self.media_port: Optional[int] = None # PJSUA2 conf bridge port
self.pending_sdp: Optional[str] = None # inbound INVITE SDP, until answered self.pending_sdp: Optional[str] = None # inbound INVITE SDP, until answered
self.dtmf_buffer: list[str] = []
def __repr__(self): def __repr__(self):
return f"<SipCallLeg {self.leg_id} {self.direction} {self.state}{self.remote_uri}>" return f"<SipCallLeg {self.leg_id} {self.direction} {self.state}{self.remote_uri}>"
@@ -297,9 +296,8 @@ class SippyEngine(SIPEngine):
] ]
elif kind == "dtmf": elif kind == "dtmf":
leg = self._legs.get(data["leg_id"]) # Received DTMF has no consumer yet; log until one exists
if leg: logger.info(f" DTMF '{data['digit']}' received on {data['leg_id']}")
leg.dtmf_buffer.append(data["digit"])
elif kind == "trunk_registered": elif kind == "trunk_registered":
self._trunk_registered = data["registered"] self._trunk_registered = data["registered"]

View File

@@ -85,24 +85,6 @@ class StoredCallFlow(Base):
return f"<StoredCallFlow {self.id} {self.phone_number}>" return f"<StoredCallFlow {self.id} {self.phone_number}>"
class Contact(Base):
__tablename__ = "contacts"
id = Column(String, primary_key=True)
name = Column(String, nullable=False)
phone_numbers = Column(JSON, nullable=False) # [{number, label, primary}, ...]
category = Column(String) # personal / business / service
routing_preference = Column(String, nullable=True) # how to handle their calls
notes = Column(Text, nullable=True)
call_count = Column(Integer, default=0)
last_call = Column(DateTime, nullable=True)
created_at = Column(DateTime, default=func.now())
updated_at = Column(DateTime, default=func.now(), onupdate=func.now())
def __repr__(self) -> str:
return f"<Contact {self.id} {self.name}>"
class Device(Base): class Device(Base):
__tablename__ = "devices" __tablename__ = "devices"

59
main.py
View File

@@ -172,6 +172,7 @@ async def lifespan(app: FastAPI):
gateway.register_mode_handler(CallMode.HOLD_SLAYER, launch_hold_slayer) gateway.register_mode_handler(CallMode.HOLD_SLAYER, launch_hold_slayer)
try:
gateway.sip_engine = build_sip_engine( gateway.sip_engine = build_sip_engine(
settings, settings,
gateway.media_pipeline, gateway.media_pipeline,
@@ -179,11 +180,15 @@ async def lifespan(app: FastAPI):
on_device_registered=gateway._on_sip_device_registered, on_device_registered=gateway._on_sip_device_registered,
on_incoming_call=receptionist.on_inbound_call, on_incoming_call=receptionist.on_inbound_call,
) )
except Exception as e:
logger.critical(f"\n❌ SIP engine failed to initialize:\n {e}")
sys.exit(1)
await routing_svc.start() await routing_svc.start()
await gateway.start() await gateway.start()
app.state.gateway = gateway app.state.gateway = gateway
app.state.routing_service = routing_svc app.state.routing_service = routing_svc
app.state.transcription_service = transcription
notification_svc = NotificationService(gateway.event_bus, settings) notification_svc = NotificationService(gateway.event_bus, settings)
await notification_svc.start() await notification_svc.start()
@@ -298,21 +303,67 @@ async def root():
@app.get("/health", tags=["System"]) @app.get("/health", tags=["System"])
async def health(): async def health():
"""Health check endpoint.""" """
Health check. "healthy" means the gateway can actually do its job:
real engine, registered trunk, reachable database. A mock engine or
a failing dependency reports "degraded" with the reason visible.
"""
from core.sip_engine import MockSIPEngine
from db.database import session_scope
gateway = getattr(app.state, "gateway", None) gateway = getattr(app.state, "gateway", None)
ready = gateway is not None and await gateway.sip_engine.is_ready() ready = gateway is not None and await gateway.sip_engine.is_ready()
trunk_status = await gateway.sip_engine.get_trunk_status() if gateway else {"registered": False} trunk_status = await gateway.sip_engine.get_trunk_status() if gateway else {"registered": False}
return { engine_mode = (
"status": "healthy" if ready else "degraded", "mock" if gateway is None or isinstance(gateway.sip_engine, MockSIPEngine)
else "sippy"
)
db_ok = False
db_error = None
try:
from sqlalchemy import text
async with session_scope() as session:
await session.execute(text("SELECT 1"))
db_ok = True
except Exception as e:
db_error = str(e)[:200]
healthy = (
ready
and db_ok
and engine_mode == "sippy"
and trunk_status.get("registered", False)
)
checks = {
"gateway": "ready" if gateway else "not initialized", "gateway": "ready" if gateway else "not initialized",
"engine": engine_mode,
"sip_engine": "ready" if ready else "not ready", "sip_engine": "ready" if ready else "not ready",
"database": "ok" if db_ok else f"error: {db_error}",
"sip_trunk": { "sip_trunk": {
"registered": trunk_status.get("registered", False), "registered": trunk_status.get("registered", False),
"host": trunk_status.get("host"), "host": trunk_status.get("host"),
"mock": trunk_status.get("mock", False),
"reason": trunk_status.get("reason"), "reason": trunk_status.get("reason"),
}, },
} }
if gateway is not None:
tts = getattr(gateway, "_tts", None)
checks["tts"] = _availability(tts)
transcription = getattr(app.state, "transcription_service", None)
checks["stt"] = _availability(transcription)
return {"status": "healthy" if healthy else "degraded", **checks}
def _availability(service) -> str:
"""Last-known reachability of an HTTP leaf service."""
if service is None:
return "not attached"
available = getattr(service, "available", None)
if available is None:
return "unknown (no requests yet)"
return "ok" if available else "unreachable"
if __name__ == "__main__": if __name__ == "__main__":

View File

@@ -1,60 +0,0 @@
"""
Contact models — People and organizations you call.
"""
from datetime import datetime
from typing import Optional
from pydantic import BaseModel, Field
class PhoneNumber(BaseModel):
"""A phone number associated with a contact."""
number: str # E.164 format
label: str = "main" # main, mobile, work, home, fax, etc.
primary: bool = False
class ContactBase(BaseModel):
"""Shared contact fields."""
name: str
phone_numbers: list[PhoneNumber]
category: Optional[str] = None # personal / business / service
routing_preference: Optional[str] = None # how to handle their calls
notes: Optional[str] = None
class Contact(ContactBase):
"""Full contact model."""
id: str
call_count: int = 0
last_call: Optional[datetime] = None
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
@property
def primary_number(self) -> Optional[str]:
"""Get the primary phone number."""
for pn in self.phone_numbers:
if pn.primary:
return pn.number
return self.phone_numbers[0].number if self.phone_numbers else None
class ContactCreate(ContactBase):
"""Request model for creating a contact."""
pass
class ContactUpdate(BaseModel):
"""Request model for updating a contact."""
name: Optional[str] = None
phone_numbers: Optional[list[PhoneNumber]] = None
category: Optional[str] = None
routing_preference: Optional[str] = None
notes: Optional[str] = None

View File

@@ -7,6 +7,7 @@ surfaces can't drift. Every function takes an AsyncSession; callers
own the transaction (get_db for REST, session_scope for MCP/services). own the transaction (get_db for REST, session_scope for MCP/services).
""" """
import asyncio
import logging import logging
import uuid import uuid
from datetime import datetime from datetime import datetime
@@ -159,9 +160,25 @@ async def persist_call_on_end(call: ActiveCall, final_status: CallStatus) -> Non
"""Insert a CallRecord and any transcript chunks for `call`. """Insert a CallRecord and any transcript chunks for `call`.
Wired into CallManager as its on_call_ended hook by the Wired into CallManager as its on_call_ended hook by the
composition root in main.py. composition root in main.py. Retries briefly — losing the row
means the call never happened as far as history is concerned, so
the final failure logs at ERROR with the payload identifiers.
""" """
for attempt in range(3):
try: try:
await _write_call_record(call, final_status)
return
except Exception as e:
if attempt == 2:
logger.error(
f"Call record lost: id={call.id} number={call.remote_number} "
f"status={final_status.value}: {e}"
)
return
await asyncio.sleep(2**attempt)
async def _write_call_record(call: ActiveCall, final_status: CallStatus) -> None:
async with session_scope() as session: async with session_scope() as session:
record = CallRecord( record = CallRecord(
id=call.id, id=call.id,
@@ -207,5 +224,3 @@ async def persist_call_on_end(call: ActiveCall, final_status: CallStatus) -> Non
speaker=speaker, speaker=speaker,
text=payload, text=payload,
)) ))
except Exception as e:
logger.warning(f"Could not persist call {call.id}: {e}")

View File

@@ -56,6 +56,29 @@ class HoldSlayerService:
self.settings = settings self.settings = settings
self.tts = tts self.tts = tts
async def _service_error(self, call_id: str, service: str, error: Exception) -> None:
"""Surface a failed dependency as a typed event, not silence."""
logger.error(f"⚠️ {service} failed for {call_id}: {error}")
try:
await self.gateway.event_bus.publish(GatewayEvent(
type=EventType.ERROR,
call_id=call_id,
data={"service": service, "error": str(error)},
message=f"⚠️ {service} failed: {error}",
))
except Exception:
pass
async def _transcribe(
self, call_id: str, audio: bytes, prompt: Optional[str] = None
) -> str:
"""Transcribe with an explicit empty-string fallback on failure."""
try:
return await self.transcription.transcribe(audio, prompt=prompt)
except Exception as e:
await self._service_error(call_id, "transcription", e)
return ""
async def run( async def run(
self, self,
call: ActiveCall, call: ActiveCall,
@@ -309,9 +332,10 @@ class HoldSlayerService:
AudioClassification.IVR_PROMPT, AudioClassification.IVR_PROMPT,
AudioClassification.LIVE_HUMAN, AudioClassification.LIVE_HUMAN,
): ):
transcript = await self.transcription.transcribe( transcript = await self._transcribe(
call.id,
audio_chunk, audio_chunk,
prompt="Phone IVR menu, customer service, press 1 for..." prompt="Phone IVR menu, customer service, press 1 for...",
) )
if transcript: if transcript:
await self.call_manager.add_transcript(call.id, transcript) await self.call_manager.add_transcript(call.id, transcript)
@@ -429,7 +453,7 @@ class HoldSlayerService:
# Check for human # Check for human
if result.audio_type == AudioClassification.LIVE_HUMAN: if result.audio_type == AudioClassification.LIVE_HUMAN:
# Verify with transcription # Verify with transcription
transcript = await self.transcription.transcribe(audio_chunk) transcript = await self._transcribe(call.id, audio_chunk)
if transcript: if transcript:
await self.call_manager.add_transcript(call.id, transcript) await self.call_manager.add_transcript(call.id, transcript)
# If we got meaningful speech, it's probably a real person # If we got meaningful speech, it's probably a real person
@@ -491,7 +515,7 @@ class HoldSlayerService:
continue continue
# Transcribe # Transcribe
transcript = await self.transcription.transcribe(audio_chunk) transcript = await self._transcribe(call.id, audio_chunk)
if not transcript: if not transcript:
continue continue
@@ -545,7 +569,7 @@ class HoldSlayerService:
AudioClassification.IVR_PROMPT, AudioClassification.IVR_PROMPT,
AudioClassification.LIVE_HUMAN, AudioClassification.LIVE_HUMAN,
): ):
text = await self.transcription.transcribe(audio_chunk) text = await self._transcribe(call.id, audio_chunk)
if text: if text:
transcript_parts.append(text) transcript_parts.append(text)
@@ -712,8 +736,12 @@ class HoldSlayerService:
fd, tmp_path = tempfile.mkstemp(suffix=".wav", prefix=f"speak_{call.id}_") fd, tmp_path = tempfile.mkstemp(suffix=".wav", prefix=f"speak_{call.id}_")
os.close(fd) os.close(fd)
try:
try: try:
ok = await self.tts.synthesize_to_file(text, tmp_path) ok = await self.tts.synthesize_to_file(text, tmp_path)
except Exception as e:
await self._service_error(call.id, "tts", e)
return False
if not ok: if not ok:
logger.warning(f"🗣️ TTS synthesis returned no audio for: '{text[:60]}'") logger.warning(f"🗣️ TTS synthesis returned no audio for: '{text[:60]}'")
return False return False

View File

@@ -327,15 +327,14 @@ class LLMClient:
except httpx.HTTPStatusError as e: except httpx.HTTPStatusError as e:
self._total_errors += 1 self._total_errors += 1
logger.error(f"LLM API error: {e.response.status_code} {e.response.text[:200]}") logger.error(f"LLM API error: {e.response.status_code} {e.response.text[:200]}")
return "" raise
except httpx.TimeoutException: except httpx.TimeoutException:
self._total_errors += 1 self._total_errors += 1
logger.error(f"LLM API timeout after {self.timeout}s") logger.error(f"LLM API timeout after {self.timeout}s")
return "" raise
except Exception as e: except Exception:
self._total_errors += 1 self._total_errors += 1
logger.error(f"LLM client error: {e}") raise
return ""
@staticmethod @staticmethod
def _parse_json_response(text: str) -> dict[str, Any]: def _parse_json_response(text: str) -> dict[str, Any]:

View File

@@ -67,7 +67,6 @@ class NotificationService:
self._event_bus = event_bus self._event_bus = event_bus
self._settings = settings self._settings = settings
self._task: Optional[asyncio.Task] = None self._task: Optional[asyncio.Task] = None
self._sms_sender: Optional[Any] = None
# Track what we've already notified (avoid spam) # Track what we've already notified (avoid spam)
self._notified: dict[str, set[str]] = {} # call_id -> set of event types self._notified: dict[str, set[str]] = {} # call_id -> set of event types
@@ -214,43 +213,3 @@ class NotificationService:
# WebSocket notifications go through the event bus # WebSocket notifications go through the event bus
# (the WebSocket handler in the API reads from EventBus directly) # (the WebSocket handler in the API reads from EventBus directly)
# SMS for critical notifications
if (
notification.priority == NotificationPriority.CRITICAL
and self._settings.notify_sms_number
):
await self._send_sms(notification)
async def _send_sms(self, notification: Notification) -> None:
"""
Send an SMS notification.
Uses a simple HTTP-based SMS gateway. In production,
this would use Twilio, AWS SNS, or similar.
"""
phone = self._settings.notify_sms_number
if not phone:
return
try:
import httpx
# Generic webhook-based SMS (configure your provider)
# This is a placeholder — wire up your preferred SMS provider
logger.info(f"📱 SMS → {phone}: {notification.title}")
# Example: Twilio-style API
# async with httpx.AsyncClient() as client:
# await client.post(
# "https://api.twilio.com/2010-04-01/Accounts/.../Messages.json",
# data={
# "To": phone,
# "From": self._settings.sip_trunk.did,
# "Body": f"{notification.title}\n{notification.message}",
# },
# auth=(account_sid, auth_token),
# )
except Exception as e:
logger.error(f"SMS send failed: {e}")

View File

@@ -203,6 +203,19 @@ class ReceptionistService:
# State machine steps # State machine steps
# ---------------------------------------------------------------- # ----------------------------------------------------------------
async def _service_error(self, call_id: str, service: str, error: Exception) -> None:
"""Surface a failed dependency as a typed event, not silence."""
logger.error(f"⚠️ {service} failed for {call_id}: {error}")
try:
await self.gateway.event_bus.publish(GatewayEvent(
type=EventType.ERROR,
call_id=call_id,
data={"service": service, "error": str(error)},
message=f"⚠️ {service} failed: {error}",
))
except Exception:
pass
async def _greet(self, call: ActiveCall, sip_leg_id: str) -> None: async def _greet(self, call: ActiveCall, sip_leg_id: str) -> None:
await self.gateway.event_bus.publish(GatewayEvent( await self.gateway.event_bus.publish(GatewayEvent(
type=EventType.RECEPTIONIST_GREETING, type=EventType.RECEPTIONIST_GREETING,
@@ -250,7 +263,11 @@ class ReceptionistService:
if not audio or self.transcription is None: if not audio or self.transcription is None:
return "" return ""
try:
return await self.transcription.transcribe(bytes(audio)) return await self.transcription.transcribe(bytes(audio))
except Exception as e:
await self._service_error(call.id, "transcription", e)
return ""
async def _classify( async def _classify(
self, self,
@@ -291,7 +308,7 @@ class ReceptionistService:
system=self.settings.llm_persona, system=self.settings.llm_persona,
) )
except Exception as e: except Exception as e:
logger.warning(f"Receptionist LLM classify failed: {e}") await self._service_error(call.id, "llm", e)
return { return {
"intent": transcript, "intent": transcript,
"urgency": "normal", "urgency": "normal",
@@ -304,10 +321,14 @@ class ReceptionistService:
routing_decision: Optional[RoutingDecision], routing_decision: Optional[RoutingDecision],
classification: dict, classification: dict,
) -> RoutingAction: ) -> RoutingAction:
"""Rules win on conflict; otherwise use the LLM's recommendation.""" """Rules win on conflict; otherwise use the LLM's recommendation.
if routing_decision and routing_decision.action.type not in (
RoutingActionType.TAKE_MESSAGE, A decision counts as a rule only when one actually matched
): (matched_rule_id set) — the no-rule default is take_message and
must stay overridable by the LLM. A matched TAKE_MESSAGE rule
wins like any other rule.
"""
if routing_decision and routing_decision.matched_rule_id:
return routing_decision.action return routing_decision.action
recommended = (classification.get("recommended_action") or "ring").lower() recommended = (classification.get("recommended_action") or "ring").lower()
@@ -347,7 +368,13 @@ class ReceptionistService:
call.id, media_pipeline=media, leg_ids=[sip_leg_id] call.id, media_pipeline=media, leg_ids=[sip_leg_id]
) )
try: try:
await asyncio.sleep(self.settings.message_max_seconds) # Record up to the cap, but stop early once the caller hangs
# up (leg termination ends the call via the leg-state wiring).
deadline = _time.monotonic() + self.settings.message_max_seconds
while _time.monotonic() < deadline:
await asyncio.sleep(1.0)
if self.gateway.call_manager.get_call(call.id) is None:
break
finally: finally:
session = await recording_svc.stop_recording( session = await recording_svc.stop_recording(
call.id, media_pipeline=media call.id, media_pipeline=media
@@ -360,7 +387,7 @@ class ReceptionistService:
audio_bytes = Path(rec_path).read_bytes() audio_bytes = Path(rec_path).read_bytes()
message_text = await self.transcription.transcribe(audio_bytes) message_text = await self.transcription.transcribe(audio_bytes)
except Exception as e: except Exception as e:
logger.warning(f"Receptionist transcribe failed: {e}") await self._service_error(call.id, "transcription", e)
if message_text: if message_text:
call.transcript_chunks.append(f"caller_message: {message_text}") call.transcript_chunks.append(f"caller_message: {message_text}")
@@ -393,8 +420,12 @@ class ReceptionistService:
fd, tmp_path = tempfile.mkstemp(suffix=".wav", prefix=f"recept_{call.id}_") fd, tmp_path = tempfile.mkstemp(suffix=".wav", prefix=f"recept_{call.id}_")
os.close(fd) os.close(fd)
try:
try: try:
ok = await tts.synthesize_to_file(text, tmp_path) ok = await tts.synthesize_to_file(text, tmp_path)
except Exception as e:
await self._service_error(call.id, "tts", e)
return
if not ok: if not ok:
return return
await media.play_wav(sip_leg_id, tmp_path) await media.play_wav(sip_leg_id, tmp_path)

View File

@@ -91,6 +91,7 @@ class RecordingService:
filepath_agent=filepath_agent, filepath_agent=filepath_agent,
started_at=datetime.now(), started_at=datetime.now(),
sample_rate=self._sample_rate, sample_rate=self._sample_rate,
leg_ids=leg_ids,
) )
# Start PJSUA2 recording if media pipeline is available # Start PJSUA2 recording if media pipeline is available
@@ -159,12 +160,18 @@ class RecordingService:
@staticmethod @staticmethod
async def _persist_recording(session: "RecordingSession") -> None: async def _persist_recording(session: "RecordingSession") -> None:
"""Write a recordings row for this session. Failures are non-fatal.""" """Write a recordings row for this session, with bounded retry.
try:
import uuid as _uuid
from db.database import RecordingRecord, get_session_factory
async with get_session_factory()() as db: Non-fatal for the call, but a lost row means the dashboard can
never find the WAV — so failures log at ERROR, not warning.
"""
import uuid as _uuid
from db.database import RecordingRecord, session_scope
for attempt in range(3):
try:
async with session_scope() as db:
db.add(RecordingRecord( db.add(RecordingRecord(
id=f"rec_{_uuid.uuid4().hex[:10]}", id=f"rec_{_uuid.uuid4().hex[:10]}",
call_id=session.call_id, call_id=session.call_id,
@@ -176,9 +183,15 @@ class RecordingService:
started_at=session.started_at, started_at=session.started_at,
ended_at=session.stopped_at, ended_at=session.stopped_at,
)) ))
await db.commit() return
except Exception as e: except Exception as e:
logger.warning(f"Recording persistence failed: {e}") if attempt == 2:
logger.error(
f"Recording row lost for {session.call_id} "
f"(path={session.filepath_mixed}): {e}"
)
return
await asyncio.sleep(2 ** attempt)
async def _recording_timeout(self, call_id: str) -> None: async def _recording_timeout(self, call_id: str) -> None:
"""Auto-stop recording after max duration.""" """Auto-stop recording after max duration."""
@@ -239,6 +252,7 @@ class RecordingSession:
filepath_agent: Optional[str] = None, filepath_agent: Optional[str] = None,
started_at: Optional[datetime] = None, started_at: Optional[datetime] = None,
sample_rate: int = 16000, sample_rate: int = 16000,
leg_ids: Optional[list[str]] = None,
): ):
self.call_id = call_id self.call_id = call_id
self.filepath_mixed = filepath_mixed self.filepath_mixed = filepath_mixed
@@ -249,7 +263,7 @@ class RecordingSession:
self.duration_seconds: Optional[int] = None self.duration_seconds: Optional[int] = None
self.file_size_bytes: Optional[int] = None self.file_size_bytes: Optional[int] = None
self.sample_rate = sample_rate self.sample_rate = sample_rate
self._leg_ids: list[str] = [] self._leg_ids: list[str] = list(leg_ids or [])
def to_dict(self) -> dict: def to_dict(self) -> dict:
return { return {

View File

@@ -27,6 +27,8 @@ class TranscriptionService:
def __init__(self, settings: SpeachesSettings): def __init__(self, settings: SpeachesSettings):
self.settings = settings self.settings = settings
self._client: Optional[httpx.AsyncClient] = None self._client: Optional[httpx.AsyncClient] = None
# Last-known reachability, surfaced by /health (None = no requests yet)
self.available: Optional[bool] = None
async def _get_client(self) -> httpx.AsyncClient: async def _get_client(self) -> httpx.AsyncClient:
"""Get or create the HTTP client.""" """Get or create the HTTP client."""
@@ -60,6 +62,9 @@ class TranscriptionService:
# Convert raw PCM to WAV format for the API # Convert raw PCM to WAV format for the API
wav_data = self._pcm_to_wav(audio_data) wav_data = self._pcm_to_wav(audio_data)
# Raises on failure — callers decide the per-call fallback and
# publish a service-error event; swallowing here made a down
# Speaches look like "the AI is deciding badly".
try: try:
response = await client.post( response = await client.post(
"/v1/audio/transcriptions", "/v1/audio/transcriptions",
@@ -72,45 +77,14 @@ class TranscriptionService:
}, },
) )
response.raise_for_status() response.raise_for_status()
except Exception:
self.available = False
raise
self.available = True
text = response.text.strip() text = response.text.strip()
logger.debug(f"Transcription: '{text}'") logger.debug(f"Transcription: '{text}'")
return text return text
except httpx.HTTPStatusError as e:
logger.error(f"Speaches API error: {e.response.status_code} {e.response.text}")
return ""
except httpx.ConnectError:
logger.error(f"Cannot connect to Speaches at {self.settings.url}")
return ""
except Exception as e:
logger.error(f"Transcription failed: {e}")
return ""
async def transcribe_stream(
self,
audio_data: bytes,
language: str = "en",
):
"""
Stream transcription — for real-time results.
Uses Speaches streaming endpoint if available,
falls back to chunked transcription.
Yields:
str: Partial transcription chunks
"""
# For now, do chunked transcription
# TODO: Implement WebSocket streaming when Speaches supports it
chunk_size = 16000 * 2 * 3 # 3 seconds of 16kHz 16-bit mono
for i in range(0, len(audio_data), chunk_size):
chunk = audio_data[i:i + chunk_size]
if len(chunk) > 0:
text = await self.transcribe(chunk, language)
if text:
yield text
async def close(self) -> None: async def close(self) -> None:
"""Close the HTTP client.""" """Close the HTTP client."""
if self._client and not self._client.is_closed: if self._client and not self._client.is_closed:

View File

@@ -22,6 +22,8 @@ class TTSService:
def __init__(self, settings: TTSSettings): def __init__(self, settings: TTSSettings):
self.settings = settings self.settings = settings
self._client: Optional[httpx.AsyncClient] = None self._client: Optional[httpx.AsyncClient] = None
# Last-known reachability, surfaced by /health (None = no requests yet)
self.available: Optional[bool] = None
async def _get_client(self) -> httpx.AsyncClient: async def _get_client(self) -> httpx.AsyncClient:
if self._client is None or self._client.is_closed: if self._client is None or self._client.is_closed:
@@ -54,19 +56,17 @@ class TTSService:
"sample_rate": self.settings.sample_rate, "sample_rate": self.settings.sample_rate,
} }
# Raises on failure — callers decide the per-call fallback and
# publish a service-error event; swallowing here made a down
# Rhema look like "the AI went quiet".
try: try:
response = await client.post("/v1/audio/speech", json=body) response = await client.post("/v1/audio/speech", json=body)
response.raise_for_status() response.raise_for_status()
except Exception:
self.available = False
raise
self.available = True
return response.content return response.content
except httpx.HTTPStatusError as e:
logger.error(f"Rhema TTS error: {e.response.status_code} {e.response.text}")
return b""
except httpx.ConnectError:
logger.error(f"Cannot connect to Rhema at {self.settings.base_url}")
return b""
except Exception as e:
logger.error(f"TTS synthesis failed: {e}")
return b""
async def synthesize_to_file( async def synthesize_to_file(
self, self,

View File

@@ -102,14 +102,9 @@ class TestEngineEventFunnel:
async def test_dtmf_and_trunk_events(self): async def test_dtmf_and_trunk_events(self):
engine = _engine_on_loop() engine = _engine_on_loop()
_post_from_thread(engine, "incoming_invite", {
"leg_id": "leg_test4", "from_uri": "a", "to_uri": "b", "sdp": None,
})
await asyncio.sleep(0.05)
_post_from_thread(engine, "dtmf", {"leg_id": "leg_test4", "digit": "5"}) _post_from_thread(engine, "dtmf", {"leg_id": "leg_test4", "digit": "5"})
_post_from_thread(engine, "trunk_registered", {"registered": True}) _post_from_thread(engine, "trunk_registered", {"registered": True})
await asyncio.sleep(0.05) await asyncio.sleep(0.05)
assert engine._legs["leg_test4"].dtmf_buffer == ["5"]
assert engine._trunk_registered is True assert engine._trunk_registered is True

View File

@@ -0,0 +1,89 @@
"""
Engine-mode and event-bus-integrity tests.
The mock engine must be requested explicitly; a full subscriber queue
drops its oldest event but never loses the subscription; history is
replayable to late joiners.
"""
import asyncio
import pytest
from config import Settings
from core.event_bus import EventBus
from core.gateway import build_sip_engine
from core.media_pipeline import MediaPipeline
from core.sip_engine import MockSIPEngine
from models.events import EventType, GatewayEvent
def _noop(*args, **kwargs):
pass
class TestEngineMode:
def _build(self, settings: Settings):
return build_sip_engine(
settings,
MediaPipeline(sample_rate=16000),
on_leg_state_change=_noop,
on_device_registered=_noop,
on_incoming_call=_noop,
)
def test_mock_engine_only_when_asked(self):
settings = Settings(use_mock_sip=True)
assert isinstance(self._build(settings), MockSIPEngine)
def test_unconfigured_trunk_refuses_to_build(self):
settings = Settings(use_mock_sip=False)
settings.sip_trunk.host = "sip.yourprovider.com"
with pytest.raises(RuntimeError, match="not configured"):
self._build(settings)
def _event(i: int) -> GatewayEvent:
return GatewayEvent(
type=EventType.CALL_INITIATED,
call_id=f"call_{i}",
data={},
message=f"event {i}",
)
class TestEventBusIntegrity:
async def test_overflow_drops_oldest_keeps_subscription(self):
bus = EventBus()
sub = bus.subscribe(max_size=3)
for i in range(5):
await bus.publish(_event(i))
assert bus.subscriber_count == 1 # never evicted
assert sub.dropped == 2
received = [await asyncio.wait_for(sub.get(), 1.0) for _ in range(3)]
assert [e.call_id for e in received] == ["call_2", "call_3", "call_4"]
async def test_replay_last_seeds_history(self):
bus = EventBus()
for i in range(10):
await bus.publish(_event(i))
sub = bus.subscribe(replay_last=3)
received = [await asyncio.wait_for(sub.get(), 1.0) for _ in range(3)]
assert [e.call_id for e in received] == ["call_7", "call_8", "call_9"]
async def test_replay_respects_type_filter(self):
bus = EventBus()
await bus.publish(_event(1))
await bus.publish(GatewayEvent(
type=EventType.HUMAN_DETECTED, call_id="call_h", data={}, message="x"
))
sub = bus.subscribe(
event_types={EventType.HUMAN_DETECTED}, replay_last=5
)
event = await asyncio.wait_for(sub.get(), 1.0)
assert event.call_id == "call_h"
assert sub._queue.empty()

View File

@@ -24,10 +24,27 @@ class TestReceptionistDecide:
gw = _make_gateway() gw = _make_gateway()
svc = ReceptionistService(gw) svc = ReceptionistService(gw)
rule_action = RoutingAction(type=RoutingActionType.REJECT, message="nope") rule_action = RoutingAction(type=RoutingActionType.REJECT, message="nope")
decision = RoutingDecision(action=rule_action, reason="rule said so") decision = RoutingDecision(
action=rule_action,
matched_rule_id="rule_1",
matched_rule_name="block",
reason="rule said so",
)
chosen = svc._decide(decision, {"recommended_action": "ring"}) chosen = svc._decide(decision, {"recommended_action": "ring"})
assert chosen.type == RoutingActionType.REJECT assert chosen.type == RoutingActionType.REJECT
def test_matched_take_message_rule_beats_llm(self):
gw = _make_gateway()
svc = ReceptionistService(gw)
decision = RoutingDecision(
action=RoutingAction(type=RoutingActionType.TAKE_MESSAGE),
matched_rule_id="rule_2",
matched_rule_name="voicemail-hours",
reason="matched rule 'voicemail-hours'",
)
chosen = svc._decide(decision, {"recommended_action": "ring"})
assert chosen.type == RoutingActionType.TAKE_MESSAGE
def test_falls_back_to_llm_when_rule_is_default_take_message(self): def test_falls_back_to_llm_when_rule_is_default_take_message(self):
gw = _make_gateway() gw = _make_gateway()
svc = ReceptionistService(gw) svc = ReceptionistService(gw)

View File

@@ -123,14 +123,14 @@ class TestLLMClient:
assert result["key"] == "value" assert result["key"] == "value"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_chat_http_error_returns_empty(self): async def test_chat_error_raises(self):
"""Verify HTTP errors return empty string gracefully.""" """Failures propagate to the caller (which owns the fallback)."""
client = self._make_client() client = self._make_client()
with patch.object(client._client, "post", new_callable=AsyncMock) as mock_post: with patch.object(client._client, "post", new_callable=AsyncMock) as mock_post:
mock_post.side_effect = Exception("Connection refused") mock_post.side_effect = Exception("Connection refused")
result = await client.chat("test", system="test") with pytest.raises(Exception, match="Connection refused"):
assert result == "" await client.chat("test", system="test")
assert client._total_errors == 1 assert client._total_errors == 1
@pytest.mark.asyncio @pytest.mark.asyncio