r/hold-slayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stage 4: honest health, explicit error policy, event-bus integrity

Browse Source 
Engine mode is now explicit: USE_MOCK_SIP=true is the only way to get
the mock engine; an unconfigured trunk fails startup with guidance
instead of silently degrading. Root-caused why the engine always ran
mock: nested pydantic-settings never read .env (no env_file on the
sub-settings classes) — all 8 now declare it.

/health stops lying: reports engine mode (sippy|mock), a live DB
SELECT 1, trunk registration state with reason, and TTS/STT
availability from their last real request; "healthy" now requires
ready + db + sippy + registered trunk.

Error policy: leaf services (tts/transcription/llm_client) raise and
track availability; call-loop callers catch, publish EventType.ERROR
naming the failed service, and apply an explicit fallback. Persistence
writes get one bounded 3x exponential retry, then an ERROR log — no
more silent data loss.

Event bus: a full subscriber queue drops its oldest event (counted)
instead of silently evicting the subscription; subscribe(replay_last=N)
delivers the advertised history replay, used by /ws/events (25).

Receptionist correctness: a matched TAKE_MESSAGE rule beats the LLM;
voicemail polls for early hangup and stops/transcribes/hangs up in
finally; RecordingSession finally keeps its leg_ids so taps detach.

Dead code removed: models/contact.py + Contact table, dtmf_buffer,
transcribe_stream stub, SMS stub in notification.py.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Robert Helewka
2026-07-10 07:01:45 -04:00
parent 67a00defc3
commit 4048ce1db6
21 changed files with 492 additions and 357 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
services/call_persistence.py
View File

@@ -7,6 +7,7 @@ surfaces can't drift. Every function takes an AsyncSession; callers
own the transaction (get_db for REST, session_scope for MCP/services).
"""
import asyncio
import logging
import uuid
from datetime import datetime
@@ -159,53 +160,67 @@ async def persist_call_on_end(call: ActiveCall, final_status: CallStatus) -> Non
"""Insert a CallRecord and any transcript chunks for `call`.
Wired into CallManager as its on_call_ended hook by the
composition root in main.py.
composition root in main.py. Retries briefly — losing the row
means the call never happened as far as history is concerned, so
the final failure logs at ERROR with the payload identifiers.
"""
try:
async with session_scope() as session:
record = CallRecord(
id=call.id,
direction=call.direction,
remote_number=call.remote_number,
status=final_status.value,
mode=call.mode.value,
intent=call.intent,
started_at=call.started_at,
ended_at=datetime.now(),
duration=int(call.duration),
hold_time=int(call.hold_time),
device_used=call.device,
call_flow_id=call.call_flow_id,
classification_timeline=[
{
"timestamp": c.timestamp,
"audio_type": c.audio_type.value,
"confidence": c.confidence,
}
for c in call.classification_history
],
metadata_={"services": list(call.services)},
)
session.add(record)
for attempt in range(3):
try:
await _write_call_record(call, final_status)
return
except Exception as e:
if attempt == 2:
logger.error(
f"Call record lost: id={call.id} number={call.remote_number} "
f"status={final_status.value}: {e}"
)
return
await asyncio.sleep(2**attempt)
# Each transcript chunk gets its own row with a sequence number
# so the dashboard can render them in order with click-to-seek.
for seq, text in enumerate(call.transcript_chunks):
speaker = "unknown"
payload = text
if ":" in text:
head, rest = text.split(":", 1)
head = head.strip().lower()
if head in {"caller", "agent", "receptionist", "caller_message"}:
speaker = head if head != "caller_message" else "caller"
payload = rest.strip()
session.add(TranscriptChunk(
id=f"tc_{uuid.uuid4().hex[:10]}",
call_id=call.id,
seq=seq,
t_offset_ms=0,
speaker=speaker,
text=payload,
))
except Exception as e:
logger.warning(f"Could not persist call {call.id}: {e}")
async def _write_call_record(call: ActiveCall, final_status: CallStatus) -> None:
async with session_scope() as session:
record = CallRecord(
id=call.id,
direction=call.direction,
remote_number=call.remote_number,
status=final_status.value,
mode=call.mode.value,
intent=call.intent,
started_at=call.started_at,
ended_at=datetime.now(),
duration=int(call.duration),
hold_time=int(call.hold_time),
device_used=call.device,
call_flow_id=call.call_flow_id,
classification_timeline=[
{
"timestamp": c.timestamp,
"audio_type": c.audio_type.value,
"confidence": c.confidence,
}
for c in call.classification_history
],
metadata_={"services": list(call.services)},
)
session.add(record)
# Each transcript chunk gets its own row with a sequence number
# so the dashboard can render them in order with click-to-seek.
for seq, text in enumerate(call.transcript_chunks):
speaker = "unknown"
payload = text
if ":" in text:
head, rest = text.split(":", 1)
head = head.strip().lower()
if head in {"caller", "agent", "receptionist", "caller_message"}:
speaker = head if head != "caller_message" else "caller"
payload = rest.strip()
session.add(TranscriptChunk(
id=f"tc_{uuid.uuid4().hex[:10]}",
call_id=call.id,
seq=seq,
t_offset_ms=0,
speaker=speaker,
text=payload,
))

40
services/hold_slayer.py
View File

@@ -56,6 +56,29 @@ class HoldSlayerService:
self.settings = settings
self.tts = tts
async def _service_error(self, call_id: str, service: str, error: Exception) -> None:
"""Surface a failed dependency as a typed event, not silence."""
logger.error(f"⚠️ {service} failed for {call_id}: {error}")
try:
await self.gateway.event_bus.publish(GatewayEvent(
type=EventType.ERROR,
call_id=call_id,
data={"service": service, "error": str(error)},
message=f"⚠️ {service} failed: {error}",
))
except Exception:
pass
async def _transcribe(
self, call_id: str, audio: bytes, prompt: Optional[str] = None
) -> str:
"""Transcribe with an explicit empty-string fallback on failure."""
try:
return await self.transcription.transcribe(audio, prompt=prompt)
except Exception as e:
await self._service_error(call_id, "transcription", e)
return ""
async def run(
self,
call: ActiveCall,
@@ -309,9 +332,10 @@ class HoldSlayerService:
AudioClassification.IVR_PROMPT,
AudioClassification.LIVE_HUMAN,
):
transcript = await self.transcription.transcribe(
transcript = await self._transcribe(
call.id,
audio_chunk,
prompt="Phone IVR menu, customer service, press 1 for..."
prompt="Phone IVR menu, customer service, press 1 for...",
)
if transcript:
await self.call_manager.add_transcript(call.id, transcript)
@@ -429,7 +453,7 @@ class HoldSlayerService:
# Check for human
if result.audio_type == AudioClassification.LIVE_HUMAN:
# Verify with transcription
transcript = await self.transcription.transcribe(audio_chunk)
transcript = await self._transcribe(call.id, audio_chunk)
if transcript:
await self.call_manager.add_transcript(call.id, transcript)
# If we got meaningful speech, it's probably a real person
@@ -491,7 +515,7 @@ class HoldSlayerService:
continue
# Transcribe
transcript = await self.transcription.transcribe(audio_chunk)
transcript = await self._transcribe(call.id, audio_chunk)
if not transcript:
continue
@@ -545,7 +569,7 @@ class HoldSlayerService:
AudioClassification.IVR_PROMPT,
AudioClassification.LIVE_HUMAN,
):
text = await self.transcription.transcribe(audio_chunk)
text = await self._transcribe(call.id, audio_chunk)
if text:
transcript_parts.append(text)
@@ -713,7 +737,11 @@ class HoldSlayerService:
os.close(fd)
try:
ok = await self.tts.synthesize_to_file(text, tmp_path)
try:
ok = await self.tts.synthesize_to_file(text, tmp_path)
except Exception as e:
await self._service_error(call.id, "tts", e)
return False
if not ok:
logger.warning(f"🗣️ TTS synthesis returned no audio for: '{text[:60]}'")
return False

9
services/llm_client.py
View File

@@ -327,15 +327,14 @@ class LLMClient:
except httpx.HTTPStatusError as e:
self._total_errors += 1
logger.error(f"LLM API error: {e.response.status_code} {e.response.text[:200]}")
return ""
raise
except httpx.TimeoutException:
self._total_errors += 1
logger.error(f"LLM API timeout after {self.timeout}s")
return ""
except Exception as e:
raise
except Exception:
self._total_errors += 1
logger.error(f"LLM client error: {e}")
return ""
raise
@staticmethod
def _parse_json_response(text: str) -> dict[str, Any]:

41
services/notification.py
View File

@@ -67,7 +67,6 @@ class NotificationService:
self._event_bus = event_bus
self._settings = settings
self._task: Optional[asyncio.Task] = None
self._sms_sender: Optional[Any] = None
# Track what we've already notified (avoid spam)
self._notified: dict[str, set[str]] = {} # call_id -> set of event types
@@ -214,43 +213,3 @@ class NotificationService:
# WebSocket notifications go through the event bus
# (the WebSocket handler in the API reads from EventBus directly)
# SMS for critical notifications
if (
notification.priority == NotificationPriority.CRITICAL
and self._settings.notify_sms_number
):
await self._send_sms(notification)
async def _send_sms(self, notification: Notification) -> None:
"""
Send an SMS notification.
Uses a simple HTTP-based SMS gateway. In production,
this would use Twilio, AWS SNS, or similar.
"""
phone = self._settings.notify_sms_number
if not phone:
return
try:
import httpx
# Generic webhook-based SMS (configure your provider)
# This is a placeholder — wire up your preferred SMS provider
logger.info(f"📱 SMS → {phone}: {notification.title}")
# Example: Twilio-style API
# async with httpx.AsyncClient() as client:
# await client.post(
# "https://api.twilio.com/2010-04-01/Accounts/.../Messages.json",
# data={
# "To": phone,
# "From": self._settings.sip_trunk.did,
# "Body": f"{notification.title}\n{notification.message}",
# },
# auth=(account_sid, auth_token),
# )
except Exception as e:
logger.error(f"SMS send failed: {e}")

89
services/receptionist.py
View File

@@ -203,6 +203,19 @@ class ReceptionistService:
# State machine steps
# ----------------------------------------------------------------
async def _service_error(self, call_id: str, service: str, error: Exception) -> None:
"""Surface a failed dependency as a typed event, not silence."""
logger.error(f"⚠️ {service} failed for {call_id}: {error}")
try:
await self.gateway.event_bus.publish(GatewayEvent(
type=EventType.ERROR,
call_id=call_id,
data={"service": service, "error": str(error)},
message=f"⚠️ {service} failed: {error}",
))
except Exception:
pass
async def _greet(self, call: ActiveCall, sip_leg_id: str) -> None:
await self.gateway.event_bus.publish(GatewayEvent(
type=EventType.RECEPTIONIST_GREETING,
@@ -250,7 +263,11 @@ class ReceptionistService:
if not audio or self.transcription is None:
return ""
return await self.transcription.transcribe(bytes(audio))
try:
return await self.transcription.transcribe(bytes(audio))
except Exception as e:
await self._service_error(call.id, "transcription", e)
return ""
async def _classify(
self,
@@ -291,7 +308,7 @@ class ReceptionistService:
system=self.settings.llm_persona,
)
except Exception as e:
logger.warning(f"Receptionist LLM classify failed: {e}")
await self._service_error(call.id, "llm", e)
return {
"intent": transcript,
"urgency": "normal",
@@ -304,10 +321,14 @@ class ReceptionistService:
routing_decision: Optional[RoutingDecision],
classification: dict,
) -> RoutingAction:
"""Rules win on conflict; otherwise use the LLM's recommendation."""
if routing_decision and routing_decision.action.type not in (
RoutingActionType.TAKE_MESSAGE,
):
"""Rules win on conflict; otherwise use the LLM's recommendation.
A decision counts as a rule only when one actually matched
(matched_rule_id set) — the no-rule default is take_message and
must stay overridable by the LLM. A matched TAKE_MESSAGE rule
wins like any other rule.
"""
if routing_decision and routing_decision.matched_rule_id:
return routing_decision.action
recommended = (classification.get("recommended_action") or "ring").lower()
@@ -347,36 +368,42 @@ class ReceptionistService:
call.id, media_pipeline=media, leg_ids=[sip_leg_id]
)
try:
await asyncio.sleep(self.settings.message_max_seconds)
# Record up to the cap, but stop early once the caller hangs
# up (leg termination ends the call via the leg-state wiring).
deadline = _time.monotonic() + self.settings.message_max_seconds
while _time.monotonic() < deadline:
await asyncio.sleep(1.0)
if self.gateway.call_manager.get_call(call.id) is None:
break
finally:
session = await recording_svc.stop_recording(
call.id, media_pipeline=media
)
message_text = ""
rec_path = session.filepath_mixed if session else None
if rec_path and Path(rec_path).exists() and self.transcription is not None:
try:
audio_bytes = Path(rec_path).read_bytes()
message_text = await self.transcription.transcribe(audio_bytes)
except Exception as e:
logger.warning(f"Receptionist transcribe failed: {e}")
message_text = ""
rec_path = session.filepath_mixed if session else None
if rec_path and Path(rec_path).exists() and self.transcription is not None:
try:
audio_bytes = Path(rec_path).read_bytes()
message_text = await self.transcription.transcribe(audio_bytes)
except Exception as e:
await self._service_error(call.id, "transcription", e)
if message_text:
call.transcript_chunks.append(f"caller_message: {message_text}")
if message_text:
call.transcript_chunks.append(f"caller_message: {message_text}")
await self.gateway.event_bus.publish(GatewayEvent(
type=EventType.RECEPTIONIST_MESSAGE_SAVED,
call_id=call.id,
data={
"path": rec_path,
"transcript": message_text,
"caller": call.remote_number,
},
message=f"📥 Message saved from {call.remote_number}",
))
await self.gateway.event_bus.publish(GatewayEvent(
type=EventType.RECEPTIONIST_MESSAGE_SAVED,
call_id=call.id,
data={
"path": rec_path,
"transcript": message_text,
"caller": call.remote_number,
},
message=f"📥 Message saved from {call.remote_number}",
))
await self._hangup(call, sip_leg_id)
await self._hangup(call, sip_leg_id)
# ----------------------------------------------------------------
# Helpers
@@ -394,7 +421,11 @@ class ReceptionistService:
fd, tmp_path = tempfile.mkstemp(suffix=".wav", prefix=f"recept_{call.id}_")
os.close(fd)
try:
ok = await tts.synthesize_to_file(text, tmp_path)
try:
ok = await tts.synthesize_to_file(text, tmp_path)
except Exception as e:
await self._service_error(call.id, "tts", e)
return
if not ok:
return
await media.play_wav(sip_leg_id, tmp_path)

54
services/recording.py
View File

@@ -91,6 +91,7 @@ class RecordingService:
filepath_agent=filepath_agent,
started_at=datetime.now(),
sample_rate=self._sample_rate,
leg_ids=leg_ids,
)
# Start PJSUA2 recording if media pipeline is available
@@ -159,26 +160,38 @@ class RecordingService:
@staticmethod
async def _persist_recording(session: "RecordingSession") -> None:
"""Write a recordings row for this session. Failures are non-fatal."""
try:
import uuid as _uuid
from db.database import RecordingRecord, get_session_factory
"""Write a recordings row for this session, with bounded retry.
async with get_session_factory()() as db:
db.add(RecordingRecord(
id=f"rec_{_uuid.uuid4().hex[:10]}",
call_id=session.call_id,
path=session.filepath_mixed or "",
format="wav",
duration_s=float(session.duration_seconds or 0),
size_bytes=int(session.file_size_bytes or 0),
channels=1,
started_at=session.started_at,
ended_at=session.stopped_at,
))
await db.commit()
except Exception as e:
logger.warning(f"Recording persistence failed: {e}")
Non-fatal for the call, but a lost row means the dashboard can
never find the WAV — so failures log at ERROR, not warning.
"""
import uuid as _uuid
from db.database import RecordingRecord, session_scope
for attempt in range(3):
try:
async with session_scope() as db:
db.add(RecordingRecord(
id=f"rec_{_uuid.uuid4().hex[:10]}",
call_id=session.call_id,
path=session.filepath_mixed or "",
format="wav",
duration_s=float(session.duration_seconds or 0),
size_bytes=int(session.file_size_bytes or 0),
channels=1,
started_at=session.started_at,
ended_at=session.stopped_at,
))
return
except Exception as e:
if attempt == 2:
logger.error(
f"Recording row lost for {session.call_id} "
f"(path={session.filepath_mixed}): {e}"
)
return
await asyncio.sleep(2 ** attempt)
async def _recording_timeout(self, call_id: str) -> None:
"""Auto-stop recording after max duration."""
@@ -239,6 +252,7 @@ class RecordingSession:
filepath_agent: Optional[str] = None,
started_at: Optional[datetime] = None,
sample_rate: int = 16000,
leg_ids: Optional[list[str]] = None,
):
self.call_id = call_id
self.filepath_mixed = filepath_mixed
@@ -249,7 +263,7 @@ class RecordingSession:
self.duration_seconds: Optional[int] = None
self.file_size_bytes: Optional[int] = None
self.sample_rate = sample_rate
self._leg_ids: list[str] = []
self._leg_ids: list[str] = list(leg_ids or [])
def to_dict(self) -> dict:
return {

50
services/transcription.py
View File

@@ -27,6 +27,8 @@ class TranscriptionService:
def __init__(self, settings: SpeachesSettings):
self.settings = settings
self._client: Optional[httpx.AsyncClient] = None
# Last-known reachability, surfaced by /health (None = no requests yet)
self.available: Optional[bool] = None
async def _get_client(self) -> httpx.AsyncClient:
"""Get or create the HTTP client."""
@@ -60,6 +62,9 @@ class TranscriptionService:
# Convert raw PCM to WAV format for the API
wav_data = self._pcm_to_wav(audio_data)
# Raises on failure — callers decide the per-call fallback and
# publish a service-error event; swallowing here made a down
# Speaches look like "the AI is deciding badly".
try:
response = await client.post(
"/v1/audio/transcriptions",
@@ -72,44 +77,13 @@ class TranscriptionService:
},
)
response.raise_for_status()
text = response.text.strip()
logger.debug(f"Transcription: '{text}'")
return text
except httpx.HTTPStatusError as e:
logger.error(f"Speaches API error: {e.response.status_code} {e.response.text}")
return ""
except httpx.ConnectError:
logger.error(f"Cannot connect to Speaches at {self.settings.url}")
return ""
except Exception as e:
logger.error(f"Transcription failed: {e}")
return ""
async def transcribe_stream(
self,
audio_data: bytes,
language: str = "en",
):
"""
Stream transcription — for real-time results.
Uses Speaches streaming endpoint if available,
falls back to chunked transcription.
Yields:
str: Partial transcription chunks
"""
# For now, do chunked transcription
# TODO: Implement WebSocket streaming when Speaches supports it
chunk_size = 16000 * 2 * 3 # 3 seconds of 16kHz 16-bit mono
for i in range(0, len(audio_data), chunk_size):
chunk = audio_data[i:i + chunk_size]
if len(chunk) > 0:
text = await self.transcribe(chunk, language)
if text:
yield text
except Exception:
self.available = False
raise
self.available = True
text = response.text.strip()
logger.debug(f"Transcription: '{text}'")
return text
async def close(self) -> None:
"""Close the HTTP client."""

20
services/tts.py
View File

@@ -22,6 +22,8 @@ class TTSService:
def __init__(self, settings: TTSSettings):
self.settings = settings
self._client: Optional[httpx.AsyncClient] = None
# Last-known reachability, surfaced by /health (None = no requests yet)
self.available: Optional[bool] = None
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None or self._client.is_closed:
@@ -54,19 +56,17 @@ class TTSService:
"sample_rate": self.settings.sample_rate,
}
# Raises on failure — callers decide the per-call fallback and
# publish a service-error event; swallowing here made a down
# Rhema look like "the AI went quiet".
try:
response = await client.post("/v1/audio/speech", json=body)
response.raise_for_status()
return response.content
except httpx.HTTPStatusError as e:
logger.error(f"Rhema TTS error: {e.response.status_code} {e.response.text}")
return b""
except httpx.ConnectError:
logger.error(f"Cannot connect to Rhema at {self.settings.base_url}")
return b""
except Exception as e:
logger.error(f"TTS synthesis failed: {e}")
return b""
except Exception:
self.available = False
raise
self.available = True
return response.content
async def synthesize_to_file(
self,