From 5880b59872ee1b2d47763e9995478720cef16eec Mon Sep 17 00:00:00 2001 From: Robert Helewka Date: Thu, 9 Jul 2026 19:53:36 -0400 Subject: [PATCH 1/5] fix: enforce thread ownership at the Sippy/PJSUA2 boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three thread domains were mutating shared dicts with no locks: Sippy's ED thread wrote _legs/_registered_devices directly from SIP handlers, the asyncio loop wrote them from make_call/hangup, and run_in_executor(None, ...) had default-pool threads driving sippy UA objects. AudioTap.feed() pushed into an asyncio.Queue (not thread-safe) from the PJSUA2 thread. New ownership rule, enforced structurally: - The asyncio loop owns all app-visible state; the only mutator is the new _on_engine_event funnel. Sippy handlers extract plain strings on the ED thread and post via run_coroutine_threadsafe. - The ED thread owns sippy objects plus _ed_ua_to_leg/_ed_leg_to_ua; loop-side commands (INVITE/BYE/DTMF/trunk register) hop over via ED2.callFromThread. UA references no longer live on SipCallLeg. - AudioTap captures its loop and feed() hops via call_soon_threadsafe. - Fix ED import: installed sippy 2.x exposes ED2, not ED — the old import could never start the event loop. Also: - Wire the never-connected on_leg_state_change callback: outbound ringing/connected/terminated now reaches CallManager; a call ends when its last leg terminates (transfers keep it alive). Adds CallManager.unmap_leg/legs_for_call. - AudioClassifier.classify(): async entry that runs the FFT work in asyncio.to_thread and updates history on the loop — all four hold_slayer call sites now route through it, fixing both the loop-blocking and the 2-of-4 history gap. DTMF Goertzel loop replaced by the equivalent vectorized DFT-bin power. - Task hygiene: gateway.spawn() tracks hold-slayer/receptionist tasks and stop() cancels them; recording safety-timeout task is retained and cancelled on stop_recording; engine tracks incoming-call dispatch tasks. 10 new tests: funnel events from a foreign thread, auto-answer fallback, AudioTap cross-thread feed, classifier history, leg-state → call status (including no stomping of ON_HOLD), stop() cancellation. Co-Authored-By: Claude Fable 5 --- core/call_manager.py | 8 + core/gateway.py | 52 +++- core/media_pipeline.py | 14 +- core/sippy_engine.py | 465 ++++++++++++++++++++--------------- services/audio_classifier.py | 33 ++- services/hold_slayer.py | 10 +- services/recording.py | 13 +- tests/test_concurrency.py | 173 +++++++++++++ 8 files changed, 548 insertions(+), 220 deletions(-) create mode 100644 tests/test_concurrency.py diff --git a/core/call_manager.py b/core/call_manager.py index 1eb229e..3ea6ee4 100644 --- a/core/call_manager.py +++ b/core/call_manager.py @@ -180,6 +180,14 @@ class CallManager: """Map a SIP leg ID to a call ID.""" self._call_legs[sip_leg_id] = call_id + def unmap_leg(self, sip_leg_id: str) -> None: + """Remove a SIP leg mapping (leg terminated).""" + self._call_legs.pop(sip_leg_id, None) + + def legs_for_call(self, call_id: str) -> list[str]: + """All SIP leg IDs currently mapped to a call.""" + return [leg for leg, cid in self._call_legs.items() if cid == call_id] + def get_call_for_leg(self, sip_leg_id: str) -> Optional[ActiveCall]: """Look up which call a SIP leg belongs to.""" call_id = self._call_legs.get(sip_leg_id) diff --git a/core/gateway.py b/core/gateway.py index 16bd2e4..da3178b 100644 --- a/core/gateway.py +++ b/core/gateway.py @@ -5,6 +5,7 @@ Ties together SIP engine, call manager, event bus, and all services. This is the top-level object that FastAPI and MCP talk to. """ +import asyncio import logging from datetime import datetime from typing import Optional @@ -57,6 +58,7 @@ def _build_sip_engine(settings: Settings, gateway: "AIPSTNGateway") -> SIPEngine domain=gw_sip.domain, did=trunk.did, media_pipeline=gateway.media_pipeline, + on_leg_state_change=gateway._on_sip_leg_state, on_device_registered=gateway._on_sip_device_registered, on_incoming_call=gateway._on_sip_incoming_call, ) @@ -101,9 +103,20 @@ class AIPSTNGateway: # Device registry (loaded from DB on start) self._devices: dict[str, Device] = {} + # Background tasks (per-call services, receptionist sessions) — + # tracked so shutdown can cancel them and GC can't drop them + self._tasks: set[asyncio.Task] = set() + # Startup time self._started_at: Optional[datetime] = None + def spawn(self, coro, name: str) -> asyncio.Task: + """Launch a tracked background task.""" + task = asyncio.get_running_loop().create_task(coro, name=name) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + return task + @classmethod def from_config(cls, sip_engine: Optional[SIPEngine] = None) -> "AIPSTNGateway": """Create gateway from environment config.""" @@ -176,6 +189,12 @@ class AIPSTNGateway: """Gracefully shut down.""" logger.info("Shutting down AI PSTN Gateway...") + # Cancel per-call background tasks before tearing down their deps + for task in list(self._tasks): + task.cancel() + if self._tasks: + await asyncio.gather(*self._tasks, return_exceptions=True) + # End all active calls for call_id in list(self.call_manager.active_calls.keys()): call = self.call_manager.get_call(call_id) @@ -278,8 +297,7 @@ class AIPSTNGateway: tts=self._tts, ) # Launch as background task — don't block - import asyncio - asyncio.create_task( + self.spawn( hold_slayer.run(call, sip_leg_id, call_flow_id), name=f"holdslayer_{call.id}", ) @@ -366,6 +384,33 @@ class AIPSTNGateway: if device: logger.info(f"📱 Device unregistered: {device.name}") + async def _on_sip_leg_state(self, leg_id: str, state: str) -> None: + """ + SIP leg state change from the engine (already on the loop). + + Maps leg transitions onto call status. Status only moves + forward from the dialing phase — hold-slayer/receptionist + states (ON_HOLD, NAVIGATING_IVR, …) are never stomped by a + late ringing/connected signal from a second leg. + """ + call = self.call_manager.get_call_for_leg(leg_id) + if call is None: + return + + if state == "ringing" and call.status == CallStatus.INITIATING: + await self.call_manager.update_status(call.id, CallStatus.RINGING) + elif state == "connected" and call.status in ( + CallStatus.INITIATING, + CallStatus.RINGING, + ): + await self.call_manager.update_status(call.id, CallStatus.CONNECTED) + elif state == "terminated": + self.call_manager.unmap_leg(leg_id) + # End the call only when its last leg is gone (a transfer + # keeps the call alive on the device leg) + if not self.call_manager.legs_for_call(call.id): + await self.call_manager.end_call(call.id) + async def _on_sip_device_registered( self, aor: str, contact: str, expires: int ) -> None: @@ -487,8 +532,7 @@ class AIPSTNGateway: # Hand off to the AI Receptionist if self._receptionist is not None and self.settings.receptionist.enabled: - import asyncio as _asyncio - _asyncio.create_task( + self.spawn( self._receptionist.handle(call, leg_id, decision), name=f"receptionist_{call.id}", ) diff --git a/core/media_pipeline.py b/core/media_pipeline.py index 2edf05b..19d3b16 100644 --- a/core/media_pipeline.py +++ b/core/media_pipeline.py @@ -52,11 +52,23 @@ class AudioTap: self._buffer: asyncio.Queue[bytes] = asyncio.Queue(maxsize=500) self._active = True self._pjsua2_port = None # PJSUA2 AudioMediaPort for tapping + # asyncio.Queue is not thread-safe; feed() hops onto this loop + try: + self._loop: Optional[asyncio.AbstractEventLoop] = asyncio.get_running_loop() + except RuntimeError: + self._loop = None def feed(self, pcm_data: bytes) -> None: - """Feed PCM audio data into the tap (called from PJSUA2 thread).""" + """Feed PCM audio data into the tap (called from the PJSUA2 thread).""" if not self._active: return + if self._loop is not None: + self._loop.call_soon_threadsafe(self._enqueue, pcm_data) + else: + self._enqueue(pcm_data) + + def _enqueue(self, pcm_data: bytes) -> None: + """Queue a frame on the owning loop, dropping oldest on overflow.""" try: self._buffer.put_nowait(pcm_data) except asyncio.QueueFull: diff --git a/core/sippy_engine.py b/core/sippy_engine.py index 7e6a67b..4c56a83 100644 --- a/core/sippy_engine.py +++ b/core/sippy_engine.py @@ -9,11 +9,22 @@ Architecture: Sippy B2BUA → SIP signaling (call control, registration, DTMF) PJSUA2 → Media anchor (conference bridge, audio tapping, recording) -Sippy B2BUA runs in its own thread (it has its own event loop). -We bridge async/sync via run_in_executor. +Thread-ownership rule: + - The asyncio loop owns all application-visible state: `_legs`, + `_bridges`, `_registered_devices`, `_trunk_registered`, and the + media pipeline. The ONLY place that state is mutated is + `_on_engine_event`, which runs on the loop. + - The Sippy ED thread owns every sippy object (UAs, transactions) + plus the `_ed_*` maps. Sippy objects are only touched by code + scheduled onto that thread via `_run_on_sippy`. + - `leg_id` strings are the only tokens that cross the boundary, + carried by `_post_from_ed` (Sippy → loop, via + run_coroutine_threadsafe) and `_run_on_sippy` (loop → Sippy, via + ED2.callFromThread). """ import asyncio +import inspect import logging import threading import uuid @@ -30,15 +41,15 @@ logger = logging.getLogger(__name__) # ================================================================ class SipCallLeg: - """Tracks a single SIP call leg managed by Sippy.""" + """Tracks a single SIP call leg. Owned by the asyncio loop.""" def __init__(self, leg_id: str, direction: str, remote_uri: str): self.leg_id = leg_id self.direction = direction # "outbound" or "inbound" self.remote_uri = remote_uri self.state = "init" # init, trying, ringing, connected, terminated - self.sippy_ua = None # Sippy UA object reference self.media_port: Optional[int] = None # PJSUA2 conf bridge port + self.pending_sdp: Optional[str] = None # inbound INVITE SDP, until answered self.dtmf_buffer: list[str] = [] def __repr__(self): @@ -65,75 +76,44 @@ class SippyCallController: """ Handles Sippy B2BUA callbacks for a single call leg. - Sippy B2BUA uses a callback model — when SIP events happen - (180 Ringing, 200 OK, BYE, etc.), the corresponding method - is called on this controller. + Runs entirely on the Sippy ED thread. It holds only the leg_id + token and forwards every state change to the asyncio loop via + the engine's event funnel — it never touches loop-owned state. """ - def __init__(self, leg: SipCallLeg, engine: "SippyEngine"): - self.leg = leg + def __init__(self, leg_id: str, engine: "SippyEngine"): + self.leg_id = leg_id self.engine = engine def on_trying(self): """100 Trying received.""" - self.leg.state = "trying" - logger.debug(f" {self.leg.leg_id}: 100 Trying") + logger.debug(f" {self.leg_id}: 100 Trying") + self.engine._post_from_ed("leg_state", {"leg_id": self.leg_id, "state": "trying"}) def on_ringing(self, ringing_code: int = 180): """180 Ringing / 183 Session Progress received.""" - self.leg.state = "ringing" - logger.info(f" {self.leg.leg_id}: {ringing_code} Ringing") - if self.engine._on_leg_state_change: - self.engine._loop.call_soon_threadsafe( - self.engine._on_leg_state_change, self.leg.leg_id, "ringing" - ) + logger.info(f" {self.leg_id}: {ringing_code} Ringing") + self.engine._post_from_ed("leg_state", {"leg_id": self.leg_id, "state": "ringing"}) def on_connected(self, sdp_body: Optional[str] = None): """200 OK — call connected, media negotiated.""" - self.leg.state = "connected" - logger.info(f" {self.leg.leg_id}: Connected") - - # Extract remote RTP endpoint from SDP for PJSUA2 media bridge - if sdp_body and self.engine.media_pipeline: - try: - remote_rtp = self.engine._parse_sdp_rtp_endpoint(sdp_body) - if remote_rtp: - port = self.engine.media_pipeline.add_remote_stream( - self.leg.leg_id, - remote_rtp["host"], - remote_rtp["port"], - remote_rtp["codec"], - ) - self.leg.media_port = port - except Exception as e: - logger.error(f" Failed to set up media for {self.leg.leg_id}: {e}") - - if self.engine._on_leg_state_change: - self.engine._loop.call_soon_threadsafe( - self.engine._on_leg_state_change, self.leg.leg_id, "connected" - ) + logger.info(f" {self.leg_id}: Connected") + self.engine._post_from_ed( + "leg_state", {"leg_id": self.leg_id, "state": "connected", "sdp": sdp_body} + ) def on_disconnected(self, reason: str = ""): """BYE received or call terminated.""" - self.leg.state = "terminated" - logger.info(f" {self.leg.leg_id}: Disconnected ({reason})") - - # Clean up media - if self.engine.media_pipeline and self.leg.media_port is not None: - try: - self.engine.media_pipeline.remove_stream(self.leg.leg_id) - except Exception as e: - logger.error(f" Failed to clean up media for {self.leg.leg_id}: {e}") - - if self.engine._on_leg_state_change: - self.engine._loop.call_soon_threadsafe( - self.engine._on_leg_state_change, self.leg.leg_id, "terminated" - ) + logger.info(f" {self.leg_id}: Disconnected ({reason})") + self.engine._ed_forget_leg(self.leg_id) + self.engine._post_from_ed( + "leg_state", {"leg_id": self.leg_id, "state": "terminated", "reason": reason} + ) def on_dtmf(self, digit: str): """DTMF digit received (RFC 2833 or SIP INFO).""" - self.leg.dtmf_buffer.append(digit) - logger.debug(f" {self.leg.leg_id}: DTMF '{digit}'") + logger.debug(f" {self.leg_id}: DTMF '{digit}'") + self.engine._post_from_ed("dtmf", {"leg_id": self.leg_id, "digit": digit}) # ================================================================ @@ -190,17 +170,140 @@ class SippyEngine(SIPEngine): self._on_incoming_call = on_incoming_call self._loop: Optional[asyncio.AbstractEventLoop] = None - # State + # Loop-owned state (mutated only in _on_engine_event and the + # async methods below, all of which run on the loop) self._ready = False self._trunk_registered = False self._legs: dict[str, SipCallLeg] = {} self._bridges: dict[str, SipBridge] = {} self._registered_devices: list[dict] = [] + self._tasks: set[asyncio.Task] = set() + + # ED-thread-owned state: sippy UA objects, only touched from + # the Sippy thread (handlers and _run_on_sippy closures) + self._ed_ua_to_leg: dict[Any, str] = {} + self._ed_leg_to_ua: dict[str, Any] = {} # Sippy B2BUA internals (set during start) self._sippy_global_config: dict[str, Any] = {} self._sippy_thread: Optional[threading.Thread] = None + # ================================================================ + # Thread-boundary crossing primitives + # ================================================================ + + def _post_from_ed(self, kind: str, data: dict) -> None: + """Sippy thread → loop: schedule the single state-mutation funnel.""" + if self._loop is None: + return + asyncio.run_coroutine_threadsafe(self._on_engine_event(kind, data), self._loop) + + def _run_on_sippy(self, fn: Callable[[], None]) -> None: + """Loop → Sippy thread: run fn where the sippy objects live.""" + try: + from sippy.Core.EventDispatcher import ED2 + except ImportError: + # Simulation mode — no sippy, no ED thread; run inline. + fn() + return + ED2.callFromThread(fn) + + def _ed_forget_leg(self, leg_id: str) -> None: + """Drop the ED-side UA maps for a leg (Sippy thread only).""" + ua = self._ed_leg_to_ua.pop(leg_id, None) + if ua is not None: + self._ed_ua_to_leg.pop(ua, None) + + def _spawn(self, coro, name: str) -> None: + """Track a background task so shutdown can cancel it.""" + task = asyncio.get_running_loop().create_task(coro, name=name) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + + async def _on_engine_event(self, kind: str, data: dict) -> None: + """ + The single funnel where Sippy-thread events mutate loop-owned + state. Everything here runs on the asyncio loop. + """ + if kind == "leg_state": + leg = self._legs.get(data["leg_id"]) + if leg is None: + return + state = data["state"] + leg.state = state + + if state == "connected": + sdp = data.get("sdp") + if sdp and self.media_pipeline: + try: + remote_rtp = self._parse_sdp_rtp_endpoint(sdp) + if remote_rtp: + leg.media_port = self.media_pipeline.add_remote_stream( + leg.leg_id, + remote_rtp["host"], + remote_rtp["port"], + remote_rtp["codec"], + ) + except Exception as e: + logger.error(f" Failed to set up media for {leg.leg_id}: {e}") + elif state == "terminated": + if self.media_pipeline and leg.media_port is not None: + try: + self.media_pipeline.remove_stream(leg.leg_id) + except Exception as e: + logger.error(f" Failed to clean up media for {leg.leg_id}: {e}") + leg.media_port = None + + if self._on_leg_state_change: + result = self._on_leg_state_change(leg.leg_id, state) + if inspect.isawaitable(result): + await result + + elif kind == "incoming_invite": + leg = SipCallLeg(data["leg_id"], "inbound", data["from_uri"]) + leg.pending_sdp = data.get("sdp") + self._legs[leg.leg_id] = leg + if self._on_incoming_call: + self._spawn( + self._on_incoming_call(data["from_uri"], data["to_uri"], leg.leg_id), + name=f"incoming_{leg.leg_id}", + ) + else: + # No routing wired — preserve the historical auto-answer + await self.accept_inbound(leg.leg_id) + + elif kind == "register": + existing = next( + (d for d in self._registered_devices if d.get("aor") == data["aor"]), + None, + ) + if existing: + existing["contact"] = data["contact"] + existing["expires"] = data["expires"] + else: + self._registered_devices.append({ + "aor": data["aor"], + "contact": data["contact"], + "expires": data["expires"], + }) + if self._on_device_registered: + await self._on_device_registered( + data["aor"], data["contact"], data["expires"] + ) + + elif kind == "deregister": + self._registered_devices = [ + d for d in self._registered_devices if d.get("aor") != data["aor"] + ] + + elif kind == "dtmf": + leg = self._legs.get(data["leg_id"]) + if leg: + leg.dtmf_buffer.append(data["digit"]) + + elif kind == "trunk_registered": + self._trunk_registered = data["registered"] + # ================================================================ # Lifecycle # ================================================================ @@ -212,7 +315,6 @@ class SippyEngine(SIPEngine): try: from sippy.SipConf import SipConf - from sippy.SipTransactionManager import SipTransactionManager # Configure Sippy SipConf.my_address = self._sip_address @@ -226,7 +328,7 @@ class SippyEngine(SIPEngine): } # Start Sippy's SIP transaction manager in a background thread - # Sippy uses its own event loop (Twisted reactor or custom loop) + # Sippy uses its own event loop (the ED2 event dispatcher) self._sippy_thread = threading.Thread( target=self._run_sippy_loop, name="sippy-b2bua", @@ -254,8 +356,8 @@ class SippyEngine(SIPEngine): def _run_sippy_loop(self): """Run Sippy B2BUA's event loop in a dedicated thread.""" try: + from sippy.Core.EventDispatcher import ED2 from sippy.SipTransactionManager import SipTransactionManager - from sippy.Timeout import Timeout # Initialize Sippy's transaction manager stm = SipTransactionManager(self._sippy_global_config, self._handle_sippy_request) @@ -263,11 +365,9 @@ class SippyEngine(SIPEngine): logger.info(" Sippy transaction manager started") - # Sippy will block here in its event loop - # For the Twisted-based version, this runs the reactor - # For the asyncore version, this runs asyncore.loop() - from sippy.Core.EventDispatcher import ED - ED.loop() + # Sippy blocks here dispatching its event loop; callbacks + # injected via ED2.callFromThread run inside this loop. + ED2.loop() except Exception as e: logger.error(f" Sippy event loop crashed: {e}") @@ -294,10 +394,9 @@ class SippyEngine(SIPEngine): """ Handle an incoming SIP REGISTER from a phone or softphone. - Extracts the AOR (address of record) from the To header, records - the contact and expiry, and sends a 200 OK. The gateway's - register_device() is called asynchronously via the event loop so - the phone gets an extension and SIP URI assigned automatically. + Runs on the Sippy thread: parses the request, replies 200 OK, + and posts the registration to the loop funnel, which owns the + device list and notifies the gateway. """ try: to_uri = str(req.getHFBody("to").getUri()) @@ -309,34 +408,13 @@ class SippyEngine(SIPEngine): logger.info(f" SIP REGISTER: {to_uri} contact={contact_uri} expires={expires}") if expires == 0: - # De-registration - self._registered_devices = [ - d for d in self._registered_devices - if d.get("aor") != to_uri - ] - logger.info(f" De-registered: {to_uri}") + self._post_from_ed("deregister", {"aor": to_uri}) else: - # Update or add registration record - existing = next( - (d for d in self._registered_devices if d.get("aor") == to_uri), - None, - ) - if existing: - existing["contact"] = contact_uri - existing["expires"] = expires - else: - self._registered_devices.append({ - "aor": to_uri, - "contact": contact_uri, - "expires": expires, - }) - - # Notify the gateway (async) so it can assign an extension - if self._loop: - self._loop.call_soon_threadsafe( - self._loop.create_task, - self._notify_registration(to_uri, contact_uri, expires), - ) + self._post_from_ed("register", { + "aor": to_uri, + "contact": contact_uri, + "expires": expires, + }) # Reply 200 OK req.sendResponse(200, "OK") @@ -348,52 +426,41 @@ class SippyEngine(SIPEngine): except Exception: pass - async def _notify_registration(self, aor: str, contact: str, expires: int): - """ - Async callback: tell the gateway about the newly registered device - so it can assign an extension if needed. - """ - if self._on_device_registered: - await self._on_device_registered(aor, contact, expires) - def _handle_incoming_invite(self, req, sip_t): - """Handle an incoming INVITE — create inbound call leg. + """Handle an incoming INVITE — surface an inbound call leg. - The gateway is notified via `on_incoming_call`; it decides - whether to answer (via `accept_inbound`) or reject the leg - based on routing rules. + Runs on the Sippy thread: extracts everything the loop needs + (URIs, SDP body) as plain strings and posts them. The gateway + decides whether to answer (via `accept_inbound`) or reject. """ from_uri = str(req.getHFBody("from").getUri()) to_uri = str(req.getHFBody("to").getUri()) + sdp = str(req.getBody()) if req.getBody() else None leg_id = f"leg_{uuid.uuid4().hex[:12]}" - leg = SipCallLeg(leg_id, "inbound", from_uri) - leg.sippy_ua = sip_t.ua if hasattr(sip_t, "ua") else None - leg.pending_invite = req - self._legs[leg_id] = leg + ua = sip_t.ua if hasattr(sip_t, "ua") else None + if ua is not None: + self._ed_ua_to_leg[ua] = leg_id + self._ed_leg_to_ua[leg_id] = ua logger.info(f" Incoming call: {from_uri} → {to_uri} (leg: {leg_id})") - # Surface to the gateway. If no callback is wired, fall back to - # auto-answer so we don't regress the previous behavior. - if self._on_incoming_call and self._loop: - asyncio.run_coroutine_threadsafe( - self._on_incoming_call(from_uri, to_uri, leg_id), - self._loop, - ) - else: - controller = SippyCallController(leg, self) - controller.on_connected(str(req.getBody()) if req.getBody() else None) + self._post_from_ed("incoming_invite", { + "leg_id": leg_id, + "from_uri": from_uri, + "to_uri": to_uri, + "sdp": sdp, + }) async def accept_inbound(self, leg_id: str) -> bool: """Answer a previously-surfaced inbound INVITE.""" leg = self._legs.get(leg_id) if not leg or leg.direction != "inbound": return False - req = getattr(leg, "pending_invite", None) - controller = SippyCallController(leg, self) - body = str(req.getBody()) if req and req.getBody() else None - controller.on_connected(body) + sdp, leg.pending_sdp = leg.pending_sdp, None + await self._on_engine_event( + "leg_state", {"leg_id": leg_id, "state": "connected", "sdp": sdp} + ) return True async def reject_inbound(self, leg_id: str, code: int = 603, reason: str = "Decline") -> bool: @@ -404,66 +471,66 @@ class SippyEngine(SIPEngine): logger.info(f" ⛔ Rejecting inbound leg {leg_id}: {code} {reason}") # Real SIP rejection would go through Sippy here; we just drop the leg # in stub mode so callers see the call terminate. + self._run_on_sippy(lambda: self._ed_forget_leg(leg_id)) return True def _handle_incoming_bye(self, req, sip_t): - """Handle incoming BYE — tear down call leg.""" - # Find the leg by Sippy's UA object - for leg in self._legs.values(): - if leg.sippy_ua and hasattr(sip_t, "ua") and leg.sippy_ua == sip_t.ua: - controller = SippyCallController(leg, self) - controller.on_disconnected("BYE received") - break + """Handle incoming BYE — tear down call leg (Sippy thread).""" + ua = sip_t.ua if hasattr(sip_t, "ua") else None + leg_id = self._ed_ua_to_leg.get(ua) if ua is not None else None + if leg_id: + SippyCallController(leg_id, self).on_disconnected("BYE received") def _handle_incoming_info(self, req, sip_t): - """Handle SIP INFO (DTMF via SIP INFO method).""" + """Handle SIP INFO (DTMF via SIP INFO method) on the Sippy thread.""" body = str(req.getBody()) if req.getBody() else "" if "dtmf" in body.lower() or "Signal=" in body: - # Extract DTMF digit from SIP INFO body + ua = sip_t.ua if hasattr(sip_t, "ua") else None + leg_id = self._ed_ua_to_leg.get(ua) if ua is not None else None + if not leg_id: + return for line in body.split("\n"): if line.startswith("Signal="): digit = line.split("=")[1].strip() - for leg in self._legs.values(): - if leg.sippy_ua and hasattr(sip_t, "ua") and leg.sippy_ua == sip_t.ua: - controller = SippyCallController(leg, self) - controller.on_dtmf(digit) - break + SippyCallController(leg_id, self).on_dtmf(digit) async def _register_trunk(self) -> None: """Register with the SIP trunk provider.""" - try: - from sippy.UA import UA - from sippy.SipRegistrationAgent import SipRegistrationAgent + logger.info(f" Registering with trunk: {self._trunk_host}:{self._trunk_port}") - logger.info(f" Registering with trunk: {self._trunk_host}:{self._trunk_port}") + def do_register(): + try: + from sippy.SipRegistrationAgent import SipRegistrationAgent - # Run registration in Sippy's thread - def do_register(): - try: - reg_agent = SipRegistrationAgent( - self._sippy_global_config, - f"sip:{self._trunk_username}@{self._trunk_host}", - f"sip:{self._trunk_host}:{self._trunk_port}", - auth_name=self._trunk_username, - auth_password=self._trunk_password, - ) - reg_agent.register() - self._trunk_registered = True - logger.info(" ✅ Trunk registration sent") - except Exception as e: - logger.error(f" ❌ Trunk registration failed: {e}") - self._trunk_registered = False + reg_agent = SipRegistrationAgent( + self._sippy_global_config, + f"sip:{self._trunk_username}@{self._trunk_host}", + f"sip:{self._trunk_host}:{self._trunk_port}", + auth_name=self._trunk_username, + auth_password=self._trunk_password, + ) + reg_agent.register() + logger.info(" ✅ Trunk registration sent") + self._post_from_ed("trunk_registered", {"registered": True}) + except ImportError: + logger.warning(" Sippy registration agent not available") + self._post_from_ed("trunk_registered", {"registered": False}) + except Exception as e: + logger.error(f" ❌ Trunk registration failed: {e}") + self._post_from_ed("trunk_registered", {"registered": False}) - await asyncio.get_event_loop().run_in_executor(None, do_register) - - except ImportError: - logger.warning(" Sippy registration agent not available") - self._trunk_registered = False + self._run_on_sippy(do_register) async def stop(self) -> None: """Gracefully shut down the SIP engine.""" logger.info("🔌 Stopping Sippy B2BUA...") + # Cancel in-flight incoming-call dispatch tasks + for task in list(self._tasks): + task.cancel() + if self._tasks: + await asyncio.gather(*self._tasks, return_exceptions=True) + # Hang up all active legs for leg_id in list(self._legs.keys()): try: @@ -473,8 +540,8 @@ class SippyEngine(SIPEngine): # Stop Sippy's event loop try: - from sippy.Core.EventDispatcher import ED - ED.breakLoop() + from sippy.Core.EventDispatcher import ED2 + ED2.breakLoop() except Exception: pass @@ -512,14 +579,16 @@ class SippyEngine(SIPEngine): logger.info(f"📞 Placing call: {from_uri} → {remote_uri} (leg: {leg_id})") - # Place the call via Sippy + # Generate SDP on the loop (allocate_rtp_port is lock-protected) + sdp_body = self._generate_sdp(leg_id) + def do_invite(): try: - from sippy.UA import UA - from sippy.SipCallId import SipCallId from sippy.CCEvents import CCEventTry + from sippy.SipCallId import SipCallId + from sippy.UA import UA - controller = SippyCallController(leg, self) + controller = SippyCallController(leg_id, self) # Create Sippy UA for this call ua = UA( @@ -527,10 +596,8 @@ class SippyEngine(SIPEngine): event_cb=controller, nh_address=(self._trunk_host, self._trunk_port), ) - leg.sippy_ua = ua - - # Generate SDP for the call - sdp_body = self._generate_sdp(leg_id) + self._ed_leg_to_ua[leg_id] = ua + self._ed_ua_to_leg[ua] = leg_id # Send INVITE event = CCEventTry( @@ -539,19 +606,19 @@ class SippyEngine(SIPEngine): ) ua.recvEvent(event) - leg.state = "trying" logger.info(f" INVITE sent for {leg_id}") + self._post_from_ed("leg_state", {"leg_id": leg_id, "state": "trying"}) except ImportError: # Sippy not installed — simulate for development logger.warning(f" Sippy not installed, simulating call for {leg_id}") - leg.state = "ringing" + self._post_from_ed("leg_state", {"leg_id": leg_id, "state": "ringing"}) except Exception as e: logger.error(f" Failed to send INVITE for {leg_id}: {e}") - leg.state = "terminated" + self._post_from_ed("leg_state", {"leg_id": leg_id, "state": "terminated"}) - await asyncio.get_event_loop().run_in_executor(None, do_invite) + self._run_on_sippy(do_invite) return leg_id async def hangup(self, call_leg_id: str) -> None: @@ -563,15 +630,18 @@ class SippyEngine(SIPEngine): def do_bye(): try: - if leg.sippy_ua: + ua = self._ed_leg_to_ua.get(call_leg_id) + if ua is not None: from sippy.CCEvents import CCEventDisconnect - leg.sippy_ua.recvEvent(CCEventDisconnect()) + ua.recvEvent(CCEventDisconnect()) except Exception as e: logger.error(f" Error sending BYE for {call_leg_id}: {e}") finally: - leg.state = "terminated" + self._ed_forget_leg(call_leg_id) - await asyncio.get_event_loop().run_in_executor(None, do_bye) + self._run_on_sippy(do_bye) + + leg.state = "terminated" # Clean up media if self.media_pipeline and leg.media_port is not None: @@ -595,13 +665,13 @@ class SippyEngine(SIPEngine): def do_dtmf(): try: - if leg.sippy_ua: - # Send via RFC 2833 (in-band RTP event) - # Sippy handles this through the UA's DTMF sender + ua = self._ed_leg_to_ua.get(call_leg_id) + if ua is not None: + # Send via SIP INFO through the UA + from sippy.CCEvents import CCEventInfo for digit in digits: - from sippy.CCEvents import CCEventInfo body = f"Signal={digit}\r\nDuration=160\r\n" - leg.sippy_ua.recvEvent(CCEventInfo(body=body)) + ua.recvEvent(CCEventInfo(body=body)) else: logger.warning(f" No UA for {call_leg_id}, DTMF not sent") except ImportError: @@ -609,7 +679,7 @@ class SippyEngine(SIPEngine): except Exception as e: logger.error(f" DTMF send error: {e}") - await asyncio.get_event_loop().run_in_executor(None, do_dtmf) + self._run_on_sippy(do_dtmf) # ================================================================ # Device Calls (for transfer) @@ -638,13 +708,15 @@ class SippyEngine(SIPEngine): logger.info(f"📱 Calling device: {device.name} ({device.sip_uri}) (leg: {leg_id})") + sdp_body = self._generate_sdp(leg_id) + def do_invite_device(): try: - from sippy.UA import UA from sippy.CCEvents import CCEventTry from sippy.SipCallId import SipCallId + from sippy.UA import UA - controller = SippyCallController(leg, self) + controller = SippyCallController(leg_id, self) # Parse device SIP URI for routing # sip:robert@192.168.1.100:5060 @@ -662,25 +734,24 @@ class SippyEngine(SIPEngine): event_cb=controller, nh_address=(host, port), ) - leg.sippy_ua = ua - - sdp_body = self._generate_sdp(leg_id) + self._ed_leg_to_ua[leg_id] = ua + self._ed_ua_to_leg[ua] = leg_id event = CCEventTry( (SipCallId(), f"sip:gateway@{self._domain}", device.sip_uri), body=sdp_body, ) ua.recvEvent(event) - leg.state = "trying" + self._post_from_ed("leg_state", {"leg_id": leg_id, "state": "trying"}) except ImportError: logger.warning(f" Sippy not installed, simulating device call for {leg_id}") - leg.state = "ringing" + self._post_from_ed("leg_state", {"leg_id": leg_id, "state": "ringing"}) except Exception as e: logger.error(f" Failed to call device {device.name}: {e}") - leg.state = "terminated" + self._post_from_ed("leg_state", {"leg_id": leg_id, "state": "terminated"}) - await asyncio.get_event_loop().run_in_executor(None, do_invite_device) + self._run_on_sippy(do_invite_device) return leg_id # ================================================================ diff --git a/services/audio_classifier.py b/services/audio_classifier.py index 76a3809..7b440ca 100644 --- a/services/audio_classifier.py +++ b/services/audio_classifier.py @@ -12,6 +12,7 @@ Uses spectral analysis (librosa/numpy) to classify audio without needing a trained ML model — just signal processing and heuristics. """ +import asyncio import logging import time from typing import Optional @@ -47,9 +48,23 @@ class AudioClassifier: self._window_samples = int(settings.window_seconds * SAMPLE_RATE) self._classification_history: list[AudioClassification] = [] + async def classify(self, audio_data: bytes) -> ClassificationResult: + """ + Classify a chunk off the event loop and record it in the history. + + The FFT/autocorrelation work is CPU-bound, so the pure + `classify_chunk` runs in a worker thread; the history update + happens back on the loop, keeping it single-threaded. This is + the call sites' entry point — routing every classification + through here is what keeps the history complete. + """ + result = await asyncio.to_thread(self.classify_chunk, audio_data) + self.update_history(result.audio_type) + return result + def classify_chunk(self, audio_data: bytes) -> ClassificationResult: """ - Classify a chunk of audio data. + Classify a chunk of audio data (pure, synchronous). Args: audio_data: Raw PCM audio (16-bit signed, 16kHz, mono) @@ -285,17 +300,15 @@ class AudioClassifier: (941, 1209): "*", (941, 1336): "0", (941, 1477): "#", (941, 1633): "D", } - # Compute power at each DTMF frequency + # Power at each DTMF frequency via the DFT bin (numerically equal + # to the Goertzel result s1² + s2² − coeff·s1·s2, but vectorized — + # the per-sample Python loop blocked for ~50ms per chunk) + n = np.arange(len(samples)) + def goertzel_power(freq: int) -> float: k = int(0.5 + len(samples) * freq / SAMPLE_RATE) - w = 2 * np.pi * k / len(samples) - coeff = 2 * np.cos(w) - s0, s1, s2 = 0.0, 0.0, 0.0 - for sample in samples: - s0 = sample + coeff * s1 - s2 - s2 = s1 - s1 = s0 - return float(s1 * s1 + s2 * s2 - coeff * s1 * s2) + bin_value = np.dot(samples, np.exp(-2j * np.pi * k * n / len(samples))) + return float(np.abs(bin_value) ** 2) # Find strongest low and high frequencies low_powers = [(f, goertzel_power(f)) for f in dtmf_freqs_low] diff --git a/services/hold_slayer.py b/services/hold_slayer.py index fd14240..d9cec27 100644 --- a/services/hold_slayer.py +++ b/services/hold_slayer.py @@ -323,8 +323,7 @@ class HoldSlayerService: continue # Classify the audio - classification = self.classifier.classify_chunk(audio_chunk) - self.classifier.update_history(classification.audio_type) + classification = await self.classifier.classify(audio_chunk) await self.call_manager.add_classification(call.id, classification) # Transcribe if it sounds like speech @@ -447,8 +446,7 @@ class HoldSlayerService: continue # Classify - result = self.classifier.classify_chunk(audio_chunk) - self.classifier.update_history(result.audio_type) + result = await self.classifier.classify(audio_chunk) await self.call_manager.add_classification(call.id, result) # Check for human @@ -508,7 +506,7 @@ class HoldSlayerService: continue # Classify first - result = self.classifier.classify_chunk(audio_chunk) + result = await self.classifier.classify(audio_chunk) if result.audio_type not in ( AudioClassification.IVR_PROMPT, AudioClassification.LIVE_HUMAN, @@ -560,7 +558,7 @@ class HoldSlayerService: if not audio_chunk: break - result = self.classifier.classify_chunk(audio_chunk) + result = await self.classifier.classify(audio_chunk) # If we're getting silence after speech, the menu prompt is done if result.audio_type == AudioClassification.SILENCE and transcript_parts: diff --git a/services/recording.py b/services/recording.py index e5d949b..f80b551 100644 --- a/services/recording.py +++ b/services/recording.py @@ -39,6 +39,7 @@ class RecordingService: self._max_recording_seconds = max_recording_seconds self._sample_rate = sample_rate self._active_recordings: dict[str, RecordingSession] = {} + self._timeout_tasks: dict[str, asyncio.Task] = {} self._metadata: list[dict] = [] async def start(self) -> None: @@ -101,8 +102,8 @@ class RecordingService: self._active_recordings[call_id] = session logger.info(f"🔴 Recording started: {call_id} → {filepath_mixed}") - # Safety timeout - asyncio.create_task( + # Safety timeout — tracked so it can be cancelled and isn't GC'd + self._timeout_tasks[call_id] = asyncio.create_task( self._recording_timeout(call_id), name=f"rec_timeout_{call_id}", ) @@ -115,6 +116,14 @@ class RecordingService: media_pipeline=None, ) -> Optional["RecordingSession"]: """Stop recording a call and finalize the WAV file.""" + timeout_task = self._timeout_tasks.pop(call_id, None) + if ( + timeout_task is not None + and timeout_task is not asyncio.current_task() + and not timeout_task.done() + ): + timeout_task.cancel() + session = self._active_recordings.pop(call_id, None) if not session: logger.warning(f" No active recording for {call_id}") diff --git a/tests/test_concurrency.py b/tests/test_concurrency.py new file mode 100644 index 0000000..570bae2 --- /dev/null +++ b/tests/test_concurrency.py @@ -0,0 +1,173 @@ +""" +Thread-ownership and task-hygiene tests. + +Covers the Sippy→loop event funnel (events posted from a foreign +thread mutate loop-owned state), the AudioTap thread-safe feed, the +off-loop classifier entry point, leg-state propagation into the +CallManager, and background-task cancellation on gateway stop. +""" + +import asyncio +import threading + +import numpy as np + +from config import ClassifierSettings, Settings +from core.gateway import AIPSTNGateway +from core.media_pipeline import AudioTap +from core.sippy_engine import SippyEngine +from models.call import CallStatus +from services.audio_classifier import AudioClassifier + + +def _engine_on_loop() -> SippyEngine: + """Engine wired to the running loop without starting the SIP stack.""" + engine = SippyEngine() + engine._loop = asyncio.get_running_loop() + return engine + + +def _post_from_thread(engine: SippyEngine, kind: str, data: dict) -> None: + """Post a funnel event from a foreign thread, like the Sippy ED thread.""" + t = threading.Thread(target=engine._post_from_ed, args=(kind, data)) + t.start() + t.join() + + +class TestEngineEventFunnel: + async def test_register_and_deregister_from_foreign_thread(self): + engine = _engine_on_loop() + + _post_from_thread(engine, "register", { + "aor": "sip:alice@gw", "contact": "sip:alice@10.0.0.5", "expires": 3600, + }) + await asyncio.sleep(0.05) + assert engine._registered_devices == [ + {"aor": "sip:alice@gw", "contact": "sip:alice@10.0.0.5", "expires": 3600} + ] + + # Re-register updates in place instead of duplicating + _post_from_thread(engine, "register", { + "aor": "sip:alice@gw", "contact": "sip:alice@10.0.0.9", "expires": 60, + }) + await asyncio.sleep(0.05) + assert len(engine._registered_devices) == 1 + assert engine._registered_devices[0]["contact"] == "sip:alice@10.0.0.9" + + _post_from_thread(engine, "deregister", {"aor": "sip:alice@gw"}) + await asyncio.sleep(0.05) + assert engine._registered_devices == [] + + async def test_incoming_invite_auto_answers_without_callback(self): + engine = _engine_on_loop() + _post_from_thread(engine, "incoming_invite", { + "leg_id": "leg_test1", + "from_uri": "sip:caller@pstn", + "to_uri": "sip:+15551234567@gw", + "sdp": None, + }) + await asyncio.sleep(0.05) + leg = engine._legs["leg_test1"] + assert leg.direction == "inbound" + assert leg.state == "connected" + + async def test_incoming_call_callback_runs_as_tracked_task(self): + engine = _engine_on_loop() + seen = asyncio.Event() + + async def on_incoming(from_uri, to_uri, leg_id): + seen.set() + + engine._on_incoming_call = on_incoming + _post_from_thread(engine, "incoming_invite", { + "leg_id": "leg_test2", "from_uri": "a", "to_uri": "b", "sdp": None, + }) + await asyncio.wait_for(seen.wait(), timeout=1.0) + assert engine._legs["leg_test2"].state == "init" # not auto-answered + + async def test_bye_terminates_leg_and_notifies(self): + engine = _engine_on_loop() + states: list[tuple[str, str]] = [] + engine._on_leg_state_change = lambda leg_id, state: states.append((leg_id, state)) + + _post_from_thread(engine, "incoming_invite", { + "leg_id": "leg_test3", "from_uri": "a", "to_uri": "b", "sdp": None, + }) + await asyncio.sleep(0.05) + _post_from_thread(engine, "leg_state", {"leg_id": "leg_test3", "state": "terminated"}) + await asyncio.sleep(0.05) + + assert engine._legs["leg_test3"].state == "terminated" + assert ("leg_test3", "terminated") in states + + async def test_dtmf_and_trunk_events(self): + engine = _engine_on_loop() + _post_from_thread(engine, "incoming_invite", { + "leg_id": "leg_test4", "from_uri": "a", "to_uri": "b", "sdp": None, + }) + await asyncio.sleep(0.05) + _post_from_thread(engine, "dtmf", {"leg_id": "leg_test4", "digit": "5"}) + _post_from_thread(engine, "trunk_registered", {"registered": True}) + await asyncio.sleep(0.05) + assert engine._legs["leg_test4"].dtmf_buffer == ["5"] + assert engine._trunk_registered is True + + +class TestAudioTapThreadSafety: + async def test_feed_from_foreign_thread_reaches_reader(self): + tap = AudioTap("leg_x") + frame = b"\x01\x02" * 320 + + t = threading.Thread(target=tap.feed, args=(frame,)) + t.start() + t.join() + + received = await tap.read_frame(timeout=1.0) + assert received == frame + + +class TestClassifierOffLoop: + async def test_classify_runs_and_records_history(self): + classifier = AudioClassifier(ClassifierSettings()) + silence = np.zeros(16000, dtype=np.int16).tobytes() + + result = await classifier.classify(silence) + + assert result.audio_type.value == "silence" + assert classifier._classification_history == [result.audio_type] + + +class TestLegStatePropagation: + async def test_leg_lifecycle_drives_call_status(self): + gateway = AIPSTNGateway(settings=Settings(max_concurrent_calls=4)) + call = await gateway.make_call("+15551234567") + assert call.status == CallStatus.RINGING + + (leg_id,) = gateway.call_manager.legs_for_call(call.id) + + await gateway._on_sip_leg_state(leg_id, "connected") + assert gateway.get_call(call.id).status == CallStatus.CONNECTED + + await gateway._on_sip_leg_state(leg_id, "terminated") + assert gateway.get_call(call.id) is None + assert gateway.call_manager.legs_for_call(call.id) == [] + + async def test_late_signals_do_not_stomp_service_states(self): + gateway = AIPSTNGateway(settings=Settings(max_concurrent_calls=4)) + call = await gateway.make_call("+15551234567") + (leg_id,) = gateway.call_manager.legs_for_call(call.id) + + await gateway.call_manager.update_status(call.id, CallStatus.ON_HOLD) + await gateway._on_sip_leg_state(leg_id, "connected") + assert gateway.get_call(call.id).status == CallStatus.ON_HOLD + + +class TestTaskHygiene: + async def test_gateway_stop_cancels_spawned_tasks(self): + gateway = AIPSTNGateway(settings=Settings(max_concurrent_calls=4)) + task = gateway.spawn(asyncio.sleep(60), name="test_sleeper") + + await gateway.stop() + + assert task.cancelled() + assert gateway._tasks == set() From 67a00defc34443e06acbb8dfe49788b3e53c2478 Mon Sep 17 00:00:00 2001 From: Robert Helewka Date: Thu, 9 Jul 2026 20:29:01 -0400 Subject: [PATCH 2/5] =?UTF-8?q?refactor:=20composition=20root=20in=20lifes?= =?UTF-8?q?pan,=20break=20core=E2=86=94services=20cycle,=20shared=20data?= =?UTF-8?q?=20layer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway was the composition root, device registry, inbound-call policy, and call-operations service in one class, with core↔services circular imports papered over by function-local imports, wiring done by assigning private attributes, and MCP tools duplicating REST query logic against their own sessions. Composition: - main.py's lifespan now builds every service and wires them by constructor/registration. gateway.from_config() is gone; core/ no longer imports services/ anywhere — the cycle is dead. - Inbound-call policy moved to ReceptionistService.on_inbound_call (routing evaluation, reject/answer, screening dispatch); wired as the engine's on_incoming_call by the lifespan. Receptionist deps (tts/transcription/recording/routing) are constructor-injected — no more gateway._tts reach-through or importing hold_slayer's private _get_llm (now services.llm_client.get_llm, shared). - Hold-slayer launch goes through a mode-handler registry (register_mode_handler); the gateway no longer knows the service's type. CallManager takes on_call_ended in its constructor. - build_sip_engine() is a pure function taking explicit callbacks. - api/routing.py uses the routing service from app.state via a proper dependency instead of gateway._routing. Shared data layer: - db.session_scope() is the one session convention (get_db wraps it). - services/call_persistence.py gains the query/write functions and the single StoredCallFlow→CallFlow mapper; api/call_flows.py, api/call_history.py, and the six DB-touching MCP tools are thin wrappers over them — the two surfaces can't drift. - legs_for_call() replaces the three private _call_legs scans (gateway transfer/hangup, REST dtmf, MCP dtmf). 7 new tests (mode-handler launch, on_call_ended hook, receptionist inbound answer/reject, call-flow CRUD round-trip and history routes against real SQLite through the shared layer). aiosqlite added to dev deps for that. Co-Authored-By: Claude Fable 5 --- api/call_flows.py | 106 ++---------- api/call_history.py | 57 +++--- api/calls.py | 11 +- api/deps.py | 8 + api/routing.py | 27 ++- core/call_manager.py | 4 +- core/gateway.py | 214 ++++++----------------- db/database.py | 17 +- main.py | 66 ++++++- mcp_server/server.py | 103 +++++------ pyproject.toml | 1 + services/call_analytics.py | 324 ----------------------------------- services/call_persistence.py | 155 ++++++++++++++++- services/hold_slayer.py | 27 +-- services/llm_client.py | 28 +++ services/receptionist.py | 115 +++++++++++-- tests/test_structure.py | 227 ++++++++++++++++++++++++ 17 files changed, 732 insertions(+), 758 deletions(-) delete mode 100644 services/call_analytics.py create mode 100644 tests/test_structure.py diff --git a/api/call_flows.py b/api/call_flows.py index e5d4bf2..0790920 100644 --- a/api/call_flows.py +++ b/api/call_flows.py @@ -2,26 +2,21 @@ Call Flows API — Store and manage IVR navigation trees. The system gets smarter every time you call somewhere. +Thin HTTP layer over the shared data functions in call_persistence. """ -import uuid -from datetime import datetime - from fastapi import APIRouter, Depends, HTTPException from slugify import slugify -from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from api.deps import get_gateway -from core.gateway import AIPSTNGateway -from db.database import StoredCallFlow, get_db +from db.database import get_db from models.call_flow import ( CallFlow, CallFlowCreate, - CallFlowStep, CallFlowSummary, CallFlowUpdate, ) +from services import call_persistence as store router = APIRouter() @@ -34,39 +29,23 @@ async def create_call_flow( """Store a new call flow for a phone number.""" flow_id = slugify(flow.name) - # Check if ID already exists - existing = await db.execute( - select(StoredCallFlow).where(StoredCallFlow.id == flow_id) - ) - if existing.scalar_one_or_none(): + if await store.get_flow(db, flow_id): raise HTTPException( status_code=409, detail=f"Call flow '{flow_id}' already exists. Use PUT to update.", ) - db_flow = StoredCallFlow( - id=flow_id, + row = await store.create_flow( + db, + flow_id=flow_id, name=flow.name, phone_number=flow.phone_number, description=flow.description, steps=[s.model_dump() for s in flow.steps], tags=flow.tags, notes=flow.notes, - last_verified=datetime.now(), - ) - db.add(db_flow) - await db.flush() - - return CallFlow( - id=flow_id, - name=flow.name, - phone_number=flow.phone_number, - description=flow.description, - steps=flow.steps, - tags=flow.tags, - notes=flow.notes, - last_verified=datetime.now(), ) + return store.flow_to_model(row) @router.get("/", response_model=list[CallFlowSummary]) @@ -74,9 +53,7 @@ async def list_call_flows( db: AsyncSession = Depends(get_db), ): """List all stored call flows.""" - result = await db.execute(select(StoredCallFlow)) - rows = result.scalars().all() - + rows = await store.list_flows(db) return [ CallFlowSummary( id=row.id, @@ -100,26 +77,10 @@ async def get_call_flow( db: AsyncSession = Depends(get_db), ): """Get a stored call flow by ID.""" - result = await db.execute( - select(StoredCallFlow).where(StoredCallFlow.id == flow_id) - ) - row = result.scalar_one_or_none() + row = await store.get_flow(db, flow_id) if not row: raise HTTPException(status_code=404, detail=f"Call flow '{flow_id}' not found") - - return CallFlow( - id=row.id, - name=row.name, - phone_number=row.phone_number, - description=row.description or "", - steps=[CallFlowStep(**s) for s in row.steps], - tags=row.tags or [], - notes=row.notes, - avg_hold_time=row.avg_hold_time, - success_rate=row.success_rate, - last_used=row.last_used, - times_used=row.times_used or 0, - ) + return store.flow_to_model(row) @router.get("/by-number/{phone_number}", response_model=CallFlow) @@ -128,29 +89,13 @@ async def get_flow_for_number( db: AsyncSession = Depends(get_db), ): """Look up stored call flow by phone number.""" - result = await db.execute( - select(StoredCallFlow).where(StoredCallFlow.phone_number == phone_number) - ) - row = result.scalar_one_or_none() + row = await store.get_flow_by_number(db, phone_number) if not row: raise HTTPException( status_code=404, detail=f"No call flow found for {phone_number}", ) - - return CallFlow( - id=row.id, - name=row.name, - phone_number=row.phone_number, - description=row.description or "", - steps=[CallFlowStep(**s) for s in row.steps], - tags=row.tags or [], - notes=row.notes, - avg_hold_time=row.avg_hold_time, - success_rate=row.success_rate, - last_used=row.last_used, - times_used=row.times_used or 0, - ) + return store.flow_to_model(row) @router.put("/{flow_id}", response_model=CallFlow) @@ -160,10 +105,7 @@ async def update_call_flow( db: AsyncSession = Depends(get_db), ): """Update an existing call flow.""" - result = await db.execute( - select(StoredCallFlow).where(StoredCallFlow.id == flow_id) - ) - row = result.scalar_one_or_none() + row = await store.get_flow(db, flow_id) if not row: raise HTTPException(status_code=404, detail=f"Call flow '{flow_id}' not found") @@ -181,20 +123,7 @@ async def update_call_flow( row.last_verified = update.last_verified await db.flush() - - return CallFlow( - id=row.id, - name=row.name, - phone_number=row.phone_number, - description=row.description or "", - steps=[CallFlowStep(**s) for s in row.steps], - tags=row.tags or [], - notes=row.notes, - avg_hold_time=row.avg_hold_time, - success_rate=row.success_rate, - last_used=row.last_used, - times_used=row.times_used or 0, - ) + return store.flow_to_model(row) @router.delete("/{flow_id}") @@ -203,10 +132,7 @@ async def delete_call_flow( db: AsyncSession = Depends(get_db), ): """Delete a stored call flow.""" - result = await db.execute( - select(StoredCallFlow).where(StoredCallFlow.id == flow_id) - ) - row = result.scalar_one_or_none() + row = await store.get_flow(db, flow_id) if not row: raise HTTPException(status_code=404, detail=f"Call flow '{flow_id}' not found") diff --git a/api/call_history.py b/api/call_history.py index 7105945..59edbc4 100644 --- a/api/call_history.py +++ b/api/call_history.py @@ -1,22 +1,18 @@ """ Call History API — Read-only access to persisted call records, transcript chunks, and recording files for the dashboard. +Thin HTTP layer over the shared data functions in call_persistence. """ +import os from datetime import datetime -from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import FileResponse -from sqlalchemy import desc, select from sqlalchemy.ext.asyncio import AsyncSession -from db.database import ( - CallRecord, - RecordingRecord, - TranscriptChunk, - get_db, -) +from db.database import get_db +from services import call_persistence as store router = APIRouter() @@ -25,24 +21,22 @@ router = APIRouter() async def list_history( limit: int = Query(50, ge=1, le=500), offset: int = Query(0, ge=0), - number: Optional[str] = None, - status: Optional[str] = None, - since: Optional[datetime] = None, - until: Optional[datetime] = None, + number: str | None = None, + status: str | None = None, + since: datetime | None = None, + until: datetime | None = None, db: AsyncSession = Depends(get_db), ): """Paged list of past calls, newest first.""" - stmt = select(CallRecord).order_by(desc(CallRecord.started_at)) - if number: - stmt = stmt.where(CallRecord.remote_number == number) - if status: - stmt = stmt.where(CallRecord.status == status) - if since: - stmt = stmt.where(CallRecord.started_at >= since) - if until: - stmt = stmt.where(CallRecord.started_at <= until) - - rows = (await db.execute(stmt.offset(offset).limit(limit))).scalars().all() + rows = await store.search_history( + db, + number=number, + status=status, + since=since, + until=until, + limit=limit, + offset=offset, + ) return [ { "id": r.id, @@ -65,9 +59,7 @@ async def list_history( @router.get("/{call_id}/record") async def get_record(call_id: str, db: AsyncSession = Depends(get_db)): """Full CallRecord with classification_timeline.""" - row = (await db.execute( - select(CallRecord).where(CallRecord.id == call_id) - )).scalar_one_or_none() + row = await store.get_record(db, call_id) if not row: raise HTTPException(status_code=404, detail=f"Call {call_id} not found") return { @@ -93,11 +85,7 @@ async def get_record(call_id: str, db: AsyncSession = Depends(get_db)): @router.get("/{call_id}/transcript") async def get_transcript(call_id: str, db: AsyncSession = Depends(get_db)): """Ordered transcript chunks for a call.""" - rows = (await db.execute( - select(TranscriptChunk) - .where(TranscriptChunk.call_id == call_id) - .order_by(TranscriptChunk.seq) - )).scalars().all() + rows = await store.get_transcript_chunks(db, call_id) return [ { "seq": c.seq, @@ -113,14 +101,9 @@ async def get_transcript(call_id: str, db: AsyncSession = Depends(get_db)): @router.get("/{call_id}/recording") async def get_recording(call_id: str, db: AsyncSession = Depends(get_db)): """Stream the WAV recording for a call.""" - row = (await db.execute( - select(RecordingRecord) - .where(RecordingRecord.call_id == call_id) - .order_by(desc(RecordingRecord.started_at)) - )).scalar_one_or_none() + row = await store.latest_recording(db, call_id) if not row or not row.path: raise HTTPException(status_code=404, detail="Recording not found") - import os if not os.path.exists(row.path): raise HTTPException(status_code=404, detail="Recording file missing on disk") return FileResponse(row.path, media_type="audio/wav", filename=os.path.basename(row.path)) diff --git a/api/calls.py b/api/calls.py index a612fcc..eb46c5d 100644 --- a/api/calls.py +++ b/api/calls.py @@ -172,10 +172,9 @@ async def send_dtmf( if not call: raise HTTPException(status_code=404, detail=f"Call {call_id} not found") - # Find the PSTN leg for this call - for leg_id, cid in gateway.call_manager._call_legs.items(): - if cid == call_id: - await gateway.sip_engine.send_dtmf(leg_id, digits) - return {"status": "sent", "digits": digits} + legs = gateway.call_manager.legs_for_call(call_id) + if not legs: + raise HTTPException(status_code=409, detail="No active SIP leg found for this call") - raise HTTPException(status_code=500, detail="No active SIP leg found for this call") + await gateway.sip_engine.send_dtmf(legs[0], digits) + return {"status": "sent", "digits": digits} diff --git a/api/deps.py b/api/deps.py index a8beedb..9698c2f 100644 --- a/api/deps.py +++ b/api/deps.py @@ -18,6 +18,14 @@ def get_gateway(request: Request) -> AIPSTNGateway: return gateway +def get_routing_service(request: Request): + """Get the routing service from app state.""" + routing = getattr(request.app.state, "routing_service", None) + if routing is None: + raise HTTPException(status_code=503, detail="Routing service not ready") + return routing + + def require_token(authorization: str | None = Header(default=None)) -> None: """ Enforce the static bearer token (API_TOKEN) on REST routes. diff --git a/api/routing.py b/api/routing.py index 9f7330e..614871e 100644 --- a/api/routing.py +++ b/api/routing.py @@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from api.deps import get_gateway +from api.deps import get_gateway, get_routing_service from core.gateway import AIPSTNGateway from db.database import Device as DeviceDB from db.database import get_db @@ -15,36 +15,31 @@ from models.routing import ( RoutingRuleCreate, RoutingRuleUpdate, ) +from services.routing import RoutingService router = APIRouter() @router.get("/rules", response_model=list[RoutingRule]) -async def list_rules(gateway: AIPSTNGateway = Depends(get_gateway)): - if gateway._routing is None: - raise HTTPException(status_code=503, detail="Routing service not ready") - return sorted(gateway._routing.rules, key=lambda r: (r.priority, r.id)) +async def list_rules(routing: RoutingService = Depends(get_routing_service)): + return sorted(routing.rules, key=lambda r: (r.priority, r.id)) @router.post("/rules", response_model=RoutingRule, status_code=201) async def create_rule( payload: RoutingRuleCreate, - gateway: AIPSTNGateway = Depends(get_gateway), + routing: RoutingService = Depends(get_routing_service), ): - if gateway._routing is None: - raise HTTPException(status_code=503, detail="Routing service not ready") - return await gateway._routing.create_rule(payload) + return await routing.create_rule(payload) @router.put("/rules/{rule_id}", response_model=RoutingRule) async def update_rule( rule_id: str, payload: RoutingRuleUpdate, - gateway: AIPSTNGateway = Depends(get_gateway), + routing: RoutingService = Depends(get_routing_service), ): - if gateway._routing is None: - raise HTTPException(status_code=503, detail="Routing service not ready") - rule = await gateway._routing.update_rule(rule_id, payload) + rule = await routing.update_rule(rule_id, payload) if rule is None: raise HTTPException(status_code=404, detail=f"Rule {rule_id} not found") return rule @@ -53,11 +48,9 @@ async def update_rule( @router.delete("/rules/{rule_id}") async def delete_rule( rule_id: str, - gateway: AIPSTNGateway = Depends(get_gateway), + routing: RoutingService = Depends(get_routing_service), ): - if gateway._routing is None: - raise HTTPException(status_code=503, detail="Routing service not ready") - ok = await gateway._routing.delete_rule(rule_id) + ok = await routing.delete_rule(rule_id) if not ok: raise HTTPException(status_code=404, detail=f"Rule {rule_id} not found") return {"status": "deleted", "rule_id": rule_id} diff --git a/core/call_manager.py b/core/call_manager.py index 3ea6ee4..1640996 100644 --- a/core/call_manager.py +++ b/core/call_manager.py @@ -26,11 +26,11 @@ class CallManager: The single source of truth for what's happening on the gateway. """ - def __init__(self, event_bus: EventBus): + def __init__(self, event_bus: EventBus, on_call_ended=None): self.event_bus = event_bus self._active_calls: dict[str, ActiveCall] = {} self._call_legs: dict[str, str] = {} # SIP leg ID -> call ID mapping - self._on_call_ended = None # async callback(call: ActiveCall, final_status) + self._on_call_ended = on_call_ended # async callback(call, final_status) # ================================================================ # Call Lifecycle diff --git a/core/gateway.py b/core/gateway.py index da3178b..327e6a1 100644 --- a/core/gateway.py +++ b/core/gateway.py @@ -1,16 +1,18 @@ """ -AI PSTN Gateway — The main orchestrator. +AI PSTN Gateway — call operations and device registry. -Ties together SIP engine, call manager, event bus, and all services. -This is the top-level object that FastAPI and MCP talk to. +The application service that FastAPI and MCP talk to for live-call +work. Composition happens in main.py's lifespan: services are built +there and attached; this module never imports from services/. """ import asyncio import logging +from collections.abc import Callable from datetime import datetime from typing import Optional -from config import Settings, get_settings +from config import Settings from core.call_manager import CallManager from core.dial_plan import is_emergency_number, next_extension from core.event_bus import EventBus @@ -18,28 +20,19 @@ from core.media_pipeline import MediaPipeline from core.sip_engine import MockSIPEngine, SIPEngine from core.sippy_engine import SippyEngine from models.call import ActiveCall, CallMode, CallStatus -from models.call_flow import CallFlow from models.device import Device, DeviceType from models.events import EventType, GatewayEvent logger = logging.getLogger(__name__) -def _extract_number(sip_uri: str) -> str: - """Pull the user part out of a SIP URI (sip:+15551212@host → +15551212).""" - if not sip_uri: - return "" - s = sip_uri.strip() - if s.startswith("<") and ">" in s: - s = s[1:s.index(">")] - if s.startswith("sip:"): - s = s[4:] - if "@" in s: - s = s.split("@", 1)[0] - return s - - -def _build_sip_engine(settings: Settings, gateway: "AIPSTNGateway") -> SIPEngine: +def build_sip_engine( + settings: Settings, + media_pipeline: MediaPipeline, + on_leg_state_change: Callable, + on_device_registered: Callable, + on_incoming_call: Callable, +) -> SIPEngine: """Build the appropriate SIP engine from config.""" trunk = settings.sip_trunk gw_sip = settings.gateway_sip @@ -57,10 +50,10 @@ def _build_sip_engine(settings: Settings, gateway: "AIPSTNGateway") -> SIPEngine trunk_transport=trunk.transport, domain=gw_sip.domain, did=trunk.did, - media_pipeline=gateway.media_pipeline, - on_leg_state_change=gateway._on_sip_leg_state, - on_device_registered=gateway._on_sip_device_registered, - on_incoming_call=gateway._on_sip_incoming_call, + media_pipeline=media_pipeline, + on_leg_state_change=on_leg_state_change, + on_device_registered=on_device_registered, + on_incoming_call=on_incoming_call, ) except Exception as e: logger.warning(f"Could not create SippyEngine: {e} — using mock") @@ -72,35 +65,31 @@ class AIPSTNGateway: """ The AI PSTN Gateway. - Central coordination point for: - - SIP engine (signaling + media) - - Call manager (state + events) - - Hold Slayer service - - Audio classifier - - Transcription service - - Device management + Owns live-call operations (make/transfer/hangup), the device + registry, and per-call background tasks. Services are attached by + the composition root; mode handlers launch per-call services + (hold slayer) without the gateway knowing their types. """ def __init__( self, settings: Settings, sip_engine: Optional[SIPEngine] = None, + on_call_ended=None, ): self.settings = settings self.event_bus = EventBus() - self.call_manager = CallManager(self.event_bus) + self.call_manager = CallManager(self.event_bus, on_call_ended=on_call_ended) self.media_pipeline = MediaPipeline(sample_rate=16000) self.sip_engine: SIPEngine = sip_engine or MockSIPEngine() - # Services (initialized in start()) - self._hold_slayer = None - self._audio_classifier = None - self._transcription = None + # Attached by the composition root (attach_services) self._tts = None - self._routing = None - self._receptionist = None - # Device registry (loaded from DB on start) + # Per-call-mode launchers registered by the composition root + self._mode_handlers: dict[CallMode, Callable] = {} + + # Device registry self._devices: dict[str, Device] = {} # Background tasks (per-call services, receptionist sessions) — @@ -117,23 +106,20 @@ class AIPSTNGateway: task.add_done_callback(self._tasks.discard) return task - @classmethod - def from_config(cls, sip_engine: Optional[SIPEngine] = None) -> "AIPSTNGateway": - """Create gateway from environment config.""" - settings = get_settings() - gw = cls(settings=settings) - if sip_engine is not None: - gw.sip_engine = sip_engine - else: - gw.sip_engine = _build_sip_engine(settings, gw) - return gw + def attach_services(self, tts=None) -> None: + """Attach shared services the gateway must manage on shutdown.""" + self._tts = tts + + def register_mode_handler(self, mode: CallMode, handler: Callable) -> None: + """Register a launcher called as handler(call, sip_leg_id, call_flow_id).""" + self._mode_handlers[mode] = handler # ================================================================ # Lifecycle # ================================================================ async def start(self) -> None: - """Boot the gateway — start SIP engine and services.""" + """Boot the gateway — start media pipeline and SIP engine.""" logger.info("🔥 Starting AI PSTN Gateway...") # Start media pipeline first so SIP engine can hand it RTP streams @@ -141,26 +127,7 @@ class AIPSTNGateway: # Start SIP engine await self.sip_engine.start() - logger.info(f" SIP Engine: ready") - - # Import services here to avoid circular imports - from services.audio_classifier import AudioClassifier - from services.transcription import TranscriptionService - from services.tts import TTSService - from services.routing import RoutingService - from services.receptionist import ReceptionistService - - self._audio_classifier = AudioClassifier(self.settings.classifier) - self._transcription = TranscriptionService(self.settings.speaches) - self._tts = TTSService(self.settings.tts) - self._routing = RoutingService(self) - await self._routing.start() - self._receptionist = ReceptionistService(self) - - # Persist completed calls to the database for history/playback. - from services.call_persistence import persist_call_on_end - - self.call_manager._on_call_ended = persist_call_on_end + logger.info(" SIP Engine: ready") self._started_at = datetime.now() @@ -283,24 +250,10 @@ class AIPSTNGateway: await self.call_manager.update_status(call.id, CallStatus.FAILED) raise - # If hold_slayer mode, launch the Hold Slayer service - if mode == CallMode.HOLD_SLAYER: - from services.hold_slayer import HoldSlayerService - - hold_slayer = HoldSlayerService( - gateway=self, - call_manager=self.call_manager, - sip_engine=self.sip_engine, - classifier=self._audio_classifier, - transcription=self._transcription, - settings=self.settings, - tts=self._tts, - ) - # Launch as background task — don't block - self.spawn( - hold_slayer.run(call, sip_leg_id, call_flow_id), - name=f"holdslayer_{call.id}", - ) + # Hand off to the registered per-mode launcher (e.g. hold slayer) + handler = self._mode_handlers.get(mode) + if handler is not None: + handler(call, sip_leg_id, call_flow_id) return call @@ -321,11 +274,14 @@ class AIPSTNGateway: self.call_manager.map_leg(device_leg_id, call_id) # Get the original PSTN leg - pstn_leg_id = None - for leg_id, cid in self.call_manager._call_legs.items(): - if cid == call_id and leg_id != device_leg_id: - pstn_leg_id = leg_id - break + pstn_leg_id = next( + ( + leg_id + for leg_id in self.call_manager.legs_for_call(call_id) + if leg_id != device_leg_id + ), + None, + ) if pstn_leg_id: # Bridge the PSTN leg and device leg @@ -342,9 +298,8 @@ class AIPSTNGateway: raise ValueError(f"Call {call_id} not found") # Hang up all legs associated with this call - for leg_id, cid in list(self.call_manager._call_legs.items()): - if cid == call_id: - await self.sip_engine.hangup(leg_id) + for leg_id in self.call_manager.legs_for_call(call_id): + await self.sip_engine.hangup(leg_id) await self.call_manager.end_call(call_id) @@ -470,73 +425,6 @@ class AIPSTNGateway: }, )) - async def _on_sip_incoming_call( - self, from_uri: str, to_uri: str, leg_id: str - ) -> None: - """ - Called by SippyEngine when an inbound INVITE arrives. - - Evaluates routing rules, then either: - - Rejects (rule says reject/DND) - - Answers + hands off to the AI Receptionist - """ - import uuid as _uuid - from models.call import CallMode, CallStatus - from models.routing import RoutingActionType - - caller_number = _extract_number(from_uri) - dnis = _extract_number(to_uri) - - # Create a call record so the dashboard sees the ringing call. - call = await self.call_manager.create_call( - remote_number=caller_number, - mode=CallMode.RECEPTIONIST, - intent=None, - call_flow_id=None, - device=None, - ) - # Mark inbound - call.direction = "inbound" - self.call_manager.map_leg(leg_id, call.id) - await self.call_manager.update_status(call.id, CallStatus.RINGING) - - decision = ( - await self._routing.evaluate(caller_number, dnis) - if self._routing is not None - else None - ) - - if decision is not None: - await self.event_bus.publish(GatewayEvent( - type=EventType.ROUTING_RULE_MATCHED, - call_id=call.id, - data={ - "matched_rule_id": decision.matched_rule_id, - "matched_rule_name": decision.matched_rule_name, - "action": decision.action.type.value, - "reason": decision.reason, - }, - message=decision.reason, - )) - - if decision.action.type in (RoutingActionType.REJECT, RoutingActionType.DND): - if hasattr(self.sip_engine, "reject_inbound"): - await self.sip_engine.reject_inbound(leg_id) - await self.call_manager.end_call(call.id, CallStatus.COMPLETED) - return - - # Answer the leg - if hasattr(self.sip_engine, "accept_inbound"): - await self.sip_engine.accept_inbound(leg_id) - await self.call_manager.update_status(call.id, CallStatus.CONNECTED) - - # Hand off to the AI Receptionist - if self._receptionist is not None and self.settings.receptionist.enabled: - self.spawn( - self._receptionist.handle(call, leg_id, decision), - name=f"receptionist_{call.id}", - ) - def preferred_device(self) -> Optional[Device]: """Get the highest-priority online device.""" online_devices = [ diff --git a/db/database.py b/db/database.py index e4d7377..c93d658 100644 --- a/db/database.py +++ b/db/database.py @@ -4,6 +4,8 @@ Database connection and session management. PostgreSQL via asyncpg + SQLAlchemy async. """ +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from datetime import datetime from sqlalchemy import ( @@ -204,8 +206,13 @@ def get_session_factory() -> async_sessionmaker[AsyncSession]: return _session_factory -async def get_db() -> AsyncSession: - """Dependency: yield an async database session.""" +@asynccontextmanager +async def session_scope() -> AsyncIterator[AsyncSession]: + """A commit-on-success session — the one session-lifecycle convention. + + REST handlers get it via the get_db dependency; services and MCP + tools use it directly. + """ factory = get_session_factory() async with factory() as session: try: @@ -216,6 +223,12 @@ async def get_db() -> AsyncSession: raise +async def get_db() -> AsyncIterator[AsyncSession]: + """FastAPI dependency: yield an async database session.""" + async with session_scope() as session: + yield session + + async def init_db(): """Create all tables. For development; use Alembic migrations in production.""" engine = get_engine() diff --git a/main.py b/main.py index 392fcce..5a6978d 100644 --- a/main.py +++ b/main.py @@ -21,9 +21,19 @@ from fastapi.staticfiles import StaticFiles from api import call_flows, call_history, calls, devices, routing, websocket from api.deps import require_token from config import Settings, get_settings -from core.gateway import AIPSTNGateway +from core.gateway import AIPSTNGateway, build_sip_engine from db.database import close_db, init_db from mcp_server.server import create_mcp_server +from models.call import CallMode +from services.audio_classifier import AudioClassifier +from services.call_persistence import persist_call_on_end +from services.hold_slayer import HoldSlayerService +from services.notification import NotificationService +from services.receptionist import ReceptionistService +from services.recording import RecordingService +from services.routing import RoutingService +from services.transcription import TranscriptionService +from services.tts import TTSService # Configure logging logging.basicConfig( @@ -126,23 +136,61 @@ async def lifespan(app: FastAPI): except Exception as e: _handle_db_error(e) - # Boot the telephony engine - gateway = AIPSTNGateway.from_config() + # === Composition root === + # Build the gateway and every service here, wiring them by + # constructor/registration — nothing constructs its own deps. + gateway = AIPSTNGateway(settings=settings, on_call_ended=persist_call_on_end) + + classifier = AudioClassifier(settings.classifier) + transcription = TranscriptionService(settings.speaches) + tts = TTSService(settings.tts) + routing_svc = RoutingService(gateway) + recording_svc = RecordingService() + receptionist = ReceptionistService( + gateway, + tts=tts, + transcription=transcription, + recording=recording_svc, + routing=routing_svc, + ) + gateway.attach_services(tts=tts) + + def launch_hold_slayer(call, sip_leg_id, call_flow_id): + svc = HoldSlayerService( + gateway=gateway, + call_manager=gateway.call_manager, + sip_engine=gateway.sip_engine, + classifier=classifier, + transcription=transcription, + settings=settings, + tts=tts, + ) + gateway.spawn( + svc.run(call, sip_leg_id, call_flow_id), + name=f"holdslayer_{call.id}", + ) + + gateway.register_mode_handler(CallMode.HOLD_SLAYER, launch_hold_slayer) + + gateway.sip_engine = build_sip_engine( + settings, + gateway.media_pipeline, + on_leg_state_change=gateway._on_sip_leg_state, + on_device_registered=gateway._on_sip_device_registered, + on_incoming_call=receptionist.on_inbound_call, + ) + + await routing_svc.start() await gateway.start() app.state.gateway = gateway - - # Start auxiliary services - from services.notification import NotificationService - from services.recording import RecordingService + app.state.routing_service = routing_svc notification_svc = NotificationService(gateway.event_bus, settings) await notification_svc.start() app.state.notification_service = notification_svc - recording_svc = RecordingService() await recording_svc.start() app.state.recording_service = recording_svc - gateway._recording_service = recording_svc logger.info("=" * 60) logger.info("🔥 Hold Slayer Gateway is LIVE") diff --git a/mcp_server/server.py b/mcp_server/server.py index a389257..3c641d3 100644 --- a/mcp_server/server.py +++ b/mcp_server/server.py @@ -184,18 +184,12 @@ def create_mcp_server( Returns the IVR navigation tree if one exists. """ - from db.database import StoredCallFlow, get_session_factory - from sqlalchemy import select + from db.database import session_scope + from services import call_persistence as store try: - factory = get_session_factory() - async with factory() as session: - result = await session.execute( - select(StoredCallFlow).where( - StoredCallFlow.phone_number == phone_number - ) - ) - row = result.scalar_one_or_none() + async with session_scope() as session: + row = await store.get_flow_by_number(session, phone_number) if not row: return f"No stored call flow for {phone_number}." @@ -243,25 +237,26 @@ def create_mcp_server( """ from slugify import slugify as do_slugify - from db.database import StoredCallFlow, get_session_factory + from db.database import session_scope + from services import call_persistence as store try: steps = json.loads(steps_json) flow_id = do_slugify(name) - factory = get_session_factory() - async with factory() as session: - db_flow = StoredCallFlow( - id=flow_id, + async with session_scope() as session: + if await store.get_flow(session, flow_id): + return f"Call flow '{flow_id}' already exists." + await store.create_flow( + session, + flow_id=flow_id, name=name, phone_number=phone_number, description="Created by AI assistant", steps=steps, - notes=notes or None, tags=["ai-created"], + notes=notes or None, ) - session.add(db_flow) - await session.commit() return f"Call flow '{name}' saved for {phone_number} (ID: {flow_id})" except json.JSONDecodeError: @@ -283,12 +278,12 @@ def create_mcp_server( if not call: return f"Call {call_id} not found." - for leg_id, cid in gateway.call_manager._call_legs.items(): - if cid == call_id: - await gateway.sip_engine.send_dtmf(leg_id, digits) - return f"Sent DTMF '{digits}' on call {call_id}." + legs = gateway.call_manager.legs_for_call(call_id) + if not legs: + return f"No active SIP leg found for call {call_id}." - return f"No active SIP leg found for call {call_id}." + await gateway.sip_engine.send_dtmf(legs[0], digits) + return f"Sent DTMF '{digits}' on call {call_id}." @mcp.tool() async def get_call_transcript(call_id: str) -> str: @@ -318,16 +313,12 @@ def create_mcp_server( Returns the recording file path and status. """ - from db.database import CallRecord, get_session_factory - from sqlalchemy import select + from db.database import session_scope + from services import call_persistence as store try: - factory = get_session_factory() - async with factory() as session: - result = await session.execute( - select(CallRecord).where(CallRecord.id == call_id) - ) - record = result.scalar_one_or_none() + async with session_scope() as session: + record = await store.get_record(session, call_id) if not record: return f"No record found for call {call_id}." if not record.recording_path: @@ -348,16 +339,12 @@ def create_mcp_server( Returns the summary, action items, and sentiment analysis. """ - from db.database import CallRecord, get_session_factory - from sqlalchemy import select + from db.database import session_scope + from services import call_persistence as store try: - factory = get_session_factory() - async with factory() as session: - result = await session.execute( - select(CallRecord).where(CallRecord.id == call_id) - ) - record = result.scalar_one_or_none() + async with session_scope() as session: + record = await store.get_record(session, call_id) if not record: return f"No record found for call {call_id}." @@ -398,27 +385,17 @@ def create_mcp_server( intent: Filter by intent text (partial match) limit: Max results to return (default 10) """ - from db.database import CallRecord, get_session_factory - from sqlalchemy import select + from db.database import session_scope + from services import call_persistence as store try: - factory = get_session_factory() - async with factory() as session: - query = select(CallRecord).order_by( - CallRecord.started_at.desc() - ).limit(limit) - - if phone_number: - query = query.where( - CallRecord.remote_number.contains(phone_number) - ) - if intent: - query = query.where( - CallRecord.intent.icontains(intent) - ) - - result = await session.execute(query) - records = result.scalars().all() + async with session_scope() as session: + records = await store.search_history( + session, + number_contains=phone_number or None, + intent_contains=intent or None, + limit=limit, + ) if not records: return "No matching call records found." @@ -482,14 +459,12 @@ def create_mcp_server( @mcp.resource("gateway://call-flows") async def resource_call_flows() -> str: """List all stored call flows.""" - from db.database import StoredCallFlow, get_session_factory - from sqlalchemy import select + from db.database import session_scope + from services import call_persistence as store try: - factory = get_session_factory() - async with factory() as session: - result = await session.execute(select(StoredCallFlow)) - rows = result.scalars().all() + async with session_scope() as session: + rows = await store.list_flows(session) flows = [ { "id": r.id, diff --git a/pyproject.toml b/pyproject.toml index 877f185..70b0676 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ dev = [ "pytest-cov>=6.0.0", "httpx>=0.28.0", "ruff>=0.8.0", + "aiosqlite>=0.22.0", ] [tool.setuptools.packages.find] diff --git a/services/call_analytics.py b/services/call_analytics.py deleted file mode 100644 index 4fa67d7..0000000 --- a/services/call_analytics.py +++ /dev/null @@ -1,324 +0,0 @@ -""" -Call Analytics Service — Tracks call metrics and generates insights. - -Monitors call patterns, hold times, success rates, and IVR navigation -efficiency. Provides data for the dashboard and API. -""" - -import logging -from collections import defaultdict -from datetime import datetime, timedelta -from typing import Any, Optional - -from models.call import ActiveCall, AudioClassification, CallMode, CallStatus - -logger = logging.getLogger(__name__) - - -class CallAnalytics: - """ - In-memory call analytics engine. - - Tracks: - - Call success/failure rates - - Hold time statistics (avg, min, max, p95) - - IVR navigation efficiency - - Human detection accuracy - - Per-number/company patterns - - Time-of-day patterns - - In production, this would be backed by TimescaleDB or similar. - For now, we keep rolling windows in memory. - """ - - def __init__(self, max_history: int = 10000): - self._max_history = max_history - self._call_records: list[CallRecord] = [] - self._company_stats: dict[str, CompanyStats] = defaultdict(CompanyStats) - - # ================================================================ - # Record Calls - # ================================================================ - - def record_call(self, call: ActiveCall) -> None: - """ - Record a completed call for analytics. - - Called when a call ends (from CallManager). - """ - record = CallRecord( - call_id=call.id, - remote_number=call.remote_number, - mode=call.mode, - status=call.status, - intent=call.intent, - started_at=call.created_at, - duration_seconds=call.duration, - hold_time_seconds=call.hold_time, - classification_history=[ - r.audio_type.value for r in call.classification_history - ], - transcript_chunks=list(call.transcript_chunks), - services=list(call.services), - ) - - self._call_records.append(record) - - # Trim history - if len(self._call_records) > self._max_history: - self._call_records = self._call_records[-self._max_history :] - - # Update company stats - company_key = self._normalize_number(call.remote_number) - self._company_stats[company_key].update(record) - - logger.debug( - f"📊 Recorded call {call.id}: " - f"{call.status.value}, {call.duration}s, hold={call.hold_time}s" - ) - - # ================================================================ - # Aggregate Stats - # ================================================================ - - def get_summary(self, hours: int = 24) -> dict[str, Any]: - """Get summary statistics for the last N hours.""" - cutoff = datetime.now() - timedelta(hours=hours) - recent = [r for r in self._call_records if r.started_at >= cutoff] - - if not recent: - return { - "period_hours": hours, - "total_calls": 0, - "success_rate": 0.0, - "avg_hold_time": 0.0, - "avg_duration": 0.0, - } - - total = len(recent) - successful = sum(1 for r in recent if r.status in ( - CallStatus.COMPLETED, CallStatus.BRIDGED, CallStatus.HUMAN_DETECTED - )) - failed = sum(1 for r in recent if r.status == CallStatus.FAILED) - - hold_times = [r.hold_time_seconds for r in recent if r.hold_time_seconds > 0] - durations = [r.duration_seconds for r in recent if r.duration_seconds > 0] - - hold_slayer_calls = [r for r in recent if r.mode == CallMode.HOLD_SLAYER] - hold_slayer_success = sum( - 1 for r in hold_slayer_calls - if r.status in (CallStatus.BRIDGED, CallStatus.HUMAN_DETECTED) - ) - - return { - "period_hours": hours, - "total_calls": total, - "successful": successful, - "failed": failed, - "success_rate": round(successful / total, 3) if total else 0.0, - "avg_duration": round(sum(durations) / len(durations), 1) if durations else 0.0, - "max_duration": max(durations) if durations else 0, - "hold_time": { - "avg": round(sum(hold_times) / len(hold_times), 1) if hold_times else 0.0, - "min": min(hold_times) if hold_times else 0, - "max": max(hold_times) if hold_times else 0, - "p95": self._percentile(hold_times, 95) if hold_times else 0, - "total": sum(hold_times), - }, - "hold_slayer": { - "total": len(hold_slayer_calls), - "success": hold_slayer_success, - "success_rate": round( - hold_slayer_success / len(hold_slayer_calls), 3 - ) if hold_slayer_calls else 0.0, - }, - "by_mode": self._group_by_mode(recent), - "by_hour": self._group_by_hour(recent), - } - - def get_company_stats(self, number: str) -> dict[str, Any]: - """Get stats for a specific company/number.""" - key = self._normalize_number(number) - stats = self._company_stats.get(key) - if not stats: - return {"number": number, "total_calls": 0} - return stats.to_dict(number) - - def get_top_numbers(self, limit: int = 10) -> list[dict[str, Any]]: - """Get the most-called numbers with their stats.""" - sorted_stats = sorted( - self._company_stats.items(), - key=lambda x: x[1].total_calls, - reverse=True, - )[:limit] - return [stats.to_dict(number) for number, stats in sorted_stats] - - # ================================================================ - # Hold Time Trends - # ================================================================ - - def get_hold_time_trend( - self, - number: Optional[str] = None, - days: int = 7, - ) -> list[dict]: - """ - Get hold time trend data for graphing. - - Returns daily average hold times for the last N days. - """ - cutoff = datetime.now() - timedelta(days=days) - records = [r for r in self._call_records if r.started_at >= cutoff] - - if number: - key = self._normalize_number(number) - records = [r for r in records if self._normalize_number(r.remote_number) == key] - - # Group by day - by_day: dict[str, list[int]] = defaultdict(list) - for r in records: - day = r.started_at.strftime("%Y-%m-%d") - if r.hold_time_seconds > 0: - by_day[day].append(r.hold_time_seconds) - - trend = [] - for i in range(days): - date = (datetime.now() - timedelta(days=days - 1 - i)).strftime("%Y-%m-%d") - times = by_day.get(date, []) - trend.append({ - "date": date, - "avg_hold_time": round(sum(times) / len(times), 1) if times else 0, - "call_count": len(times), - "max_hold_time": max(times) if times else 0, - }) - - return trend - - # ================================================================ - # Helpers - # ================================================================ - - @staticmethod - def _normalize_number(number: str) -> str: - """Normalize phone number for grouping.""" - # Strip formatting, keep last 10 digits - digits = "".join(c for c in number if c.isdigit()) - return digits[-10:] if len(digits) >= 10 else digits - - @staticmethod - def _percentile(values: list, pct: int) -> float: - """Calculate percentile value.""" - if not values: - return 0.0 - sorted_vals = sorted(values) - idx = int(len(sorted_vals) * pct / 100) - idx = min(idx, len(sorted_vals) - 1) - return float(sorted_vals[idx]) - - @staticmethod - def _group_by_mode(records: list["CallRecord"]) -> dict[str, int]: - """Group call counts by mode.""" - by_mode: dict[str, int] = defaultdict(int) - for r in records: - by_mode[r.mode.value] += 1 - return dict(by_mode) - - @staticmethod - def _group_by_hour(records: list["CallRecord"]) -> dict[int, int]: - """Group call counts by hour of day.""" - by_hour: dict[int, int] = defaultdict(int) - for r in records: - by_hour[r.started_at.hour] += 1 - return dict(sorted(by_hour.items())) - - @property - def total_calls_recorded(self) -> int: - return len(self._call_records) - - -# ================================================================ -# Data Models -# ================================================================ - -class CallRecord: - """A completed call record for analytics.""" - - def __init__( - self, - call_id: str, - remote_number: str, - mode: CallMode, - status: CallStatus, - intent: Optional[str] = None, - started_at: Optional[datetime] = None, - duration_seconds: int = 0, - hold_time_seconds: int = 0, - classification_history: Optional[list[str]] = None, - transcript_chunks: Optional[list[str]] = None, - services: Optional[list[str]] = None, - ): - self.call_id = call_id - self.remote_number = remote_number - self.mode = mode - self.status = status - self.intent = intent - self.started_at = started_at or datetime.now() - self.duration_seconds = duration_seconds - self.hold_time_seconds = hold_time_seconds - self.classification_history = classification_history or [] - self.transcript_chunks = transcript_chunks or [] - self.services = services or [] - - -class CompanyStats: - """Aggregated stats for a specific company/phone number.""" - - def __init__(self): - self.total_calls = 0 - self.successful_calls = 0 - self.failed_calls = 0 - self.total_hold_time = 0 - self.hold_times: list[int] = [] - self.total_duration = 0 - self.last_called: Optional[datetime] = None - self.intents: dict[str, int] = defaultdict(int) - - def update(self, record: CallRecord) -> None: - """Update stats with a new call record.""" - self.total_calls += 1 - self.total_duration += record.duration_seconds - self.last_called = record.started_at - - if record.status in (CallStatus.COMPLETED, CallStatus.BRIDGED, CallStatus.HUMAN_DETECTED): - self.successful_calls += 1 - elif record.status == CallStatus.FAILED: - self.failed_calls += 1 - - if record.hold_time_seconds > 0: - self.total_hold_time += record.hold_time_seconds - self.hold_times.append(record.hold_time_seconds) - - if record.intent: - self.intents[record.intent] += 1 - - def to_dict(self, number: str) -> dict[str, Any]: - return { - "number": number, - "total_calls": self.total_calls, - "successful_calls": self.successful_calls, - "failed_calls": self.failed_calls, - "success_rate": round( - self.successful_calls / self.total_calls, 3 - ) if self.total_calls else 0.0, - "avg_hold_time": round( - self.total_hold_time / len(self.hold_times), 1 - ) if self.hold_times else 0.0, - "max_hold_time": max(self.hold_times) if self.hold_times else 0, - "avg_duration": round( - self.total_duration / self.total_calls, 1 - ) if self.total_calls else 0.0, - "last_called": self.last_called.isoformat() if self.last_called else None, - "top_intents": dict( - sorted(self.intents.items(), key=lambda x: x[1], reverse=True)[:5] - ), - } diff --git a/services/call_persistence.py b/services/call_persistence.py index 393954a..af57d77 100644 --- a/services/call_persistence.py +++ b/services/call_persistence.py @@ -1,25 +1,168 @@ """ -Call Persistence — Writes completed calls and their transcript chunks -to the database when CallManager.end_call() fires. +Call Persistence — the data-access layer for calls and call flows. + +Holds the on-hangup persistence hook plus the query/write functions +that both the REST handlers and the MCP tools call, so the two +surfaces can't drift. Every function takes an AsyncSession; callers +own the transaction (get_db for REST, session_scope for MCP/services). """ import logging import uuid from datetime import datetime -from db.database import CallRecord, TranscriptChunk, get_session_factory +from sqlalchemy import desc, select +from sqlalchemy.ext.asyncio import AsyncSession + +from db.database import ( + CallRecord, + RecordingRecord, + StoredCallFlow, + TranscriptChunk, + session_scope, +) from models.call import ActiveCall, CallStatus +from models.call_flow import CallFlow, CallFlowStep logger = logging.getLogger(__name__) +def flow_to_model(row: StoredCallFlow) -> CallFlow: + """The one StoredCallFlow-row → CallFlow-model mapping.""" + return CallFlow( + id=row.id, + name=row.name, + phone_number=row.phone_number, + description=row.description or "", + steps=[CallFlowStep(**s) for s in (row.steps or [])], + tags=row.tags or [], + notes=row.notes, + avg_hold_time=row.avg_hold_time, + success_rate=row.success_rate, + last_used=row.last_used, + times_used=row.times_used or 0, + ) + + +# ================================================================ +# Call flows +# ================================================================ + +async def get_flow(session: AsyncSession, flow_id: str) -> StoredCallFlow | None: + result = await session.execute( + select(StoredCallFlow).where(StoredCallFlow.id == flow_id) + ) + return result.scalar_one_or_none() + + +async def get_flow_by_number( + session: AsyncSession, phone_number: str +) -> StoredCallFlow | None: + result = await session.execute( + select(StoredCallFlow).where(StoredCallFlow.phone_number == phone_number) + ) + return result.scalar_one_or_none() + + +async def list_flows(session: AsyncSession) -> list[StoredCallFlow]: + result = await session.execute(select(StoredCallFlow)) + return list(result.scalars().all()) + + +async def create_flow( + session: AsyncSession, + flow_id: str, + name: str, + phone_number: str, + steps: list[dict], + description: str | None = None, + tags: list[str] | None = None, + notes: str | None = None, +) -> StoredCallFlow: + row = StoredCallFlow( + id=flow_id, + name=name, + phone_number=phone_number, + description=description, + steps=steps, + tags=tags, + notes=notes, + last_verified=datetime.now(), + ) + session.add(row) + await session.flush() + return row + + +# ================================================================ +# Call history / records +# ================================================================ + +async def get_record(session: AsyncSession, call_id: str) -> CallRecord | None: + result = await session.execute( + select(CallRecord).where(CallRecord.id == call_id) + ) + return result.scalar_one_or_none() + + +async def search_history( + session: AsyncSession, + number: str | None = None, + number_contains: str | None = None, + intent_contains: str | None = None, + status: str | None = None, + since: datetime | None = None, + until: datetime | None = None, + limit: int = 50, + offset: int = 0, +) -> list[CallRecord]: + stmt = select(CallRecord).order_by(desc(CallRecord.started_at)) + if number: + stmt = stmt.where(CallRecord.remote_number == number) + if number_contains: + stmt = stmt.where(CallRecord.remote_number.contains(number_contains)) + if intent_contains: + stmt = stmt.where(CallRecord.intent.icontains(intent_contains)) + if status: + stmt = stmt.where(CallRecord.status == status) + if since: + stmt = stmt.where(CallRecord.started_at >= since) + if until: + stmt = stmt.where(CallRecord.started_at <= until) + result = await session.execute(stmt.offset(offset).limit(limit)) + return list(result.scalars().all()) + + +async def get_transcript_chunks( + session: AsyncSession, call_id: str +) -> list[TranscriptChunk]: + result = await session.execute( + select(TranscriptChunk) + .where(TranscriptChunk.call_id == call_id) + .order_by(TranscriptChunk.seq) + ) + return list(result.scalars().all()) + + +async def latest_recording( + session: AsyncSession, call_id: str +) -> RecordingRecord | None: + result = await session.execute( + select(RecordingRecord) + .where(RecordingRecord.call_id == call_id) + .order_by(desc(RecordingRecord.started_at)) + ) + return result.scalars().first() + + async def persist_call_on_end(call: ActiveCall, final_status: CallStatus) -> None: """Insert a CallRecord and any transcript chunks for `call`. - Wired into CallManager via _on_call_ended in gateway.start(). + Wired into CallManager as its on_call_ended hook by the + composition root in main.py. """ try: - async with get_session_factory()() as session: + async with session_scope() as session: record = CallRecord( id=call.id, direction=call.direction, @@ -64,7 +207,5 @@ async def persist_call_on_end(call: ActiveCall, final_status: CallStatus) -> Non speaker=speaker, text=payload, )) - - await session.commit() except Exception as e: logger.warning(f"Could not persist call {call.id}: {e}") diff --git a/services/hold_slayer.py b/services/hold_slayer.py index d9cec27..4d6031b 100644 --- a/services/hold_slayer.py +++ b/services/hold_slayer.py @@ -23,35 +23,12 @@ from models.call import ActiveCall, AudioClassification, CallStatus, Classificat from models.call_flow import ActionType, CallFlow, CallFlowStep from models.events import EventType, GatewayEvent from services.audio_classifier import AudioClassifier +from services.llm_client import get_llm from services.transcription import TranscriptionService from services.tts import TTSService logger = logging.getLogger(__name__) -# LLM client is optional — imported at use time -_llm_client = None - - -def _get_llm(): - """Lazy-load LLM client (optional dependency).""" - global _llm_client - if _llm_client is None: - try: - from config import get_settings - from services.llm_client import LLMClient - - settings = get_settings() - _llm_client = LLMClient( - base_url=settings.llm.base_url, - model=settings.llm.model, - api_key=settings.llm.api_key.get_secret_value(), - timeout=settings.llm.timeout, - ) - except Exception as e: - logger.debug(f"LLM client not available: {e}") - _llm_client = False # Sentinel: don't retry - return _llm_client if _llm_client is not False else None - class HoldSlayerService: """ @@ -228,7 +205,7 @@ class HoldSlayerService: # Phase 2: LLM fallback if regex couldn't decide if not decision and transcript: - llm = _get_llm() + llm = get_llm() if llm: try: logger.info("🤖 Regex inconclusive, asking LLM...") diff --git a/services/llm_client.py b/services/llm_client.py index 623b157..b754afa 100644 --- a/services/llm_client.py +++ b/services/llm_client.py @@ -389,3 +389,31 @@ class LLMClient: "model": self.model, "base_url": self.base_url, } + + +# ================================================================ +# Shared lazy client +# ================================================================ + +_shared_client: Optional["LLMClient"] = None +_shared_failed = False + + +def get_llm() -> Optional["LLMClient"]: + """Lazily build the shared LLMClient from settings (None if unavailable).""" + global _shared_client, _shared_failed + if _shared_client is None and not _shared_failed: + try: + from config import get_settings + + settings = get_settings() + _shared_client = LLMClient( + base_url=settings.llm.base_url, + model=settings.llm.model, + api_key=settings.llm.api_key.get_secret_value(), + timeout=settings.llm.timeout, + ) + except Exception as e: + logger.debug(f"LLM client not available: {e}") + _shared_failed = True # don't retry + return _shared_client diff --git a/services/receptionist.py b/services/receptionist.py index a27e827..1ecc071 100644 --- a/services/receptionist.py +++ b/services/receptionist.py @@ -27,12 +27,100 @@ from models.routing import RoutingAction, RoutingActionType, RoutingDecision logger = logging.getLogger(__name__) -class ReceptionistService: - """Drives the receptionist state machine for a single inbound call.""" +def _extract_number(sip_uri: str) -> str: + """Pull the user part out of a SIP URI (sip:+15551212@host → +15551212).""" + if not sip_uri: + return "" + s = sip_uri.strip() + if s.startswith("<") and ">" in s: + s = s[1 : s.index(">")] + if s.startswith("sip:"): + s = s[4:] + if "@" in s: + s = s.split("@", 1)[0] + return s - def __init__(self, gateway): + +class ReceptionistService: + """Owns inbound-call policy: routing evaluation, screening, voicemail.""" + + def __init__( + self, + gateway, + tts=None, + transcription=None, + recording=None, + routing=None, + ): self.gateway = gateway self.settings = gateway.settings.receptionist + self.tts = tts + self.transcription = transcription + self.recording = recording + self.routing = routing + + async def on_inbound_call(self, from_uri: str, to_uri: str, leg_id: str) -> None: + """ + Entry point for an inbound INVITE (wired as the SIP engine's + on_incoming_call by the composition root). + + Evaluates routing rules, then either rejects (rule says + reject/DND) or answers and runs the screening flow. + """ + from models.call import CallMode + + gateway = self.gateway + caller_number = _extract_number(from_uri) + dnis = _extract_number(to_uri) + + # Create a call record so the dashboard sees the ringing call. + call = await gateway.call_manager.create_call( + remote_number=caller_number, + mode=CallMode.RECEPTIONIST, + intent=None, + call_flow_id=None, + device=None, + ) + call.direction = "inbound" + gateway.call_manager.map_leg(leg_id, call.id) + await gateway.call_manager.update_status(call.id, CallStatus.RINGING) + + decision = ( + await self.routing.evaluate(caller_number, dnis) + if self.routing is not None + else None + ) + + if decision is not None: + await gateway.event_bus.publish(GatewayEvent( + type=EventType.ROUTING_RULE_MATCHED, + call_id=call.id, + data={ + "matched_rule_id": decision.matched_rule_id, + "matched_rule_name": decision.matched_rule_name, + "action": decision.action.type.value, + "reason": decision.reason, + }, + message=decision.reason, + )) + + if decision.action.type in (RoutingActionType.REJECT, RoutingActionType.DND): + if hasattr(gateway.sip_engine, "reject_inbound"): + await gateway.sip_engine.reject_inbound(leg_id) + await gateway.call_manager.end_call(call.id, CallStatus.COMPLETED) + return + + # Answer the leg + if hasattr(gateway.sip_engine, "accept_inbound"): + await gateway.sip_engine.accept_inbound(leg_id) + await gateway.call_manager.update_status(call.id, CallStatus.CONNECTED) + + # Screen the caller (unless the receptionist is disabled) + if self.settings.enabled: + gateway.spawn( + self.handle(call, leg_id, decision), + name=f"receptionist_{call.id}", + ) async def handle( self, @@ -89,7 +177,10 @@ class ReceptionistService: await self._speak( call, sip_leg_id, "One moment, I'll connect you now." ) - answered = await self.gateway._routing.ring_chain( + if self.routing is None: + await self._take_message(call, sip_leg_id) + return + answered = await self.routing.ring_chain( call.id, devices, action.ring_timeout ) if answered: @@ -156,10 +247,10 @@ class ReceptionistService: finally: tap.close() - if not audio: + if not audio or self.transcription is None: return "" - return await self.gateway._transcription.transcribe(bytes(audio)) + return await self.transcription.transcribe(bytes(audio)) async def _classify( self, @@ -168,9 +259,9 @@ class ReceptionistService: routing_decision: Optional[RoutingDecision], ) -> dict: """Ask the LLM to interpret the caller's utterance.""" - from services.hold_slayer import _get_llm + from services.llm_client import get_llm - llm = _get_llm() + llm = get_llm() if llm is None or not transcript.strip(): return { "intent": transcript or "unknown", @@ -245,7 +336,7 @@ class ReceptionistService: await self._speak(call, sip_leg_id, self.settings.message_prompt) media = self.gateway.media_pipeline - recording_svc = getattr(self.gateway, "_recording_service", None) + recording_svc = self.recording if recording_svc is None or media is None: logger.warning("Receptionist: recording unavailable, ending call") await self._hangup(call, sip_leg_id) @@ -264,10 +355,10 @@ class ReceptionistService: message_text = "" rec_path = session.filepath_mixed if session else None - if rec_path and Path(rec_path).exists(): + if rec_path and Path(rec_path).exists() and self.transcription is not None: try: audio_bytes = Path(rec_path).read_bytes() - message_text = await self.gateway._transcription.transcribe(audio_bytes) + message_text = await self.transcription.transcribe(audio_bytes) except Exception as e: logger.warning(f"Receptionist transcribe failed: {e}") @@ -292,7 +383,7 @@ class ReceptionistService: # ---------------------------------------------------------------- async def _speak(self, call: ActiveCall, sip_leg_id: str, text: str) -> None: - tts = self.gateway._tts + tts = self.tts media = self.gateway.media_pipeline if tts is None or media is None or not text.strip(): return diff --git a/tests/test_structure.py b/tests/test_structure.py new file mode 100644 index 0000000..beb9d5b --- /dev/null +++ b/tests/test_structure.py @@ -0,0 +1,227 @@ +""" +Composition and API-surface tests. + +Covers the gateway composed the way main.py's lifespan composes it +(mode handlers, on_call_ended hook, receptionist-owned inbound +policy) and the REST routes running against a real (SQLite) database +through the shared data layer in services/call_persistence.py. +""" + +import httpx +import pytest +from pydantic import SecretStr +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from sqlalchemy.pool import StaticPool + +import main +from config import ReceptionistSettings, Settings, get_settings +from core.gateway import AIPSTNGateway +from db.database import Base, CallRecord, get_db +from models.call import CallMode, CallStatus +from models.routing import RoutingAction, RoutingActionType, RoutingDecision +from services.receptionist import ReceptionistService + +# ================================================================ +# Gateway composition +# ================================================================ + +class TestGatewayComposition: + async def test_mode_handler_launches_per_call(self): + gateway = AIPSTNGateway(settings=Settings(max_concurrent_calls=4)) + launched: list[tuple] = [] + gateway.register_mode_handler( + CallMode.HOLD_SLAYER, + lambda call, leg_id, flow_id: launched.append((call.id, leg_id, flow_id)), + ) + + call = await gateway.make_call("+15551234567", mode=CallMode.HOLD_SLAYER, + call_flow_id="acme-main") + + assert launched == [(call.id, gateway.call_manager.legs_for_call(call.id)[0], "acme-main")] + + async def test_direct_mode_needs_no_handler(self): + gateway = AIPSTNGateway(settings=Settings(max_concurrent_calls=4)) + call = await gateway.make_call("+15551234567") + assert call.status == CallStatus.RINGING + + async def test_on_call_ended_hook_from_constructor(self): + ended: list[tuple] = [] + + async def hook(call, status): + ended.append((call.id, status)) + + gateway = AIPSTNGateway( + settings=Settings(max_concurrent_calls=4), on_call_ended=hook + ) + call = await gateway.make_call("+15551234567") + await gateway.hangup_call(call.id) + + assert ended == [(call.id, CallStatus.COMPLETED)] + + +# ================================================================ +# Receptionist-owned inbound policy +# ================================================================ + +class _StubRouting: + def __init__(self, decision): + self._decision = decision + + async def evaluate(self, caller_number, dnis): + return self._decision + + +class TestInboundPolicy: + def _gateway(self) -> AIPSTNGateway: + settings = Settings(max_concurrent_calls=4) + settings.receptionist = ReceptionistSettings(enabled=False) + return AIPSTNGateway(settings=settings) + + async def test_inbound_call_answered_and_tracked(self): + gateway = self._gateway() + receptionist = ReceptionistService(gateway) + + await receptionist.on_inbound_call( + "sip:+16135550100@pstn", "sip:+15551234567@gw", "leg_in1" + ) + + calls = list(gateway.call_manager.active_calls.values()) + assert len(calls) == 1 + call = calls[0] + assert call.direction == "inbound" + assert call.remote_number == "+16135550100" + assert call.status == CallStatus.CONNECTED + assert gateway.call_manager.legs_for_call(call.id) == ["leg_in1"] + + async def test_reject_rule_declines_before_answer(self): + gateway = self._gateway() + decision = RoutingDecision( + action=RoutingAction(type=RoutingActionType.REJECT), + matched_rule_id="rule_x", + matched_rule_name="block", + reason="matched rule 'block'", + ) + receptionist = ReceptionistService(gateway, routing=_StubRouting(decision)) + + await receptionist.on_inbound_call( + "sip:+18005550100@pstn", "sip:+15551234567@gw", "leg_in2" + ) + + assert gateway.call_manager.active_calls == {} + + +# ================================================================ +# REST routes on the shared data layer (real SQLite) +# ================================================================ + +@pytest.fixture +async def client(monkeypatch): + monkeypatch.setattr(get_settings(), "api_token", SecretStr("")) + + engine = create_async_engine( + "sqlite+aiosqlite:///:memory:", + poolclass=StaticPool, + connect_args={"check_same_thread": False}, + ) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + factory = async_sessionmaker(engine, expire_on_commit=False) + + async def _get_db(): + async with factory() as session: + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise + + main.app.dependency_overrides[get_db] = _get_db + transport = httpx.ASGITransport(app=main.app) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as c: + c.db_factory = factory + yield c + main.app.dependency_overrides.pop(get_db, None) + await engine.dispose() + + +FLOW_PAYLOAD = { + "name": "Acme Main Line", + "phone_number": "+18005551234", + "description": "Main IVR", + "steps": [ + { + "id": "step1", + "description": "Press 2 for billing", + "action": "dtmf", + "action_value": "2", + } + ], + "tags": ["test"], +} + + +class TestCallFlowRoutes: + async def test_crud_round_trip(self, client): + resp = await client.post("/api/call-flows/", json=FLOW_PAYLOAD) + assert resp.status_code == 200, resp.text + flow_id = resp.json()["id"] + assert flow_id == "acme-main-line" + + resp = await client.post("/api/call-flows/", json=FLOW_PAYLOAD) + assert resp.status_code == 409 + + resp = await client.get("/api/call-flows/") + assert [f["id"] for f in resp.json()] == [flow_id] + + resp = await client.get(f"/api/call-flows/{flow_id}") + assert resp.json()["steps"][0]["action_value"] == "2" + + resp = await client.get("/api/call-flows/by-number/+18005551234") + assert resp.json()["id"] == flow_id + + resp = await client.put( + f"/api/call-flows/{flow_id}", json={"notes": "updated"} + ) + assert resp.json()["notes"] == "updated" + + resp = await client.delete(f"/api/call-flows/{flow_id}") + assert resp.json()["status"] == "deleted" + + resp = await client.get(f"/api/call-flows/{flow_id}") + assert resp.status_code == 404 + + +class TestCallHistoryRoutes: + async def test_history_and_record(self, client): + resp = await client.get("/api/calls/history") + assert resp.status_code == 200 + assert resp.json() == [] + + async with client.db_factory() as session: + session.add(CallRecord( + id="call_hist1", + direction="outbound", + remote_number="+18005551234", + status="completed", + mode="hold_slayer", + intent="dispute charge", + duration=120, + hold_time=90, + )) + await session.commit() + + resp = await client.get("/api/calls/history") + assert [r["id"] for r in resp.json()] == ["call_hist1"] + + resp = await client.get("/api/calls/history?number=%2B18005551234") + assert len(resp.json()) == 1 + + resp = await client.get("/api/calls/call_hist1/record") + assert resp.json()["intent"] == "dispute charge" + + resp = await client.get("/api/calls/call_missing/record") + assert resp.status_code == 404 + + resp = await client.get("/api/calls/call_hist1/transcript") + assert resp.json() == [] From 4048ce1db69a841108e9e25d57e334f8e9333088 Mon Sep 17 00:00:00 2001 From: Robert Helewka Date: Fri, 10 Jul 2026 07:01:45 -0400 Subject: [PATCH 3/5] Stage 4: honest health, explicit error policy, event-bus integrity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine mode is now explicit: USE_MOCK_SIP=true is the only way to get the mock engine; an unconfigured trunk fails startup with guidance instead of silently degrading. Root-caused why the engine always ran mock: nested pydantic-settings never read .env (no env_file on the sub-settings classes) — all 8 now declare it. /health stops lying: reports engine mode (sippy|mock), a live DB SELECT 1, trunk registration state with reason, and TTS/STT availability from their last real request; "healthy" now requires ready + db + sippy + registered trunk. Error policy: leaf services (tts/transcription/llm_client) raise and track availability; call-loop callers catch, publish EventType.ERROR naming the failed service, and apply an explicit fallback. Persistence writes get one bounded 3x exponential retry, then an ERROR log — no more silent data loss. Event bus: a full subscriber queue drops its oldest event (counted) instead of silently evicting the subscription; subscribe(replay_last=N) delivers the advertised history replay, used by /ws/events (25). Receptionist correctness: a matched TAKE_MESSAGE rule beats the LLM; voicemail polls for early hangup and stops/transcribes/hangs up in finally; RecordingSession finally keeps its leg_ids so taps detach. Dead code removed: models/contact.py + Contact table, dtmf_buffer, transcribe_stream stub, SMS stub in notification.py. Co-Authored-By: Claude Fable 5 --- .env.example | 3 + api/websocket.py | 2 +- config.py | 21 ++++--- core/event_bus.py | 76 ++++++++++++++++-------- core/gateway.py | 55 ++++++++++------- core/sippy_engine.py | 6 +- db/database.py | 18 ------ main.py | 73 +++++++++++++++++++---- models/contact.py | 60 ------------------- services/call_persistence.py | 111 ++++++++++++++++++++--------------- services/hold_slayer.py | 40 +++++++++++-- services/llm_client.py | 9 ++- services/notification.py | 41 ------------- services/receptionist.py | 89 +++++++++++++++++++--------- services/recording.py | 54 ++++++++++------- services/transcription.py | 50 ++++------------ services/tts.py | 20 +++---- tests/test_concurrency.py | 5 -- tests/test_health_policy.py | 89 ++++++++++++++++++++++++++++ tests/test_receptionist.py | 19 +++++- tests/test_services.py | 8 +-- 21 files changed, 492 insertions(+), 357 deletions(-) delete mode 100644 models/contact.py create mode 100644 tests/test_health_policy.py diff --git a/.env.example b/.env.example index 25f0a69..3cb31b6 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,9 @@ DATABASE_URL=postgresql+asyncpg://holdslayer:@localhost:5432/holdsl API_TOKEN= # --- SIP Trunk --- +# The mock engine must be requested explicitly; an unconfigured trunk +# without USE_MOCK_SIP=true refuses to start. +USE_MOCK_SIP=false SIP_TRUNK_HOST=sip.yourprovider.com SIP_TRUNK_PORT=5060 SIP_TRUNK_USERNAME=your_sip_username diff --git a/api/websocket.py b/api/websocket.py index 85bef08..9673dfd 100644 --- a/api/websocket.py +++ b/api/websocket.py @@ -94,7 +94,7 @@ async def event_stream(websocket: WebSocket): # Immediately push current trunk status so the dashboard doesn't start blank await _send_trunk_status(websocket, gateway) - subscription = gateway.event_bus.subscribe() + subscription = gateway.event_bus.subscribe(replay_last=25) try: async for event in subscription: diff --git a/config.py b/config.py index f637bca..eee0626 100644 --- a/config.py +++ b/config.py @@ -11,7 +11,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict class SIPTrunkSettings(BaseSettings): """SIP trunk provider configuration.""" - model_config = SettingsConfigDict(env_prefix="SIP_TRUNK_") + model_config = SettingsConfigDict(env_prefix="SIP_TRUNK_", env_file=".env", extra="ignore") host: str = "sip.provider.com" port: int = 5060 @@ -24,7 +24,7 @@ class SIPTrunkSettings(BaseSettings): class GatewaySIPSettings(BaseSettings): """Gateway SIP listener for device registration.""" - model_config = SettingsConfigDict(env_prefix="GATEWAY_SIP_") + model_config = SettingsConfigDict(env_prefix="GATEWAY_SIP_", env_file=".env", extra="ignore") host: str = "0.0.0.0" port: int = 5060 @@ -34,7 +34,7 @@ class GatewaySIPSettings(BaseSettings): class SpeachesSettings(BaseSettings): """Speaches STT service configuration.""" - model_config = SettingsConfigDict(env_prefix="SPEACHES_") + model_config = SettingsConfigDict(env_prefix="SPEACHES_", env_file=".env", extra="ignore") url: str = "http://localhost:22070" model: str = "whisper-large-v3" @@ -43,7 +43,7 @@ class SpeachesSettings(BaseSettings): class ClassifierSettings(BaseSettings): """Audio classifier thresholds.""" - model_config = SettingsConfigDict(env_prefix="CLASSIFIER_") + model_config = SettingsConfigDict(env_prefix="CLASSIFIER_", env_file=".env", extra="ignore") music_threshold: float = 0.7 speech_threshold: float = 0.6 @@ -54,7 +54,7 @@ class ClassifierSettings(BaseSettings): class LLMSettings(BaseSettings): """LLM service configuration (OpenAI-compatible API).""" - model_config = SettingsConfigDict(env_prefix="LLM_") + model_config = SettingsConfigDict(env_prefix="LLM_", env_file=".env", extra="ignore") base_url: str = "http://localhost:11434/v1" model: str = "llama3" @@ -67,7 +67,7 @@ class LLMSettings(BaseSettings): class HoldSlayerSettings(BaseSettings): """Hold Slayer behavior settings.""" - model_config = SettingsConfigDict(env_prefix="HOLD_SLAYER_", env_prefix_allow_empty=True) + model_config = SettingsConfigDict(env_prefix="HOLD_SLAYER_", env_prefix_allow_empty=True, env_file=".env", extra="ignore") default_transfer_device: str = Field( default="sip_phone", validation_alias="DEFAULT_TRANSFER_DEVICE" @@ -79,7 +79,7 @@ class HoldSlayerSettings(BaseSettings): class TTSSettings(BaseSettings): """Rhema TTS service configuration (OpenAI-compatible /v1/audio/speech).""" - model_config = SettingsConfigDict(env_prefix="TTS_") + model_config = SettingsConfigDict(env_prefix="TTS_", env_file=".env", extra="ignore") base_url: str = "http://localhost:8000" model: str = "speaches-ai/Kokoro-82M-v1.0-ONNX" @@ -92,7 +92,7 @@ class TTSSettings(BaseSettings): class ReceptionistSettings(BaseSettings): """AI Receptionist behavior settings.""" - model_config = SettingsConfigDict(env_prefix="RECEPTIONIST_") + model_config = SettingsConfigDict(env_prefix="RECEPTIONIST_", env_file=".env", extra="ignore") enabled: bool = True greeting_template: str = ( @@ -133,6 +133,11 @@ class Settings(BaseSettings): # Outbound-call safety cap (REST + MCP make_call) max_concurrent_calls: int = 4 + # Explicit engine mode — the mock engine must be asked for. An + # unconfigured trunk without this flag fails startup instead of + # silently degrading to a gateway that can't place real calls. + use_mock_sip: bool = False + # Notifications notify_sms_number: str = "" diff --git a/core/event_bus.py b/core/event_bus.py index f6270a7..00715ce 100644 --- a/core/event_bus.py +++ b/core/event_bus.py @@ -20,63 +20,70 @@ class EventBus: Features: - Non-blocking publish (put_nowait) - - Automatic dead-subscriber cleanup (full queues are removed) - - Event history (last N events for late joiners) + - Slow subscribers lose their oldest event, never their subscription + - Event history (last N events, replayable to late joiners) - Typed event filtering on subscriptions - Async iteration via EventSubscription """ def __init__(self, max_history: int = 1000): - self._subscribers: list[tuple[asyncio.Queue[GatewayEvent], Optional[set[EventType]]]] = [] + self._subscribers: list[EventSubscription] = [] self._history: list[GatewayEvent] = [] self._max_history = max_history async def publish(self, event: GatewayEvent) -> None: - """Publish an event to all subscribers.""" + """Publish an event to all subscribers. + + A full subscriber queue drops its oldest event (counted on the + subscription) — a slow dashboard must never be silently + unsubscribed while its socket stays open. + """ self._history.append(event) if len(self._history) > self._max_history: self._history = self._history[-self._max_history :] logger.info(f"📡 Event: {event.type.value} | {event.message or ''}") - dead_queues = [] - for queue, type_filter in self._subscribers: - # Skip if subscriber has a type filter and this event doesn't match - if type_filter and event.type not in type_filter: + for sub in self._subscribers: + if sub.type_filter and event.type not in sub.type_filter: continue - try: - queue.put_nowait(event) - except asyncio.QueueFull: - dead_queues.append((queue, type_filter)) - - for entry in dead_queues: - self._subscribers.remove(entry) + sub.deliver(event) def subscribe( self, max_size: int = 100, event_types: Optional[set[EventType]] = None, + replay_last: int = 0, ) -> "EventSubscription": """ Create a new subscription. Args: - max_size: Queue depth before subscriber is considered dead. + max_size: Queue depth; overflow drops the oldest event. event_types: Optional filter — only receive these event types. None means receive everything. + replay_last: Seed the queue with up to N most recent + history events (post-filter) before live ones. Returns: An async iterator of GatewayEvents. """ queue: asyncio.Queue[GatewayEvent] = asyncio.Queue(maxsize=max_size) - entry = (queue, event_types) - self._subscribers.append(entry) - return EventSubscription(queue, self, entry) + sub = EventSubscription(queue, self, event_types) + if replay_last > 0: + replayable = [ + e for e in self._history + if not event_types or e.type in event_types + ] + for event in replayable[-replay_last:]: + sub.deliver(event) + self._subscribers.append(sub) + return sub - def unsubscribe(self, entry: tuple) -> None: + def unsubscribe(self, sub: "EventSubscription") -> None: """Remove a subscriber.""" - if entry in self._subscribers: - self._subscribers.remove(entry) + if sub in self._subscribers: + self._subscribers.remove(sub) @property def recent_events(self) -> list[GatewayEvent]: @@ -95,11 +102,28 @@ class EventSubscription: self, queue: asyncio.Queue[GatewayEvent], bus: EventBus, - entry: tuple, + type_filter: Optional[set[EventType]] = None, ): self._queue = queue self._bus = bus - self._entry = entry + self.type_filter = type_filter + self.dropped = 0 # events lost to queue overflow + + def deliver(self, event: GatewayEvent) -> None: + """Enqueue an event, dropping the oldest on overflow.""" + try: + self._queue.put_nowait(event) + except asyncio.QueueFull: + try: + self._queue.get_nowait() + self._queue.put_nowait(event) + except (asyncio.QueueEmpty, asyncio.QueueFull): + pass + self.dropped += 1 + if self.dropped in (1, 10, 100) or self.dropped % 1000 == 0: + logger.warning( + f"📡 Slow subscriber: {self.dropped} events dropped" + ) def __aiter__(self): return self @@ -108,7 +132,7 @@ class EventSubscription: try: return await self._queue.get() except asyncio.CancelledError: - self._bus.unsubscribe(self._entry) + self._bus.unsubscribe(self) raise async def get(self, timeout: Optional[float] = None) -> GatewayEvent: @@ -117,4 +141,4 @@ class EventSubscription: def close(self): """Unsubscribe from the event bus.""" - self._bus.unsubscribe(self._entry) + self._bus.unsubscribe(self) diff --git a/core/gateway.py b/core/gateway.py index 327e6a1..608f058 100644 --- a/core/gateway.py +++ b/core/gateway.py @@ -33,32 +33,43 @@ def build_sip_engine( on_device_registered: Callable, on_incoming_call: Callable, ) -> SIPEngine: - """Build the appropriate SIP engine from config.""" + """ + Build the SIP engine from config. + + The mock engine must be requested explicitly (USE_MOCK_SIP=true). + An unconfigured trunk or a failed SippyEngine construction raises — + the caller fails startup rather than running a gateway that can't + place real calls while reporting healthy. + """ + if settings.use_mock_sip: + logger.warning("🧪 USE_MOCK_SIP=true — SIP engine is a mock, no real calls") + return MockSIPEngine() + trunk = settings.sip_trunk gw_sip = settings.gateway_sip - if trunk.host and trunk.host != "sip.provider.com": - # Real trunk configured — use Sippy B2BUA - try: - return SippyEngine( - sip_address=gw_sip.host, - sip_port=gw_sip.port, - trunk_host=trunk.host, - trunk_port=trunk.port, - trunk_username=trunk.username, - trunk_password=trunk.password.get_secret_value(), - trunk_transport=trunk.transport, - domain=gw_sip.domain, - did=trunk.did, - media_pipeline=media_pipeline, - on_leg_state_change=on_leg_state_change, - on_device_registered=on_device_registered, - on_incoming_call=on_incoming_call, - ) - except Exception as e: - logger.warning(f"Could not create SippyEngine: {e} — using mock") + if not trunk.host or trunk.host in ("sip.provider.com", "sip.yourprovider.com"): + raise RuntimeError( + "SIP trunk is not configured (SIP_TRUNK_HOST is unset or a " + "placeholder). Set SIP_TRUNK_* in .env, or set USE_MOCK_SIP=true " + "for development without a trunk." + ) - return MockSIPEngine() + return SippyEngine( + sip_address=gw_sip.host, + sip_port=gw_sip.port, + trunk_host=trunk.host, + trunk_port=trunk.port, + trunk_username=trunk.username, + trunk_password=trunk.password.get_secret_value(), + trunk_transport=trunk.transport, + domain=gw_sip.domain, + did=trunk.did, + media_pipeline=media_pipeline, + on_leg_state_change=on_leg_state_change, + on_device_registered=on_device_registered, + on_incoming_call=on_incoming_call, + ) class AIPSTNGateway: diff --git a/core/sippy_engine.py b/core/sippy_engine.py index 4c56a83..aeaf6b0 100644 --- a/core/sippy_engine.py +++ b/core/sippy_engine.py @@ -50,7 +50,6 @@ class SipCallLeg: self.state = "init" # init, trying, ringing, connected, terminated self.media_port: Optional[int] = None # PJSUA2 conf bridge port self.pending_sdp: Optional[str] = None # inbound INVITE SDP, until answered - self.dtmf_buffer: list[str] = [] def __repr__(self): return f"" @@ -297,9 +296,8 @@ class SippyEngine(SIPEngine): ] elif kind == "dtmf": - leg = self._legs.get(data["leg_id"]) - if leg: - leg.dtmf_buffer.append(data["digit"]) + # Received DTMF has no consumer yet; log until one exists + logger.info(f" DTMF '{data['digit']}' received on {data['leg_id']}") elif kind == "trunk_registered": self._trunk_registered = data["registered"] diff --git a/db/database.py b/db/database.py index c93d658..5ddd6cf 100644 --- a/db/database.py +++ b/db/database.py @@ -85,24 +85,6 @@ class StoredCallFlow(Base): return f"" -class Contact(Base): - __tablename__ = "contacts" - - id = Column(String, primary_key=True) - name = Column(String, nullable=False) - phone_numbers = Column(JSON, nullable=False) # [{number, label, primary}, ...] - category = Column(String) # personal / business / service - routing_preference = Column(String, nullable=True) # how to handle their calls - notes = Column(Text, nullable=True) - call_count = Column(Integer, default=0) - last_call = Column(DateTime, nullable=True) - created_at = Column(DateTime, default=func.now()) - updated_at = Column(DateTime, default=func.now(), onupdate=func.now()) - - def __repr__(self) -> str: - return f"" - - class Device(Base): __tablename__ = "devices" diff --git a/main.py b/main.py index 5a6978d..dd11fc7 100644 --- a/main.py +++ b/main.py @@ -172,18 +172,23 @@ async def lifespan(app: FastAPI): gateway.register_mode_handler(CallMode.HOLD_SLAYER, launch_hold_slayer) - gateway.sip_engine = build_sip_engine( - settings, - gateway.media_pipeline, - on_leg_state_change=gateway._on_sip_leg_state, - on_device_registered=gateway._on_sip_device_registered, - on_incoming_call=receptionist.on_inbound_call, - ) + try: + gateway.sip_engine = build_sip_engine( + settings, + gateway.media_pipeline, + on_leg_state_change=gateway._on_sip_leg_state, + on_device_registered=gateway._on_sip_device_registered, + on_incoming_call=receptionist.on_inbound_call, + ) + except Exception as e: + logger.critical(f"\n❌ SIP engine failed to initialize:\n {e}") + sys.exit(1) await routing_svc.start() await gateway.start() app.state.gateway = gateway app.state.routing_service = routing_svc + app.state.transcription_service = transcription notification_svc = NotificationService(gateway.event_bus, settings) await notification_svc.start() @@ -298,21 +303,67 @@ async def root(): @app.get("/health", tags=["System"]) async def health(): - """Health check endpoint.""" + """ + Health check. "healthy" means the gateway can actually do its job: + real engine, registered trunk, reachable database. A mock engine or + a failing dependency reports "degraded" with the reason visible. + """ + from core.sip_engine import MockSIPEngine + from db.database import session_scope + gateway = getattr(app.state, "gateway", None) ready = gateway is not None and await gateway.sip_engine.is_ready() trunk_status = await gateway.sip_engine.get_trunk_status() if gateway else {"registered": False} - return { - "status": "healthy" if ready else "degraded", + engine_mode = ( + "mock" if gateway is None or isinstance(gateway.sip_engine, MockSIPEngine) + else "sippy" + ) + + db_ok = False + db_error = None + try: + from sqlalchemy import text + async with session_scope() as session: + await session.execute(text("SELECT 1")) + db_ok = True + except Exception as e: + db_error = str(e)[:200] + + healthy = ( + ready + and db_ok + and engine_mode == "sippy" + and trunk_status.get("registered", False) + ) + + checks = { "gateway": "ready" if gateway else "not initialized", + "engine": engine_mode, "sip_engine": "ready" if ready else "not ready", + "database": "ok" if db_ok else f"error: {db_error}", "sip_trunk": { "registered": trunk_status.get("registered", False), "host": trunk_status.get("host"), - "mock": trunk_status.get("mock", False), "reason": trunk_status.get("reason"), }, } + if gateway is not None: + tts = getattr(gateway, "_tts", None) + checks["tts"] = _availability(tts) + transcription = getattr(app.state, "transcription_service", None) + checks["stt"] = _availability(transcription) + + return {"status": "healthy" if healthy else "degraded", **checks} + + +def _availability(service) -> str: + """Last-known reachability of an HTTP leaf service.""" + if service is None: + return "not attached" + available = getattr(service, "available", None) + if available is None: + return "unknown (no requests yet)" + return "ok" if available else "unreachable" if __name__ == "__main__": diff --git a/models/contact.py b/models/contact.py deleted file mode 100644 index 9a0e973..0000000 --- a/models/contact.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Contact models — People and organizations you call. -""" - -from datetime import datetime -from typing import Optional - -from pydantic import BaseModel, Field - - -class PhoneNumber(BaseModel): - """A phone number associated with a contact.""" - - number: str # E.164 format - label: str = "main" # main, mobile, work, home, fax, etc. - primary: bool = False - - -class ContactBase(BaseModel): - """Shared contact fields.""" - - name: str - phone_numbers: list[PhoneNumber] - category: Optional[str] = None # personal / business / service - routing_preference: Optional[str] = None # how to handle their calls - notes: Optional[str] = None - - -class Contact(ContactBase): - """Full contact model.""" - - id: str - call_count: int = 0 - last_call: Optional[datetime] = None - created_at: Optional[datetime] = None - updated_at: Optional[datetime] = None - - @property - def primary_number(self) -> Optional[str]: - """Get the primary phone number.""" - for pn in self.phone_numbers: - if pn.primary: - return pn.number - return self.phone_numbers[0].number if self.phone_numbers else None - - -class ContactCreate(ContactBase): - """Request model for creating a contact.""" - - pass - - -class ContactUpdate(BaseModel): - """Request model for updating a contact.""" - - name: Optional[str] = None - phone_numbers: Optional[list[PhoneNumber]] = None - category: Optional[str] = None - routing_preference: Optional[str] = None - notes: Optional[str] = None diff --git a/services/call_persistence.py b/services/call_persistence.py index af57d77..d12c8c5 100644 --- a/services/call_persistence.py +++ b/services/call_persistence.py @@ -7,6 +7,7 @@ surfaces can't drift. Every function takes an AsyncSession; callers own the transaction (get_db for REST, session_scope for MCP/services). """ +import asyncio import logging import uuid from datetime import datetime @@ -159,53 +160,67 @@ async def persist_call_on_end(call: ActiveCall, final_status: CallStatus) -> Non """Insert a CallRecord and any transcript chunks for `call`. Wired into CallManager as its on_call_ended hook by the - composition root in main.py. + composition root in main.py. Retries briefly — losing the row + means the call never happened as far as history is concerned, so + the final failure logs at ERROR with the payload identifiers. """ - try: - async with session_scope() as session: - record = CallRecord( - id=call.id, - direction=call.direction, - remote_number=call.remote_number, - status=final_status.value, - mode=call.mode.value, - intent=call.intent, - started_at=call.started_at, - ended_at=datetime.now(), - duration=int(call.duration), - hold_time=int(call.hold_time), - device_used=call.device, - call_flow_id=call.call_flow_id, - classification_timeline=[ - { - "timestamp": c.timestamp, - "audio_type": c.audio_type.value, - "confidence": c.confidence, - } - for c in call.classification_history - ], - metadata_={"services": list(call.services)}, - ) - session.add(record) + for attempt in range(3): + try: + await _write_call_record(call, final_status) + return + except Exception as e: + if attempt == 2: + logger.error( + f"Call record lost: id={call.id} number={call.remote_number} " + f"status={final_status.value}: {e}" + ) + return + await asyncio.sleep(2**attempt) - # Each transcript chunk gets its own row with a sequence number - # so the dashboard can render them in order with click-to-seek. - for seq, text in enumerate(call.transcript_chunks): - speaker = "unknown" - payload = text - if ":" in text: - head, rest = text.split(":", 1) - head = head.strip().lower() - if head in {"caller", "agent", "receptionist", "caller_message"}: - speaker = head if head != "caller_message" else "caller" - payload = rest.strip() - session.add(TranscriptChunk( - id=f"tc_{uuid.uuid4().hex[:10]}", - call_id=call.id, - seq=seq, - t_offset_ms=0, - speaker=speaker, - text=payload, - )) - except Exception as e: - logger.warning(f"Could not persist call {call.id}: {e}") + +async def _write_call_record(call: ActiveCall, final_status: CallStatus) -> None: + async with session_scope() as session: + record = CallRecord( + id=call.id, + direction=call.direction, + remote_number=call.remote_number, + status=final_status.value, + mode=call.mode.value, + intent=call.intent, + started_at=call.started_at, + ended_at=datetime.now(), + duration=int(call.duration), + hold_time=int(call.hold_time), + device_used=call.device, + call_flow_id=call.call_flow_id, + classification_timeline=[ + { + "timestamp": c.timestamp, + "audio_type": c.audio_type.value, + "confidence": c.confidence, + } + for c in call.classification_history + ], + metadata_={"services": list(call.services)}, + ) + session.add(record) + + # Each transcript chunk gets its own row with a sequence number + # so the dashboard can render them in order with click-to-seek. + for seq, text in enumerate(call.transcript_chunks): + speaker = "unknown" + payload = text + if ":" in text: + head, rest = text.split(":", 1) + head = head.strip().lower() + if head in {"caller", "agent", "receptionist", "caller_message"}: + speaker = head if head != "caller_message" else "caller" + payload = rest.strip() + session.add(TranscriptChunk( + id=f"tc_{uuid.uuid4().hex[:10]}", + call_id=call.id, + seq=seq, + t_offset_ms=0, + speaker=speaker, + text=payload, + )) diff --git a/services/hold_slayer.py b/services/hold_slayer.py index 4d6031b..b92f781 100644 --- a/services/hold_slayer.py +++ b/services/hold_slayer.py @@ -56,6 +56,29 @@ class HoldSlayerService: self.settings = settings self.tts = tts + async def _service_error(self, call_id: str, service: str, error: Exception) -> None: + """Surface a failed dependency as a typed event, not silence.""" + logger.error(f"⚠️ {service} failed for {call_id}: {error}") + try: + await self.gateway.event_bus.publish(GatewayEvent( + type=EventType.ERROR, + call_id=call_id, + data={"service": service, "error": str(error)}, + message=f"⚠️ {service} failed: {error}", + )) + except Exception: + pass + + async def _transcribe( + self, call_id: str, audio: bytes, prompt: Optional[str] = None + ) -> str: + """Transcribe with an explicit empty-string fallback on failure.""" + try: + return await self.transcription.transcribe(audio, prompt=prompt) + except Exception as e: + await self._service_error(call_id, "transcription", e) + return "" + async def run( self, call: ActiveCall, @@ -309,9 +332,10 @@ class HoldSlayerService: AudioClassification.IVR_PROMPT, AudioClassification.LIVE_HUMAN, ): - transcript = await self.transcription.transcribe( + transcript = await self._transcribe( + call.id, audio_chunk, - prompt="Phone IVR menu, customer service, press 1 for..." + prompt="Phone IVR menu, customer service, press 1 for...", ) if transcript: await self.call_manager.add_transcript(call.id, transcript) @@ -429,7 +453,7 @@ class HoldSlayerService: # Check for human if result.audio_type == AudioClassification.LIVE_HUMAN: # Verify with transcription - transcript = await self.transcription.transcribe(audio_chunk) + transcript = await self._transcribe(call.id, audio_chunk) if transcript: await self.call_manager.add_transcript(call.id, transcript) # If we got meaningful speech, it's probably a real person @@ -491,7 +515,7 @@ class HoldSlayerService: continue # Transcribe - transcript = await self.transcription.transcribe(audio_chunk) + transcript = await self._transcribe(call.id, audio_chunk) if not transcript: continue @@ -545,7 +569,7 @@ class HoldSlayerService: AudioClassification.IVR_PROMPT, AudioClassification.LIVE_HUMAN, ): - text = await self.transcription.transcribe(audio_chunk) + text = await self._transcribe(call.id, audio_chunk) if text: transcript_parts.append(text) @@ -713,7 +737,11 @@ class HoldSlayerService: os.close(fd) try: - ok = await self.tts.synthesize_to_file(text, tmp_path) + try: + ok = await self.tts.synthesize_to_file(text, tmp_path) + except Exception as e: + await self._service_error(call.id, "tts", e) + return False if not ok: logger.warning(f"🗣️ TTS synthesis returned no audio for: '{text[:60]}'") return False diff --git a/services/llm_client.py b/services/llm_client.py index b754afa..667c12a 100644 --- a/services/llm_client.py +++ b/services/llm_client.py @@ -327,15 +327,14 @@ class LLMClient: except httpx.HTTPStatusError as e: self._total_errors += 1 logger.error(f"LLM API error: {e.response.status_code} {e.response.text[:200]}") - return "" + raise except httpx.TimeoutException: self._total_errors += 1 logger.error(f"LLM API timeout after {self.timeout}s") - return "" - except Exception as e: + raise + except Exception: self._total_errors += 1 - logger.error(f"LLM client error: {e}") - return "" + raise @staticmethod def _parse_json_response(text: str) -> dict[str, Any]: diff --git a/services/notification.py b/services/notification.py index 0a17999..c1d859d 100644 --- a/services/notification.py +++ b/services/notification.py @@ -67,7 +67,6 @@ class NotificationService: self._event_bus = event_bus self._settings = settings self._task: Optional[asyncio.Task] = None - self._sms_sender: Optional[Any] = None # Track what we've already notified (avoid spam) self._notified: dict[str, set[str]] = {} # call_id -> set of event types @@ -214,43 +213,3 @@ class NotificationService: # WebSocket notifications go through the event bus # (the WebSocket handler in the API reads from EventBus directly) - - # SMS for critical notifications - if ( - notification.priority == NotificationPriority.CRITICAL - and self._settings.notify_sms_number - ): - await self._send_sms(notification) - - async def _send_sms(self, notification: Notification) -> None: - """ - Send an SMS notification. - - Uses a simple HTTP-based SMS gateway. In production, - this would use Twilio, AWS SNS, or similar. - """ - phone = self._settings.notify_sms_number - if not phone: - return - - try: - import httpx - - # Generic webhook-based SMS (configure your provider) - # This is a placeholder — wire up your preferred SMS provider - logger.info(f"📱 SMS → {phone}: {notification.title}") - - # Example: Twilio-style API - # async with httpx.AsyncClient() as client: - # await client.post( - # "https://api.twilio.com/2010-04-01/Accounts/.../Messages.json", - # data={ - # "To": phone, - # "From": self._settings.sip_trunk.did, - # "Body": f"{notification.title}\n{notification.message}", - # }, - # auth=(account_sid, auth_token), - # ) - - except Exception as e: - logger.error(f"SMS send failed: {e}") diff --git a/services/receptionist.py b/services/receptionist.py index 1ecc071..8999b30 100644 --- a/services/receptionist.py +++ b/services/receptionist.py @@ -203,6 +203,19 @@ class ReceptionistService: # State machine steps # ---------------------------------------------------------------- + async def _service_error(self, call_id: str, service: str, error: Exception) -> None: + """Surface a failed dependency as a typed event, not silence.""" + logger.error(f"⚠️ {service} failed for {call_id}: {error}") + try: + await self.gateway.event_bus.publish(GatewayEvent( + type=EventType.ERROR, + call_id=call_id, + data={"service": service, "error": str(error)}, + message=f"⚠️ {service} failed: {error}", + )) + except Exception: + pass + async def _greet(self, call: ActiveCall, sip_leg_id: str) -> None: await self.gateway.event_bus.publish(GatewayEvent( type=EventType.RECEPTIONIST_GREETING, @@ -250,7 +263,11 @@ class ReceptionistService: if not audio or self.transcription is None: return "" - return await self.transcription.transcribe(bytes(audio)) + try: + return await self.transcription.transcribe(bytes(audio)) + except Exception as e: + await self._service_error(call.id, "transcription", e) + return "" async def _classify( self, @@ -291,7 +308,7 @@ class ReceptionistService: system=self.settings.llm_persona, ) except Exception as e: - logger.warning(f"Receptionist LLM classify failed: {e}") + await self._service_error(call.id, "llm", e) return { "intent": transcript, "urgency": "normal", @@ -304,10 +321,14 @@ class ReceptionistService: routing_decision: Optional[RoutingDecision], classification: dict, ) -> RoutingAction: - """Rules win on conflict; otherwise use the LLM's recommendation.""" - if routing_decision and routing_decision.action.type not in ( - RoutingActionType.TAKE_MESSAGE, - ): + """Rules win on conflict; otherwise use the LLM's recommendation. + + A decision counts as a rule only when one actually matched + (matched_rule_id set) — the no-rule default is take_message and + must stay overridable by the LLM. A matched TAKE_MESSAGE rule + wins like any other rule. + """ + if routing_decision and routing_decision.matched_rule_id: return routing_decision.action recommended = (classification.get("recommended_action") or "ring").lower() @@ -347,36 +368,42 @@ class ReceptionistService: call.id, media_pipeline=media, leg_ids=[sip_leg_id] ) try: - await asyncio.sleep(self.settings.message_max_seconds) + # Record up to the cap, but stop early once the caller hangs + # up (leg termination ends the call via the leg-state wiring). + deadline = _time.monotonic() + self.settings.message_max_seconds + while _time.monotonic() < deadline: + await asyncio.sleep(1.0) + if self.gateway.call_manager.get_call(call.id) is None: + break finally: session = await recording_svc.stop_recording( call.id, media_pipeline=media ) - message_text = "" - rec_path = session.filepath_mixed if session else None - if rec_path and Path(rec_path).exists() and self.transcription is not None: - try: - audio_bytes = Path(rec_path).read_bytes() - message_text = await self.transcription.transcribe(audio_bytes) - except Exception as e: - logger.warning(f"Receptionist transcribe failed: {e}") + message_text = "" + rec_path = session.filepath_mixed if session else None + if rec_path and Path(rec_path).exists() and self.transcription is not None: + try: + audio_bytes = Path(rec_path).read_bytes() + message_text = await self.transcription.transcribe(audio_bytes) + except Exception as e: + await self._service_error(call.id, "transcription", e) - if message_text: - call.transcript_chunks.append(f"caller_message: {message_text}") + if message_text: + call.transcript_chunks.append(f"caller_message: {message_text}") - await self.gateway.event_bus.publish(GatewayEvent( - type=EventType.RECEPTIONIST_MESSAGE_SAVED, - call_id=call.id, - data={ - "path": rec_path, - "transcript": message_text, - "caller": call.remote_number, - }, - message=f"📥 Message saved from {call.remote_number}", - )) + await self.gateway.event_bus.publish(GatewayEvent( + type=EventType.RECEPTIONIST_MESSAGE_SAVED, + call_id=call.id, + data={ + "path": rec_path, + "transcript": message_text, + "caller": call.remote_number, + }, + message=f"📥 Message saved from {call.remote_number}", + )) - await self._hangup(call, sip_leg_id) + await self._hangup(call, sip_leg_id) # ---------------------------------------------------------------- # Helpers @@ -394,7 +421,11 @@ class ReceptionistService: fd, tmp_path = tempfile.mkstemp(suffix=".wav", prefix=f"recept_{call.id}_") os.close(fd) try: - ok = await tts.synthesize_to_file(text, tmp_path) + try: + ok = await tts.synthesize_to_file(text, tmp_path) + except Exception as e: + await self._service_error(call.id, "tts", e) + return if not ok: return await media.play_wav(sip_leg_id, tmp_path) diff --git a/services/recording.py b/services/recording.py index f80b551..375e306 100644 --- a/services/recording.py +++ b/services/recording.py @@ -91,6 +91,7 @@ class RecordingService: filepath_agent=filepath_agent, started_at=datetime.now(), sample_rate=self._sample_rate, + leg_ids=leg_ids, ) # Start PJSUA2 recording if media pipeline is available @@ -159,26 +160,38 @@ class RecordingService: @staticmethod async def _persist_recording(session: "RecordingSession") -> None: - """Write a recordings row for this session. Failures are non-fatal.""" - try: - import uuid as _uuid - from db.database import RecordingRecord, get_session_factory + """Write a recordings row for this session, with bounded retry. - async with get_session_factory()() as db: - db.add(RecordingRecord( - id=f"rec_{_uuid.uuid4().hex[:10]}", - call_id=session.call_id, - path=session.filepath_mixed or "", - format="wav", - duration_s=float(session.duration_seconds or 0), - size_bytes=int(session.file_size_bytes or 0), - channels=1, - started_at=session.started_at, - ended_at=session.stopped_at, - )) - await db.commit() - except Exception as e: - logger.warning(f"Recording persistence failed: {e}") + Non-fatal for the call, but a lost row means the dashboard can + never find the WAV — so failures log at ERROR, not warning. + """ + import uuid as _uuid + + from db.database import RecordingRecord, session_scope + + for attempt in range(3): + try: + async with session_scope() as db: + db.add(RecordingRecord( + id=f"rec_{_uuid.uuid4().hex[:10]}", + call_id=session.call_id, + path=session.filepath_mixed or "", + format="wav", + duration_s=float(session.duration_seconds or 0), + size_bytes=int(session.file_size_bytes or 0), + channels=1, + started_at=session.started_at, + ended_at=session.stopped_at, + )) + return + except Exception as e: + if attempt == 2: + logger.error( + f"Recording row lost for {session.call_id} " + f"(path={session.filepath_mixed}): {e}" + ) + return + await asyncio.sleep(2 ** attempt) async def _recording_timeout(self, call_id: str) -> None: """Auto-stop recording after max duration.""" @@ -239,6 +252,7 @@ class RecordingSession: filepath_agent: Optional[str] = None, started_at: Optional[datetime] = None, sample_rate: int = 16000, + leg_ids: Optional[list[str]] = None, ): self.call_id = call_id self.filepath_mixed = filepath_mixed @@ -249,7 +263,7 @@ class RecordingSession: self.duration_seconds: Optional[int] = None self.file_size_bytes: Optional[int] = None self.sample_rate = sample_rate - self._leg_ids: list[str] = [] + self._leg_ids: list[str] = list(leg_ids or []) def to_dict(self) -> dict: return { diff --git a/services/transcription.py b/services/transcription.py index 183fed1..9a745cd 100644 --- a/services/transcription.py +++ b/services/transcription.py @@ -27,6 +27,8 @@ class TranscriptionService: def __init__(self, settings: SpeachesSettings): self.settings = settings self._client: Optional[httpx.AsyncClient] = None + # Last-known reachability, surfaced by /health (None = no requests yet) + self.available: Optional[bool] = None async def _get_client(self) -> httpx.AsyncClient: """Get or create the HTTP client.""" @@ -60,6 +62,9 @@ class TranscriptionService: # Convert raw PCM to WAV format for the API wav_data = self._pcm_to_wav(audio_data) + # Raises on failure — callers decide the per-call fallback and + # publish a service-error event; swallowing here made a down + # Speaches look like "the AI is deciding badly". try: response = await client.post( "/v1/audio/transcriptions", @@ -72,44 +77,13 @@ class TranscriptionService: }, ) response.raise_for_status() - text = response.text.strip() - logger.debug(f"Transcription: '{text}'") - return text - - except httpx.HTTPStatusError as e: - logger.error(f"Speaches API error: {e.response.status_code} {e.response.text}") - return "" - except httpx.ConnectError: - logger.error(f"Cannot connect to Speaches at {self.settings.url}") - return "" - except Exception as e: - logger.error(f"Transcription failed: {e}") - return "" - - async def transcribe_stream( - self, - audio_data: bytes, - language: str = "en", - ): - """ - Stream transcription — for real-time results. - - Uses Speaches streaming endpoint if available, - falls back to chunked transcription. - - Yields: - str: Partial transcription chunks - """ - # For now, do chunked transcription - # TODO: Implement WebSocket streaming when Speaches supports it - chunk_size = 16000 * 2 * 3 # 3 seconds of 16kHz 16-bit mono - - for i in range(0, len(audio_data), chunk_size): - chunk = audio_data[i:i + chunk_size] - if len(chunk) > 0: - text = await self.transcribe(chunk, language) - if text: - yield text + except Exception: + self.available = False + raise + self.available = True + text = response.text.strip() + logger.debug(f"Transcription: '{text}'") + return text async def close(self) -> None: """Close the HTTP client.""" diff --git a/services/tts.py b/services/tts.py index 2865c60..81e6e53 100644 --- a/services/tts.py +++ b/services/tts.py @@ -22,6 +22,8 @@ class TTSService: def __init__(self, settings: TTSSettings): self.settings = settings self._client: Optional[httpx.AsyncClient] = None + # Last-known reachability, surfaced by /health (None = no requests yet) + self.available: Optional[bool] = None async def _get_client(self) -> httpx.AsyncClient: if self._client is None or self._client.is_closed: @@ -54,19 +56,17 @@ class TTSService: "sample_rate": self.settings.sample_rate, } + # Raises on failure — callers decide the per-call fallback and + # publish a service-error event; swallowing here made a down + # Rhema look like "the AI went quiet". try: response = await client.post("/v1/audio/speech", json=body) response.raise_for_status() - return response.content - except httpx.HTTPStatusError as e: - logger.error(f"Rhema TTS error: {e.response.status_code} {e.response.text}") - return b"" - except httpx.ConnectError: - logger.error(f"Cannot connect to Rhema at {self.settings.base_url}") - return b"" - except Exception as e: - logger.error(f"TTS synthesis failed: {e}") - return b"" + except Exception: + self.available = False + raise + self.available = True + return response.content async def synthesize_to_file( self, diff --git a/tests/test_concurrency.py b/tests/test_concurrency.py index 570bae2..1cc1d05 100644 --- a/tests/test_concurrency.py +++ b/tests/test_concurrency.py @@ -102,14 +102,9 @@ class TestEngineEventFunnel: async def test_dtmf_and_trunk_events(self): engine = _engine_on_loop() - _post_from_thread(engine, "incoming_invite", { - "leg_id": "leg_test4", "from_uri": "a", "to_uri": "b", "sdp": None, - }) - await asyncio.sleep(0.05) _post_from_thread(engine, "dtmf", {"leg_id": "leg_test4", "digit": "5"}) _post_from_thread(engine, "trunk_registered", {"registered": True}) await asyncio.sleep(0.05) - assert engine._legs["leg_test4"].dtmf_buffer == ["5"] assert engine._trunk_registered is True diff --git a/tests/test_health_policy.py b/tests/test_health_policy.py new file mode 100644 index 0000000..4d6930a --- /dev/null +++ b/tests/test_health_policy.py @@ -0,0 +1,89 @@ +""" +Engine-mode and event-bus-integrity tests. + +The mock engine must be requested explicitly; a full subscriber queue +drops its oldest event but never loses the subscription; history is +replayable to late joiners. +""" + +import asyncio + +import pytest + +from config import Settings +from core.event_bus import EventBus +from core.gateway import build_sip_engine +from core.media_pipeline import MediaPipeline +from core.sip_engine import MockSIPEngine +from models.events import EventType, GatewayEvent + + +def _noop(*args, **kwargs): + pass + + +class TestEngineMode: + def _build(self, settings: Settings): + return build_sip_engine( + settings, + MediaPipeline(sample_rate=16000), + on_leg_state_change=_noop, + on_device_registered=_noop, + on_incoming_call=_noop, + ) + + def test_mock_engine_only_when_asked(self): + settings = Settings(use_mock_sip=True) + assert isinstance(self._build(settings), MockSIPEngine) + + def test_unconfigured_trunk_refuses_to_build(self): + settings = Settings(use_mock_sip=False) + settings.sip_trunk.host = "sip.yourprovider.com" + with pytest.raises(RuntimeError, match="not configured"): + self._build(settings) + + +def _event(i: int) -> GatewayEvent: + return GatewayEvent( + type=EventType.CALL_INITIATED, + call_id=f"call_{i}", + data={}, + message=f"event {i}", + ) + + +class TestEventBusIntegrity: + async def test_overflow_drops_oldest_keeps_subscription(self): + bus = EventBus() + sub = bus.subscribe(max_size=3) + + for i in range(5): + await bus.publish(_event(i)) + + assert bus.subscriber_count == 1 # never evicted + assert sub.dropped == 2 + received = [await asyncio.wait_for(sub.get(), 1.0) for _ in range(3)] + assert [e.call_id for e in received] == ["call_2", "call_3", "call_4"] + + async def test_replay_last_seeds_history(self): + bus = EventBus() + for i in range(10): + await bus.publish(_event(i)) + + sub = bus.subscribe(replay_last=3) + received = [await asyncio.wait_for(sub.get(), 1.0) for _ in range(3)] + assert [e.call_id for e in received] == ["call_7", "call_8", "call_9"] + + async def test_replay_respects_type_filter(self): + bus = EventBus() + await bus.publish(_event(1)) + await bus.publish(GatewayEvent( + type=EventType.HUMAN_DETECTED, call_id="call_h", data={}, message="x" + )) + + sub = bus.subscribe( + event_types={EventType.HUMAN_DETECTED}, replay_last=5 + ) + event = await asyncio.wait_for(sub.get(), 1.0) + assert event.call_id == "call_h" + assert sub._queue.empty() diff --git a/tests/test_receptionist.py b/tests/test_receptionist.py index d0b32f0..e90b923 100644 --- a/tests/test_receptionist.py +++ b/tests/test_receptionist.py @@ -24,10 +24,27 @@ class TestReceptionistDecide: gw = _make_gateway() svc = ReceptionistService(gw) rule_action = RoutingAction(type=RoutingActionType.REJECT, message="nope") - decision = RoutingDecision(action=rule_action, reason="rule said so") + decision = RoutingDecision( + action=rule_action, + matched_rule_id="rule_1", + matched_rule_name="block", + reason="rule said so", + ) chosen = svc._decide(decision, {"recommended_action": "ring"}) assert chosen.type == RoutingActionType.REJECT + def test_matched_take_message_rule_beats_llm(self): + gw = _make_gateway() + svc = ReceptionistService(gw) + decision = RoutingDecision( + action=RoutingAction(type=RoutingActionType.TAKE_MESSAGE), + matched_rule_id="rule_2", + matched_rule_name="voicemail-hours", + reason="matched rule 'voicemail-hours'", + ) + chosen = svc._decide(decision, {"recommended_action": "ring"}) + assert chosen.type == RoutingActionType.TAKE_MESSAGE + def test_falls_back_to_llm_when_rule_is_default_take_message(self): gw = _make_gateway() svc = ReceptionistService(gw) diff --git a/tests/test_services.py b/tests/test_services.py index ab9b5bf..fc54b4e 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -123,14 +123,14 @@ class TestLLMClient: assert result["key"] == "value" @pytest.mark.asyncio - async def test_chat_http_error_returns_empty(self): - """Verify HTTP errors return empty string gracefully.""" + async def test_chat_error_raises(self): + """Failures propagate to the caller (which owns the fallback).""" client = self._make_client() with patch.object(client._client, "post", new_callable=AsyncMock) as mock_post: mock_post.side_effect = Exception("Connection refused") - result = await client.chat("test", system="test") - assert result == "" + with pytest.raises(Exception, match="Connection refused"): + await client.chat("test", system="test") assert client._total_errors == 1 @pytest.mark.asyncio From f7a11f2f2086ce3fc6deaf1b7e5feed381a5650c Mon Sep 17 00:00:00 2001 From: Robert Helewka Date: Fri, 10 Jul 2026 07:42:52 -0400 Subject: [PATCH 4/5] Stage 5: Alembic migrations, durable call rows, one transcript truth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Alembic replaces create_all as the schema authority: async env.py against Base.metadata (CLI and in-app entry paths share it via config.attributes["connection"]), an autogenerated baseline of the create_all-era schema, and init_db now runs upgrade head — stamping the baseline first on a pre-Alembic database so existing deployments adopt cleanly. create_all remains for tests only. Calls are durable from the start: CallManager gains an on_call_created hook (wired to persist_call_on_create) that inserts an in_progress CallRecord the moment a call is created; persist_call_on_end finalizes that same row. A SIGKILL mid-call now leaves an in_progress row instead of erasing the call from history (verified live against the dev database). One transcript representation: ActiveCall.transcript_chunks holds TranscriptEntry (t_offset_ms, speaker, text) — add_transcript stamps real offsets from connect time, receptionist passes speaker instead of encoding it into "caller: ..." strings, persisted chunks carry real seek offsets, and the dead CallRecord.transcript Text column is dropped by migration. Device.is_online migrates String → Boolean (with a USING cast for existing rows). Model de-triplication: CallResponse/CallStatusResponse build via from_call classmethods (one ActiveCall→response mapping); DeviceStatus deleted — can_receive_call is a computed field on Device and the list endpoint returns the domain model; all row↔dict and row↔domain mapping now lives in call_persistence.py (record_summary/record_detail/chunk_to_dict + device row functions). New tests/test_data_layer.py: upgrade-head-matches-models, pre-Alembic adoption, durable in_progress rows, end-without-create fallback, transcript offsets, consolidated response models. Co-Authored-By: Claude Fable 5 --- alembic.ini | 42 ++++ api/call_history.py | 48 +---- api/calls.py | 30 +-- api/devices.py | 65 +----- core/call_manager.py | 39 +++- core/gateway.py | 7 +- db/database.py | 29 ++- db/migrations/env.py | 68 +++++++ db/migrations/script.py.mako | 26 +++ .../versions/1173a71329ed_baseline_schema.py | 129 ++++++++++++ ...drop_dead_transcript_column_boolean_is_.py | 45 +++++ main.py | 8 +- models/call.py | 41 +++- models/device.py | 14 +- pyproject.toml | 4 + services/call_persistence.py | 183 +++++++++++++---- services/receptionist.py | 14 +- tests/test_data_layer.py | 187 ++++++++++++++++++ 18 files changed, 777 insertions(+), 202 deletions(-) create mode 100644 alembic.ini create mode 100644 db/migrations/env.py create mode 100644 db/migrations/script.py.mako create mode 100644 db/migrations/versions/1173a71329ed_baseline_schema.py create mode 100644 db/migrations/versions/5187577efc23_drop_dead_transcript_column_boolean_is_.py create mode 100644 tests/test_data_layer.py diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..d58477c --- /dev/null +++ b/alembic.ini @@ -0,0 +1,42 @@ +# Alembic configuration. The database URL is not set here — env.py +# reads it from config.Settings (environment / .env), so CLI runs and +# app startup migrate the same database the app uses. + +[alembic] +script_location = db/migrations +prepend_sys_path = . +path_separator = os + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/api/call_history.py b/api/call_history.py index 59edbc4..aff3f5b 100644 --- a/api/call_history.py +++ b/api/call_history.py @@ -37,23 +37,7 @@ async def list_history( limit=limit, offset=offset, ) - return [ - { - "id": r.id, - "direction": r.direction, - "remote_number": r.remote_number, - "status": r.status, - "mode": r.mode, - "intent": r.intent, - "started_at": r.started_at.isoformat() if r.started_at else None, - "ended_at": r.ended_at.isoformat() if r.ended_at else None, - "duration": r.duration, - "hold_time": r.hold_time, - "device_used": r.device_used, - "summary": r.summary, - } - for r in rows - ] + return [store.record_summary(r) for r in rows] @router.get("/{call_id}/record") @@ -62,40 +46,14 @@ async def get_record(call_id: str, db: AsyncSession = Depends(get_db)): row = await store.get_record(db, call_id) if not row: raise HTTPException(status_code=404, detail=f"Call {call_id} not found") - return { - "id": row.id, - "direction": row.direction, - "remote_number": row.remote_number, - "status": row.status, - "mode": row.mode, - "intent": row.intent, - "started_at": row.started_at.isoformat() if row.started_at else None, - "ended_at": row.ended_at.isoformat() if row.ended_at else None, - "duration": row.duration, - "hold_time": row.hold_time, - "device_used": row.device_used, - "summary": row.summary, - "action_items": row.action_items, - "sentiment": row.sentiment, - "call_flow_id": row.call_flow_id, - "classification_timeline": row.classification_timeline, - } + return store.record_detail(row) @router.get("/{call_id}/transcript") async def get_transcript(call_id: str, db: AsyncSession = Depends(get_db)): """Ordered transcript chunks for a call.""" rows = await store.get_transcript_chunks(db, call_id) - return [ - { - "seq": c.seq, - "t_offset_ms": c.t_offset_ms, - "speaker": c.speaker, - "text": c.text, - "confidence": c.confidence, - } - for c in rows - ] + return [store.chunk_to_dict(c) for c in rows] @router.get("/{call_id}/recording") diff --git a/api/calls.py b/api/calls.py index eb46c5d..e6f84c6 100644 --- a/api/calls.py +++ b/api/calls.py @@ -40,12 +40,7 @@ async def make_call( call_flow_id=request.call_flow_id, services=request.services, ) - return CallResponse( - call_id=call.id, - status=call.status.value, - number=request.number, - mode=request.mode.value, - ) + return CallResponse.from_call(call) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: @@ -81,11 +76,8 @@ async def hold_slayer( call_flow_id=request.call_flow_id, device=request.transfer_to, ) - return CallResponse( - call_id=call.id, - status="navigating_ivr", - number=request.number, - mode="hold_slayer", + return CallResponse.from_call( + call, message="Hold Slayer activated. I'll ring you when a human picks up. ☕", ) except ValueError as e: @@ -113,21 +105,7 @@ async def get_call( if not call: raise HTTPException(status_code=404, detail=f"Call {call_id} not found") - return CallStatusResponse( - call_id=call.id, - status=call.status.value, - direction=call.direction, - remote_number=call.remote_number, - mode=call.mode.value, - duration=call.duration, - hold_time=call.hold_time, - audio_type=call.current_classification.value, - intent=call.intent, - transcript_excerpt=call.transcript[-500:] if call.transcript else None, - classification_history=call.classification_history[-50:], - current_step=call.current_step_id, - services=call.services, - ) + return CallStatusResponse.from_call(call) @router.post("/{call_id}/transfer") diff --git a/api/devices.py b/api/devices.py index aaded66..b550fa1 100644 --- a/api/devices.py +++ b/api/devices.py @@ -1,19 +1,19 @@ """ Device Management API — Register and manage phones/softphones. +Row mapping lives in call_persistence; this layer works with the +Device domain model only. """ import uuid -from datetime import datetime from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from api.deps import get_gateway from core.gateway import AIPSTNGateway -from db.database import Device as DeviceDB from db.database import get_db -from models.device import Device, DeviceCreate, DeviceStatus, DeviceUpdate +from models.device import Device, DeviceCreate, DeviceUpdate +from services import call_persistence as store router = APIRouter() @@ -25,45 +25,18 @@ async def register_device( db: AsyncSession = Depends(get_db), ): """Register a new device with the gateway.""" - device_id = f"dev_{uuid.uuid4().hex[:8]}" - - # Save to DB - db_device = DeviceDB( - id=device_id, - name=device.name, - type=device.type.value, - sip_uri=device.sip_uri, - phone_number=device.phone_number, - priority=device.priority, - capabilities=device.capabilities, - is_online="false", - ) - db.add(db_device) - await db.flush() - - # Register with gateway - dev = Device(id=device_id, **device.model_dump()) + dev = Device(id=f"dev_{uuid.uuid4().hex[:8]}", **device.model_dump()) + await store.create_device_row(db, dev) gateway.register_device(dev) - return dev -@router.get("/", response_model=list[DeviceStatus]) +@router.get("/", response_model=list[Device]) async def list_devices( gateway: AIPSTNGateway = Depends(get_gateway), ): """List all registered devices and their status.""" - return [ - DeviceStatus( - id=d.id, - name=d.name, - type=d.type, - is_online=d.is_online, - last_seen=d.last_seen, - can_receive_call=d.can_receive_call, - ) - for d in gateway.devices.values() - ] + return list(gateway.devices.values()) @router.get("/{device_id}", response_model=Device) @@ -90,22 +63,11 @@ async def update_device( if not device: raise HTTPException(status_code=404, detail=f"Device {device_id} not found") - # Update in-memory update_data = update.model_dump(exclude_unset=True) for key, value in update_data.items(): setattr(device, key, value) - # Update in DB - result = await db.execute( - select(DeviceDB).where(DeviceDB.id == device_id) - ) - db_device = result.scalar_one_or_none() - if db_device: - for key, value in update_data.items(): - if key == "type" and value is not None: - value = value.value if hasattr(value, "value") else value - setattr(db_device, key, value) - + await store.update_device_row(db, device_id, update_data) return device @@ -120,12 +82,5 @@ async def unregister_device( raise HTTPException(status_code=404, detail=f"Device {device_id} not found") gateway.unregister_device(device_id) - - result = await db.execute( - select(DeviceDB).where(DeviceDB.id == device_id) - ) - db_device = result.scalar_one_or_none() - if db_device: - await db.delete(db_device) - + await store.delete_device_row(db, device_id) return {"status": "unregistered", "device_id": device_id} diff --git a/core/call_manager.py b/core/call_manager.py index 1640996..dea04ab 100644 --- a/core/call_manager.py +++ b/core/call_manager.py @@ -5,15 +5,19 @@ Central nervous system of the gateway. Tracks all active calls, publishes events, and coordinates between SIP engine and services. """ -import asyncio import logging import uuid -from collections.abc import AsyncIterator from datetime import datetime from typing import Optional -from core.event_bus import EventBus, EventSubscription -from models.call import ActiveCall, AudioClassification, CallMode, CallStatus, ClassificationResult +from core.event_bus import EventBus +from models.call import ( + ActiveCall, + CallMode, + CallStatus, + ClassificationResult, + TranscriptEntry, +) from models.events import EventType, GatewayEvent logger = logging.getLogger(__name__) @@ -26,10 +30,11 @@ class CallManager: The single source of truth for what's happening on the gateway. """ - def __init__(self, event_bus: EventBus, on_call_ended=None): + def __init__(self, event_bus: EventBus, on_call_created=None, on_call_ended=None): self.event_bus = event_bus self._active_calls: dict[str, ActiveCall] = {} self._call_legs: dict[str, str] = {} # SIP leg ID -> call ID mapping + self._on_call_created = on_call_created # async callback(call) self._on_call_ended = on_call_ended # async callback(call, final_status) # ================================================================ @@ -67,6 +72,14 @@ class CallManager: message=f"📞 Calling {remote_number} ({mode.value})", )) + # Durable in_progress row — a crash mid-call must not erase the + # call from history. The hook does its own retrying/logging. + if self._on_call_created is not None: + try: + await self._on_call_created(call) + except Exception as e: + logger.warning(f"on_call_created hook failed for {call_id}: {e}") + return call async def update_status(self, call_id: str, status: CallStatus) -> None: @@ -135,18 +148,26 @@ class CallManager: message=f"🎵 Audio: {result.audio_type.value} ({result.confidence:.0%})", )) - async def add_transcript(self, call_id: str, text: str) -> None: - """Add a transcript chunk to a call.""" + async def add_transcript( + self, call_id: str, text: str, speaker: str = "unknown" + ) -> None: + """Add a transcript entry to a call, stamped with its offset.""" call = self._active_calls.get(call_id) if not call: return - call.transcript_chunks.append(text) + anchor = call.connected_at or call.started_at + entry = TranscriptEntry( + t_offset_ms=int((datetime.now() - anchor).total_seconds() * 1000), + speaker=speaker, + text=text, + ) + call.transcript_chunks.append(entry) await self.event_bus.publish(GatewayEvent( type=EventType.TRANSCRIPT_CHUNK, call_id=call_id, - data={"text": text}, + data={"text": text, "speaker": speaker, "t_offset_ms": entry.t_offset_ms}, message=f"📝 '{text[:80]}...' " if len(text) > 80 else f"📝 '{text}'", )) diff --git a/core/gateway.py b/core/gateway.py index 608f058..5f8f94e 100644 --- a/core/gateway.py +++ b/core/gateway.py @@ -86,11 +86,16 @@ class AIPSTNGateway: self, settings: Settings, sip_engine: Optional[SIPEngine] = None, + on_call_created=None, on_call_ended=None, ): self.settings = settings self.event_bus = EventBus() - self.call_manager = CallManager(self.event_bus, on_call_ended=on_call_ended) + self.call_manager = CallManager( + self.event_bus, + on_call_created=on_call_created, + on_call_ended=on_call_ended, + ) self.media_pipeline = MediaPipeline(sample_rate=16000) self.sip_engine: SIPEngine = sip_engine or MockSIPEngine() diff --git a/db/database.py b/db/database.py index 5ddd6cf..f7bbe2a 100644 --- a/db/database.py +++ b/db/database.py @@ -6,7 +6,7 @@ PostgreSQL via asyncpg + SQLAlchemy async. from collections.abc import AsyncIterator from contextlib import asynccontextmanager -from datetime import datetime +from pathlib import Path from sqlalchemy import ( JSON, @@ -51,7 +51,6 @@ class CallRecord(Base): hold_time = Column(Integer, default=0) # seconds spent on hold device_used = Column(String) recording_path = Column(String, nullable=True) - transcript = Column(Text, nullable=True) summary = Column(Text, nullable=True) action_items = Column(JSON, nullable=True) sentiment = Column(String, nullable=True) @@ -94,7 +93,7 @@ class Device(Base): sip_uri = Column(String, nullable=True) # sip:robert@gateway.helu.ca phone_number = Column(String, nullable=True) # For PSTN devices priority = Column(Integer, default=10) # Routing priority (lower = higher priority) - is_online = Column(String, default="false") + is_online = Column(Boolean, default=False, nullable=False) capabilities = Column(JSON, default=list) # ["voice", "video", "sms"] dnd = Column(Boolean, default=False, nullable=False) last_seen = Column(DateTime, nullable=True) @@ -211,11 +210,31 @@ async def get_db() -> AsyncIterator[AsyncSession]: yield session +# The autogenerated baseline revision — a schema created by the old +# create_all path is identical to it, so such databases are stamped +# here and then migrated forward like any other. +_BASELINE_REVISION = "1173a71329ed" + + +def _upgrade_to_head(connection) -> None: + from alembic import command + from alembic.config import Config + from sqlalchemy import inspect + + cfg = Config(str(Path(__file__).resolve().parent.parent / "alembic.ini")) + cfg.attributes["connection"] = connection + + inspector = inspect(connection) + if not inspector.has_table("alembic_version") and inspector.has_table("call_records"): + command.stamp(cfg, _BASELINE_REVISION) + command.upgrade(cfg, "head") + + async def init_db(): - """Create all tables. For development; use Alembic migrations in production.""" + """Bring the schema to Alembic head (tests create tables directly).""" engine = get_engine() async with engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) + await conn.run_sync(_upgrade_to_head) async def close_db(): diff --git a/db/migrations/env.py b/db/migrations/env.py new file mode 100644 index 0000000..c55fcd0 --- /dev/null +++ b/db/migrations/env.py @@ -0,0 +1,68 @@ +""" +Alembic environment — async engine against Base.metadata. + +Two entry paths: +- CLI (``alembic upgrade head``): builds an async engine from + Settings.database_url and runs migrations on it. +- App startup (db.database.init_db): passes an already-open + connection via ``config.attributes["connection"]`` so migrations + run inside the app's engine instead of opening a second one. +""" + +import asyncio +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.ext.asyncio import create_async_engine + +from config import get_settings +from db.database import Base + +config = context.config + +# Only configure logging on standalone CLI runs — inside the app this +# would clobber uvicorn's logger setup. +if config.config_file_name is not None and config.attributes.get("connection") is None: + fileConfig(config.config_file_name, disable_existing_loggers=False) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + """Emit SQL to stdout without a live connection (--sql mode).""" + context.configure( + url=get_settings().database_url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + engine = create_async_engine(get_settings().database_url, poolclass=pool.NullPool) + async with engine.connect() as connection: + await connection.run_sync(do_run_migrations) + await engine.dispose() + + +def run_migrations_online() -> None: + connection = config.attributes.get("connection") + if connection is not None: + do_run_migrations(connection) + else: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/db/migrations/script.py.mako b/db/migrations/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/db/migrations/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/db/migrations/versions/1173a71329ed_baseline_schema.py b/db/migrations/versions/1173a71329ed_baseline_schema.py new file mode 100644 index 0000000..2330617 --- /dev/null +++ b/db/migrations/versions/1173a71329ed_baseline_schema.py @@ -0,0 +1,129 @@ +"""baseline schema + +Revision ID: 1173a71329ed +Revises: +Create Date: 2026-07-10 07:19:08.321778 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = '1173a71329ed' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('call_flows', + sa.Column('id', sa.String(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('phone_number', sa.String(), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('steps', sa.JSON(), nullable=False), + sa.Column('last_verified', sa.DateTime(), nullable=True), + sa.Column('avg_hold_time', sa.Integer(), nullable=True), + sa.Column('success_rate', sa.Float(), nullable=True), + sa.Column('times_used', sa.Integer(), nullable=True), + sa.Column('last_used', sa.DateTime(), nullable=True), + sa.Column('notes', sa.Text(), nullable=True), + sa.Column('tags', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_call_flows_phone_number'), 'call_flows', ['phone_number'], unique=False) + op.create_table('call_records', + sa.Column('id', sa.String(), nullable=False), + sa.Column('direction', sa.String(), nullable=False), + sa.Column('remote_number', sa.String(), nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('mode', sa.String(), nullable=False), + sa.Column('intent', sa.Text(), nullable=True), + sa.Column('started_at', sa.DateTime(), nullable=True), + sa.Column('ended_at', sa.DateTime(), nullable=True), + sa.Column('duration', sa.Integer(), nullable=True), + sa.Column('hold_time', sa.Integer(), nullable=True), + sa.Column('device_used', sa.String(), nullable=True), + sa.Column('recording_path', sa.String(), nullable=True), + sa.Column('transcript', sa.Text(), nullable=True), + sa.Column('summary', sa.Text(), nullable=True), + sa.Column('action_items', sa.JSON(), nullable=True), + sa.Column('sentiment', sa.String(), nullable=True), + sa.Column('call_flow_id', sa.String(), nullable=True), + sa.Column('classification_timeline', sa.JSON(), nullable=True), + sa.Column('metadata', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_call_records_remote_number'), 'call_records', ['remote_number'], unique=False) + op.create_table('devices', + sa.Column('id', sa.String(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('type', sa.String(), nullable=False), + sa.Column('sip_uri', sa.String(), nullable=True), + sa.Column('phone_number', sa.String(), nullable=True), + sa.Column('priority', sa.Integer(), nullable=True), + sa.Column('is_online', sa.String(), nullable=True), + sa.Column('capabilities', sa.JSON(), nullable=True), + sa.Column('dnd', sa.Boolean(), nullable=False), + sa.Column('last_seen', sa.DateTime(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('recordings', + sa.Column('id', sa.String(), nullable=False), + sa.Column('call_id', sa.String(), nullable=False), + sa.Column('path', sa.String(), nullable=False), + sa.Column('format', sa.String(), nullable=True), + sa.Column('duration_s', sa.Float(), nullable=True), + sa.Column('size_bytes', sa.Integer(), nullable=True), + sa.Column('channels', sa.Integer(), nullable=True), + sa.Column('started_at', sa.DateTime(), nullable=True), + sa.Column('ended_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_recordings_call_id'), 'recordings', ['call_id'], unique=False) + op.create_table('routing_rules', + sa.Column('id', sa.String(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('priority', sa.Integer(), nullable=False), + sa.Column('enabled', sa.Boolean(), nullable=False), + sa.Column('match', sa.JSON(), nullable=False), + sa.Column('action', sa.JSON(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('transcript_chunks', + sa.Column('id', sa.String(), nullable=False), + sa.Column('call_id', sa.String(), nullable=False), + sa.Column('seq', sa.Integer(), nullable=False), + sa.Column('t_offset_ms', sa.Integer(), nullable=True), + sa.Column('speaker', sa.String(), nullable=True), + sa.Column('text', sa.Text(), nullable=False), + sa.Column('confidence', sa.Float(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_transcript_chunks_call_id'), 'transcript_chunks', ['call_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_transcript_chunks_call_id'), table_name='transcript_chunks') + op.drop_table('transcript_chunks') + op.drop_table('routing_rules') + op.drop_index(op.f('ix_recordings_call_id'), table_name='recordings') + op.drop_table('recordings') + op.drop_table('devices') + op.drop_index(op.f('ix_call_records_remote_number'), table_name='call_records') + op.drop_table('call_records') + op.drop_index(op.f('ix_call_flows_phone_number'), table_name='call_flows') + op.drop_table('call_flows') + # ### end Alembic commands ### diff --git a/db/migrations/versions/5187577efc23_drop_dead_transcript_column_boolean_is_.py b/db/migrations/versions/5187577efc23_drop_dead_transcript_column_boolean_is_.py new file mode 100644 index 0000000..9eaf9f5 --- /dev/null +++ b/db/migrations/versions/5187577efc23_drop_dead_transcript_column_boolean_is_.py @@ -0,0 +1,45 @@ +"""drop dead transcript column, boolean is_online + +Revision ID: 5187577efc23 +Revises: 1173a71329ed +Create Date: 2026-07-10 07:19:40.741327 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = '5187577efc23' +down_revision: Union[str, None] = '1173a71329ed' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Transcript text lives solely in transcript_chunks rows now. + op.drop_column('call_records', 'transcript') + + # String "true"/"false" (or NULL) -> real boolean; NULLs become false. + # batch mode so the table-recreate path works on SQLite too. + with op.batch_alter_table('devices') as batch_op: + batch_op.alter_column( + 'is_online', + existing_type=sa.VARCHAR(), + type_=sa.Boolean(), + nullable=False, + postgresql_using="coalesce(lower(is_online) in ('true', 't', '1'), false)", + ) + + +def downgrade() -> None: + with op.batch_alter_table('devices') as batch_op: + batch_op.alter_column( + 'is_online', + existing_type=sa.Boolean(), + type_=sa.VARCHAR(), + nullable=True, + postgresql_using="case when is_online then 'true' else 'false' end", + ) + op.add_column('call_records', sa.Column('transcript', sa.TEXT(), nullable=True)) diff --git a/main.py b/main.py index dd11fc7..d9a95fd 100644 --- a/main.py +++ b/main.py @@ -26,7 +26,7 @@ from db.database import close_db, init_db from mcp_server.server import create_mcp_server from models.call import CallMode from services.audio_classifier import AudioClassifier -from services.call_persistence import persist_call_on_end +from services.call_persistence import persist_call_on_create, persist_call_on_end from services.hold_slayer import HoldSlayerService from services.notification import NotificationService from services.receptionist import ReceptionistService @@ -139,7 +139,11 @@ async def lifespan(app: FastAPI): # === Composition root === # Build the gateway and every service here, wiring them by # constructor/registration — nothing constructs its own deps. - gateway = AIPSTNGateway(settings=settings, on_call_ended=persist_call_on_end) + gateway = AIPSTNGateway( + settings=settings, + on_call_created=persist_call_on_create, + on_call_ended=persist_call_on_end, + ) classifier = AudioClassifier(settings.classifier) transcription = TranscriptionService(settings.speaches) diff --git a/models/call.py b/models/call.py index 785e795..d9863c4 100644 --- a/models/call.py +++ b/models/call.py @@ -55,6 +55,14 @@ class ClassificationResult(BaseModel): details: Optional[dict] = None # Extra analysis data +class TranscriptEntry(BaseModel): + """One transcribed utterance, offset from call start for seek.""" + + t_offset_ms: int + speaker: str = "unknown" # caller / agent / receptionist / unknown + text: str + + class ActiveCall(BaseModel): """In-memory state for an active call.""" @@ -71,7 +79,7 @@ class ActiveCall(BaseModel): hold_started_at: Optional[datetime] = None current_classification: AudioClassification = AudioClassification.UNKNOWN classification_history: list[ClassificationResult] = Field(default_factory=list) - transcript_chunks: list[str] = Field(default_factory=list) + transcript_chunks: list[TranscriptEntry] = Field(default_factory=list) current_step_id: Optional[str] = None # Current position in call flow services: list[str] = Field(default_factory=list) # Active services on this call @@ -92,7 +100,7 @@ class ActiveCall(BaseModel): @property def transcript(self) -> str: """Full transcript so far.""" - return "\n".join(self.transcript_chunks) + return "\n".join(e.text for e in self.transcript_chunks) def summary(self) -> dict: """Compact summary for list views.""" @@ -145,6 +153,16 @@ class CallResponse(BaseModel): mode: str message: Optional[str] = None + @classmethod + def from_call(cls, call: "ActiveCall", message: Optional[str] = None) -> "CallResponse": + return cls( + call_id=call.id, + status=call.status.value, + number=call.remote_number, + mode=call.mode.value, + message=message, + ) + class CallStatusResponse(BaseModel): """Full status of an active or completed call.""" @@ -163,6 +181,25 @@ class CallStatusResponse(BaseModel): current_step: Optional[str] = None services: list[str] = Field(default_factory=list) + @classmethod + def from_call(cls, call: "ActiveCall") -> "CallStatusResponse": + """The one ActiveCall → status-response mapping.""" + return cls( + call_id=call.id, + status=call.status.value, + direction=call.direction, + remote_number=call.remote_number, + mode=call.mode.value, + duration=call.duration, + hold_time=call.hold_time, + audio_type=call.current_classification.value, + intent=call.intent, + transcript_excerpt=call.transcript[-500:] if call.transcript else None, + classification_history=call.classification_history[-20:], + current_step=call.current_step_id, + services=call.services, + ) + class TransferRequest(BaseModel): """Request to transfer a call to a device.""" diff --git a/models/device.py b/models/device.py index 641b538..481b766 100644 --- a/models/device.py +++ b/models/device.py @@ -8,7 +8,7 @@ from datetime import datetime from enum import Enum from typing import Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, computed_field class DeviceType(str, Enum): @@ -43,6 +43,7 @@ class Device(DeviceBase): created_at: Optional[datetime] = None updated_at: Optional[datetime] = None + @computed_field # serialized so API consumers see routability directly @property def can_receive_call(self) -> bool: """Can this device receive a call right now?""" @@ -71,14 +72,3 @@ class DeviceUpdate(BaseModel): phone_number: Optional[str] = None priority: Optional[int] = None capabilities: Optional[list[str]] = None - - -class DeviceStatus(BaseModel): - """Lightweight device status for list views.""" - - id: str - name: str - type: DeviceType - is_online: bool - last_seen: Optional[datetime] = None - can_receive_call: bool diff --git a/pyproject.toml b/pyproject.toml index 70b0676..ed8ab45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,3 +70,7 @@ line-length = 100 [tool.ruff.lint] select = ["E", "F", "I", "N", "W", "UP"] + +[tool.ruff.lint.per-file-ignores] +# Alembic-generated migrations keep the standard template style. +"db/migrations/versions/*" = ["E501", "UP007", "UP035", "W291"] diff --git a/services/call_persistence.py b/services/call_persistence.py index d12c8c5..6390eda 100644 --- a/services/call_persistence.py +++ b/services/call_persistence.py @@ -22,8 +22,10 @@ from db.database import ( TranscriptChunk, session_scope, ) +from db.database import Device as DeviceRow from models.call import ActiveCall, CallStatus from models.call_flow import CallFlow, CallFlowStep +from models.device import Device logger = logging.getLogger(__name__) @@ -45,6 +47,45 @@ def flow_to_model(row: StoredCallFlow) -> CallFlow: ) +def record_summary(row: CallRecord) -> dict: + """The one CallRecord-row → list-item mapping.""" + return { + "id": row.id, + "direction": row.direction, + "remote_number": row.remote_number, + "status": row.status, + "mode": row.mode, + "intent": row.intent, + "started_at": row.started_at.isoformat() if row.started_at else None, + "ended_at": row.ended_at.isoformat() if row.ended_at else None, + "duration": row.duration, + "hold_time": row.hold_time, + "device_used": row.device_used, + "summary": row.summary, + } + + +def record_detail(row: CallRecord) -> dict: + """Full CallRecord-row mapping, superset of record_summary.""" + return record_summary(row) | { + "action_items": row.action_items, + "sentiment": row.sentiment, + "call_flow_id": row.call_flow_id, + "classification_timeline": row.classification_timeline, + } + + +def chunk_to_dict(row: TranscriptChunk) -> dict: + """The one TranscriptChunk-row → dict mapping.""" + return { + "seq": row.seq, + "t_offset_ms": row.t_offset_ms, + "speaker": row.speaker, + "text": row.text, + "confidence": row.confidence, + } + + # ================================================================ # Call flows # ================================================================ @@ -95,6 +136,49 @@ async def create_flow( return row +# ================================================================ +# Devices +# ================================================================ + +async def create_device_row(session: AsyncSession, device: Device) -> None: + """The one Device-model → row mapping.""" + session.add(DeviceRow( + id=device.id, + name=device.name, + type=device.type.value, + sip_uri=device.sip_uri, + phone_number=device.phone_number, + priority=device.priority, + capabilities=device.capabilities, + is_online=device.is_online, + )) + await session.flush() + + +async def update_device_row( + session: AsyncSession, device_id: str, values: dict +) -> None: + result = await session.execute( + select(DeviceRow).where(DeviceRow.id == device_id) + ) + row = result.scalar_one_or_none() + if row is None: + return + for key, value in values.items(): + if key == "type" and value is not None: + value = value.value if hasattr(value, "value") else value + setattr(row, key, value) + + +async def delete_device_row(session: AsyncSession, device_id: str) -> None: + result = await session.execute( + select(DeviceRow).where(DeviceRow.id == device_id) + ) + row = result.scalar_one_or_none() + if row is not None: + await session.delete(row) + + # ================================================================ # Call history / records # ================================================================ @@ -156,71 +240,96 @@ async def latest_recording( return result.scalars().first() +async def persist_call_on_create(call: ActiveCall) -> None: + """Insert an in_progress CallRecord the moment a call starts. + + Wired into CallManager as its on_call_created hook — a crash + mid-call leaves this row behind instead of erasing the call from + history. persist_call_on_end updates it to the terminal state. + """ + await _with_retry(_insert_in_progress_record, call) + + async def persist_call_on_end(call: ActiveCall, final_status: CallStatus) -> None: - """Insert a CallRecord and any transcript chunks for `call`. + """Finalize the CallRecord and write transcript chunks for `call`. Wired into CallManager as its on_call_ended hook by the - composition root in main.py. Retries briefly — losing the row - means the call never happened as far as history is concerned, so - the final failure logs at ERROR with the payload identifiers. + composition root in main.py. """ + await _with_retry(_finalize_call_record, call, final_status) + + +async def _with_retry(write, call: ActiveCall, *args) -> None: + """Losing the row means the call never happened as far as history + is concerned, so the final failure logs at ERROR with identifiers.""" for attempt in range(3): try: - await _write_call_record(call, final_status) + await write(call, *args) return except Exception as e: if attempt == 2: logger.error( - f"Call record lost: id={call.id} number={call.remote_number} " - f"status={final_status.value}: {e}" + f"Call record lost ({write.__name__}): id={call.id} " + f"number={call.remote_number}: {e}" ) return await asyncio.sleep(2**attempt) -async def _write_call_record(call: ActiveCall, final_status: CallStatus) -> None: +async def _insert_in_progress_record(call: ActiveCall) -> None: async with session_scope() as session: - record = CallRecord( + session.add(CallRecord( id=call.id, direction=call.direction, remote_number=call.remote_number, - status=final_status.value, + status="in_progress", mode=call.mode.value, intent=call.intent, started_at=call.started_at, - ended_at=datetime.now(), - duration=int(call.duration), - hold_time=int(call.hold_time), device_used=call.device, call_flow_id=call.call_flow_id, - classification_timeline=[ - { - "timestamp": c.timestamp, - "audio_type": c.audio_type.value, - "confidence": c.confidence, - } - for c in call.classification_history - ], metadata_={"services": list(call.services)}, - ) - session.add(record) + )) - # Each transcript chunk gets its own row with a sequence number - # so the dashboard can render them in order with click-to-seek. - for seq, text in enumerate(call.transcript_chunks): - speaker = "unknown" - payload = text - if ":" in text: - head, rest = text.split(":", 1) - head = head.strip().lower() - if head in {"caller", "agent", "receptionist", "caller_message"}: - speaker = head if head != "caller_message" else "caller" - payload = rest.strip() + +async def _finalize_call_record(call: ActiveCall, final_status: CallStatus) -> None: + async with session_scope() as session: + record = await get_record(session, call.id) + if record is None: + # The create-time insert failed (or predates the hook); + # write the whole row now instead. + record = CallRecord(id=call.id) + session.add(record) + + record.direction = call.direction + record.remote_number = call.remote_number + record.status = final_status.value + record.mode = call.mode.value + record.intent = call.intent + record.started_at = call.started_at + record.ended_at = datetime.now() + record.duration = int(call.duration) + record.hold_time = int(call.hold_time) + record.device_used = call.device + record.call_flow_id = call.call_flow_id + record.classification_timeline = [ + { + "timestamp": c.timestamp, + "audio_type": c.audio_type.value, + "confidence": c.confidence, + } + for c in call.classification_history + ] + record.metadata_ = {"services": list(call.services)} + + # Each transcript entry gets its own row with a sequence number + # and real offset so the dashboard can render click-to-seek. + for seq, entry in enumerate(call.transcript_chunks): session.add(TranscriptChunk( id=f"tc_{uuid.uuid4().hex[:10]}", call_id=call.id, seq=seq, - t_offset_ms=0, - speaker=speaker, - text=payload, + t_offset_ms=entry.t_offset_ms, + speaker=entry.speaker, + text=entry.text, )) diff --git a/services/receptionist.py b/services/receptionist.py index 8999b30..4cf21ca 100644 --- a/services/receptionist.py +++ b/services/receptionist.py @@ -134,13 +134,9 @@ class ReceptionistService: transcript = await self._listen(call, sip_leg_id) if transcript: - call.transcript_chunks.append(f"caller: {transcript}") - await self.gateway.event_bus.publish(GatewayEvent( - type=EventType.TRANSCRIPT_CHUNK, - call_id=call.id, - data={"text": transcript, "speaker": "caller"}, - message=f"📝 caller: {transcript[:80]}", - )) + await self.gateway.call_manager.add_transcript( + call.id, transcript, speaker="caller" + ) classification = await self._classify(call, transcript, routing_decision) call.intent = classification.get("intent") @@ -390,7 +386,9 @@ class ReceptionistService: await self._service_error(call.id, "transcription", e) if message_text: - call.transcript_chunks.append(f"caller_message: {message_text}") + await self.gateway.call_manager.add_transcript( + call.id, message_text, speaker="caller" + ) await self.gateway.event_bus.publish(GatewayEvent( type=EventType.RECEPTIONIST_MESSAGE_SAVED, diff --git a/tests/test_data_layer.py b/tests/test_data_layer.py new file mode 100644 index 0000000..513b19c --- /dev/null +++ b/tests/test_data_layer.py @@ -0,0 +1,187 @@ +""" +Data-layer tests. + +Alembic migrations produce the schema the ORM models declare (and +adopt a pre-Alembic database); a call gets a durable in_progress row +the moment it starts; transcript entries carry real offsets; the +consolidated response models map from the domain in one place. +""" + +import asyncio + +import pytest +from sqlalchemy import Boolean, inspect, text +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from sqlalchemy.pool import StaticPool + +import db.database as dbmod +from core.call_manager import CallManager +from core.event_bus import EventBus +from db.database import Base +from models.call import ActiveCall, CallStatus, CallStatusResponse +from models.device import Device, DeviceType +from services import call_persistence as store + +# ================================================================ +# Alembic migrations +# ================================================================ + +def _run_alembic(connection, revision: str) -> None: + from alembic import command + from alembic.config import Config + + cfg = Config("alembic.ini") + cfg.attributes["connection"] = connection + command.upgrade(cfg, revision) + + +def _schema_info(sync_conn) -> dict: + inspector = inspect(sync_conn) + return { + "tables": set(inspector.get_table_names()) - {"alembic_version"}, + "call_record_cols": {c["name"] for c in inspector.get_columns("call_records")}, + "is_online_type": next( + c["type"] for c in inspector.get_columns("devices") + if c["name"] == "is_online" + ), + } + + +class TestMigrations: + async def test_upgrade_head_matches_models(self, tmp_path): + engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/mig.db") + async with engine.begin() as conn: + await conn.run_sync(dbmod._upgrade_to_head) + async with engine.connect() as conn: + info = await conn.run_sync(_schema_info) + await engine.dispose() + + assert info["tables"] == set(Base.metadata.tables) + assert "transcript" not in info["call_record_cols"] + assert isinstance(info["is_online_type"], Boolean) + + async def test_adopts_pre_alembic_schema(self, tmp_path): + """A create_all-era database (baseline schema, no alembic_version) + is stamped and migrated forward instead of failing.""" + engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/legacy.db") + async with engine.begin() as conn: + await conn.run_sync( + lambda c: _run_alembic(c, dbmod._BASELINE_REVISION) + ) + await conn.execute(text("DROP TABLE alembic_version")) + + async with engine.begin() as conn: + await conn.run_sync(dbmod._upgrade_to_head) + async with engine.connect() as conn: + info = await conn.run_sync(_schema_info) + await engine.dispose() + + assert "transcript" not in info["call_record_cols"] + assert isinstance(info["is_online_type"], Boolean) + + +# ================================================================ +# Durable call rows +# ================================================================ + +@pytest.fixture +async def mem_db(monkeypatch): + engine = create_async_engine( + "sqlite+aiosqlite:///:memory:", + poolclass=StaticPool, + connect_args={"check_same_thread": False}, + ) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + factory = async_sessionmaker(engine, expire_on_commit=False) + monkeypatch.setattr(dbmod, "_engine", engine) + monkeypatch.setattr(dbmod, "_session_factory", factory) + yield factory + await engine.dispose() + + +class TestDurableCallRows: + async def test_in_progress_row_from_the_start(self, mem_db): + cm = CallManager( + EventBus(), + on_call_created=store.persist_call_on_create, + on_call_ended=store.persist_call_on_end, + ) + call = await cm.create_call("+15551230000", intent="dispute a charge") + + async with mem_db() as session: + row = await store.get_record(session, call.id) + assert row is not None + assert row.status == "in_progress" + assert row.ended_at is None + assert row.intent == "dispute a charge" + + await cm.add_transcript(call.id, "hello, billing please", speaker="caller") + await cm.end_call(call.id, CallStatus.COMPLETED) + + async with mem_db() as session: + row = await store.get_record(session, call.id) + chunks = await store.get_transcript_chunks(session, call.id) + assert row.status == "completed" + assert row.ended_at is not None + assert [(c.seq, c.speaker, c.text) for c in chunks] == [ + (0, "caller", "hello, billing please") + ] + + async def test_end_without_create_still_writes_row(self, mem_db): + """If the create-time insert never happened, finalize inserts.""" + cm = CallManager(EventBus(), on_call_ended=store.persist_call_on_end) + call = await cm.create_call("+15551230001") + await cm.end_call(call.id, CallStatus.FAILED) + + async with mem_db() as session: + row = await store.get_record(session, call.id) + assert row is not None + assert row.status == "failed" + + +# ================================================================ +# Transcript offsets +# ================================================================ + +class TestTranscriptOffsets: + async def test_entries_carry_offset_and_speaker(self): + cm = CallManager(EventBus()) + call = await cm.create_call("+15551230002") + + await cm.add_transcript(call.id, "one") + await asyncio.sleep(0.02) + await cm.add_transcript(call.id, "two", speaker="agent") + + first, second = call.transcript_chunks + assert first.t_offset_ms >= 0 + assert second.t_offset_ms > first.t_offset_ms + assert first.speaker == "unknown" + assert second.speaker == "agent" + assert call.transcript == "one\ntwo" + + +# ================================================================ +# Consolidated response models +# ================================================================ + +class TestResponseModels: + def test_status_response_from_call(self): + call = ActiveCall(id="call_x", remote_number="+15550000000", intent="pay bill") + resp = CallStatusResponse.from_call(call) + assert resp.call_id == "call_x" + assert resp.status == "initiating" + assert resp.remote_number == "+15550000000" + assert resp.intent == "pay bill" + + def test_device_serializes_routability(self): + device = Device( + id="dev_1", + name="Desk Phone", + type=DeviceType.SIP_PHONE, + sip_uri="sip:desk@gw", + is_online=True, + ) + assert device.model_dump()["can_receive_call"] is True + device.dnd = True + assert device.model_dump()["can_receive_call"] is False From ff7ea8623afefe50465b670afa1c3e4166253f80 Mon Sep 17 00:00:00 2001 From: Robert Helewka Date: Fri, 10 Jul 2026 13:45:35 -0400 Subject: [PATCH 5/5] Stage 6: learn_call_flow rebuilt on the learner, docs truth sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The call-flow learner finally gets fed: exploration mode records its IVR discoveries on the call (ActiveCall.exploration_steps) instead of throwing them away, persistence stores them in the call record's metadata, and the rebuilt learn_call_flow MCP tool turns a completed exploration call into a stored flow via CallFlowLearner — correct constructor (llm_client from get_llm, heuristic labels when the LLM is unavailable), build for a new number, merge/refine when a flow already exists. save_learned_flow/update_flow_from_model keep the CallFlow↔row mapping in call_persistence. Test gaps closed: tests/test_learner.py (discoveries→linked steps, exploration persistence, learn-then-refine through the in-memory MCP client, no-data and unknown-call answers) and tests/test_websocket.py (4401 without token, trunk-status-then-replay on connect, per-call stream filtering). Docs aligned to code: README (15 tools incl. learn_call_flow, HTTP not SSE, Python 3.12+, PostgreSQL+Alembic — no SQLite fallback, media pipeline marked stub-mode until pjsua2 installed, Alembic and honest /health checked off); docs/mcp-server.md rewritten against the actual tool surface (hangup not end_call, real params, 3 real resources, /mcp/ streamable HTTP + bearer auth); architecture/development/ configuration drift fixed. pyproject: pruned never-imported deps (websockets, librosa, soundfile, python-multipart). Co-Authored-By: Claude Fable 5 --- README.md | 25 ++--- docs/architecture.md | 6 +- docs/configuration.md | 2 +- docs/development.md | 2 +- docs/mcp-server.md | 189 +++++++++++++++++++++-------------- mcp_server/server.py | 64 ++++++++++++ models/call.py | 3 + pyproject.toml | 18 ++-- services/call_persistence.py | 35 ++++++- services/hold_slayer.py | 2 +- tests/test_learner.py | 137 +++++++++++++++++++++++++ tests/test_mcp.py | 1 + tests/test_websocket.py | 67 +++++++++++++ 13 files changed, 444 insertions(+), 107 deletions(-) create mode 100644 tests/test_learner.py create mode 100644 tests/test_websocket.py diff --git a/README.md b/README.md index 86d49a2..8d7895f 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ You give it a phone number and an intent ("dispute a charge on my December state │ │ │ ┌──────────┐ ┌──────────┐ ┌───────────┐ ┌──────────────┐ │ │ │ REST API │ │WebSocket │ │MCP Server │ │ Dashboard │ │ -│ │ /api/* │ │ /ws/* │ │ (SSE) │ │ /dashboard │ │ +│ │ /api/* │ │ /ws/* │ │ (HTTP) │ │ /dashboard │ │ │ └────┬─────┘ └────┬─────┘ └─────┬─────┘ └──────────────┘ │ │ │ │ │ │ │ ┌────┴──────────────┴──────────────┴────┐ │ @@ -53,7 +53,7 @@ You give it a phone number and an intent ("dispute a charge on my December state ### Core Engine - **Sippy B2BUA Engine** (`core/sippy_engine.py`) — SIP call control, DTMF, bridging, conference, trunk registration -- **PJSUA2 Media Pipeline** (`core/media_pipeline.py`) — Audio routing, recording ports, conference bridge, WAV playback +- **PJSUA2 Media Pipeline** (`core/media_pipeline.py`) — Audio routing, recording ports, conference bridge, WAV playback (stub mode until the `pjsua2` bindings are installed — see note below) - **Call Manager** (`core/call_manager.py`) — Active call state tracking, lifecycle management - **Event Bus** (`core/event_bus.py`) — Async pub/sub with per-subscriber queues, type filtering, history @@ -78,7 +78,7 @@ You give it a phone number and an intent ("dispute a charge on my December state ### API Surface - **REST API** — Call management, call history, transcripts, recordings, routing rules, device DND, call flow CRUD - **WebSocket** — Real-time call events, transcripts, classification updates, receptionist state transitions -- **MCP Server** — 14 tools + 3 resources for AI assistant integration (make calls, send DTMF, get transcripts, manage flows), served over streamable HTTP at `/mcp/` +- **MCP Server** — 15 tools + 3 resources for AI assistant integration (make calls, send DTMF, get transcripts, manage flows), served over streamable HTTP at `/mcp/` - **Dashboard** — SvelteKit UI served at `/dashboard` with live monitor, call history with transcript playback, and a routing-rules editor ### Data Models @@ -130,7 +130,7 @@ hold-slayer/ │ ├── calls/[call_id]/ # Detail page + transcript playback │ └── routing/ # Rules editor + DND toggles ├── mcp_server/ -│ └── server.py # MCP tools + resources (10 tools) +│ └── server.py # MCP tools + resources (15 tools) ├── models/ │ ├── call.py # Call state models │ ├── call_flow.py # IVR tree models @@ -139,7 +139,7 @@ hold-slayer/ │ ├── device.py # Device models │ └── contact.py # Contact models ├── db/ -│ └── database.py # SQLAlchemy async (PostgreSQL/SQLite) +│ └── database.py # SQLAlchemy async (PostgreSQL + Alembic) └── tests/ ├── test_audio_classifier.py # 18 tests — waveform analysis ├── test_call_flows.py # 10 tests — call flow models @@ -283,7 +283,7 @@ claude mcp add hold-slayer --transport http http://localhost:8000/mcp/ \ --header "Authorization: Bearer $API_TOKEN" ``` -It exposes 14 tools and 3 resources (`gateway://status`, +It exposes 15 tools and 3 resources (`gateway://status`, `gateway://call-flows`, `gateway://active-calls`): | Tool | Description | @@ -302,6 +302,7 @@ It exposes 14 tools and 3 resources (`gateway://status`, | `create_call_flow` | Store a new IVR call flow | | `get_call_summary` | Stored summary and action items for a call | | `search_call_history` | Search past calls by number or intent | +| `learn_call_flow` | Build/refine a reusable IVR flow from an exploration call | ## How It Works @@ -352,7 +353,7 @@ All configuration is via environment variables (see `.env.example`): ## Tech Stack -- **Python 3.13** + **asyncio** — Single-process async architecture +- **Python 3.12+** + **asyncio** — Single-process async architecture - **FastAPI** — REST API + WebSocket server - **SvelteKit** — Dashboard UI (built static, served by FastAPI at `/dashboard`) - **Sippy B2BUA** — SIP call control and DTMF @@ -360,7 +361,7 @@ All configuration is via environment variables (see `.env.example`): - **Speaches** (Whisper) — Speech-to-text - **Rhema** (Kokoro) — Text-to-speech (OpenAI-compatible `/v1/audio/speech`) - **Ollama / vLLM / OpenAI** — LLM for IVR menu analysis and receptionist intent capture -- **SQLAlchemy** — Async database (PostgreSQL or SQLite) +- **SQLAlchemy + Alembic** — Async database (PostgreSQL; schema managed by migrations) - **MCP (Model Context Protocol)** — AI assistant integration ## Documentation @@ -384,7 +385,7 @@ Full documentation is in [`/docs`](docs/README.md): - [x] Extract EventBus to dedicated module with typed filtering - [x] Implement Sippy B2BUA SIP engine (signaling, DTMF, bridging) -- [x] Implement PJSUA2 media pipeline (conference bridge, audio tapping, recording) +- [x] PJSUA2 media pipeline contract (conference bridge, audio tapping, recording) — runs in stub mode until `pjsua2` bindings are installed - [x] Call manager with active call state tracking - [x] Gateway orchestrator wiring all components @@ -400,18 +401,18 @@ Full documentation is in [`/docs`](docs/README.md): - [x] REST API — calls, call flows, devices, DTMF - [x] WebSocket real-time event streaming -- [x] MCP server with 14 tools + 3 resources, mounted at `/mcp/` (streamable HTTP) +- [x] MCP server with 15 tools + 3 resources, mounted at `/mcp/` (streamable HTTP) - [x] Notification service (WebSocket + SMS) - [x] Service wiring in main.py lifespan ### Phase 4: Production Hardening 🚧 -- [ ] Alembic database migrations +- [x] Alembic database migrations (baseline + upgrade-on-boot) - [x] API authentication — static bearer token across REST/WS/MCP - [x] Emergency-number guard + concurrent-call cap on outbound calls - [ ] Rate limiting on API endpoints - [ ] Structured JSON logging -- [ ] Health check endpoints for all dependencies +- [x] Honest /health — engine mode, DB ping, trunk registration, STT/TTS availability - [ ] Graceful degradation (classifier works without STT, etc.) - [ ] Docker Compose (Hold Slayer + PostgreSQL) diff --git a/docs/architecture.md b/docs/architecture.md index 36d5af9..ce6a144 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,7 +10,7 @@ Hold Slayer is a single-process async Python application built on FastAPI. It ac │ │ │ ┌──────────┐ ┌──────────┐ ┌───────────┐ ┌──────────────┐ │ │ │ REST API │ │WebSocket │ │MCP Server │ │ Dashboard │ │ -│ │ /api/* │ │ /ws/* │ │ (SSE) │ │ /dashboard │ │ +│ │ /api/* │ │ /ws/* │ │ (HTTP) │ │ /dashboard │ │ │ └────┬─────┘ └────┬─────┘ └─────┬─────┘ └──────────────┘ │ │ │ │ │ │ │ ┌────┴──────────────┴──────────────┴────┐ │ @@ -46,7 +46,7 @@ Hold Slayer is a single-process async Python application built on FastAPI. It ac |-----------|------|----------|---------| | REST API | `api/calls.py`, `api/call_flows.py`, `api/devices.py` | HTTP | Call management, CRUD, configuration | | WebSocket | `api/websocket.py` | WS | Real-time event streaming to clients | -| MCP Server | `mcp_server/server.py` | SSE | AI assistant tool integration | +| MCP Server | `mcp_server/server.py` | Streamable HTTP at `/mcp/` | AI assistant tool integration | ### Orchestration Layer @@ -75,7 +75,7 @@ Hold Slayer is a single-process async Python application built on FastAPI. It ac | Recording | `services/recording.py` | WAV file management and storage | | Analytics | `services/call_analytics.py` | Call metrics, hold time stats, trends | | Notifications | `services/notification.py` | WebSocket + SMS alerts | -| Database | `db/database.py` | SQLAlchemy async (PostgreSQL or SQLite) | +| Database | `db/database.py` | SQLAlchemy async (PostgreSQL, Alembic migrations) | ## Data Flow — Hold Slayer Call diff --git a/docs/configuration.md b/docs/configuration.md index aa89621..614b860 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -126,7 +126,7 @@ uvicorn main:app --host 0.0.0.0 --port 8000 --reload ### Production ```bash -# Use PostgreSQL instead of SQLite +# PostgreSQL is required (no SQLite fallback) DATABASE_URL=postgresql+asyncpg://user:pass@localhost/hold_slayer # Use vLLM for faster inference diff --git a/docs/development.md b/docs/development.md index 4aa8d9b..a45c5e5 100644 --- a/docs/development.md +++ b/docs/development.md @@ -4,7 +4,7 @@ ### Prerequisites -- Python 3.13+ +- Python 3.12+ - Ollama (or any OpenAI-compatible LLM) — for IVR menu analysis - Speaches or Whisper API — for speech-to-text (optional for dev) - A SIP trunk account — for making real calls (optional for dev) diff --git a/docs/mcp-server.md b/docs/mcp-server.md index 93e8228..3cbe417 100644 --- a/docs/mcp-server.md +++ b/docs/mcp-server.md @@ -1,10 +1,20 @@ # MCP Server -The MCP (Model Context Protocol) server lets any MCP-compatible AI assistant control the Hold Slayer gateway. Built with [FastMCP](https://github.com/jlowin/fastmcp), it exposes tools and resources over SSE. +The MCP (Model Context Protocol) server lets any MCP-compatible AI assistant +control the Hold Slayer gateway. Built with [FastMCP](https://github.com/jlowin/fastmcp), +it is mounted on the FastAPI app at **`/mcp/`** (trailing slash) over +**streamable HTTP** and authenticates with the same static bearer token as the +REST API and WebSocket. ## Overview -An AI assistant connects via SSE to the MCP server and gains access to tools for placing calls, checking status, sending DTMF, getting transcripts, and managing call flows. The assistant can orchestrate an entire call through natural language. +An AI assistant connects to the MCP endpoint and gains access to 15 tools and +3 resources for placing calls, checking status, sending DTMF, getting +transcripts, and managing call flows. The assistant can orchestrate an entire +call through natural language. + +`make_call` places a **real PSTN call** that may incur charges; emergency +numbers are always refused, and the concurrent-call cap applies. ## Tools @@ -15,13 +25,32 @@ Place an outbound call through the SIP trunk. | Param | Type | Required | Description | |-------|------|----------|-------------| | `number` | string | Yes | Phone number to call (E.164 format) | -| `mode` | string | No | Call mode: `direct`, `hold_slayer`, `ai_assisted` (default: `hold_slayer`) | +| `mode` | string | No | `direct`, `hold_slayer`, or `ai_assisted` (default: `direct`) | | `intent` | string | No | What you want to accomplish on the call | | `call_flow_id` | string | No | ID of a stored call flow to follow | +| `device` | string | No | Device to transfer to when a human is detected | -Returns: Call ID and initial status. +Returns: call ID and initial status. -### end_call +### get_call_status + +Check the current state of a call — status, duration, hold time, current +audio classification, recent transcript. + +| Param | Type | Required | Description | +|-------|------|----------|-------------| +| `call_id` | string | Yes | The call to check | + +### transfer_call + +Transfer an active call to a registered device. + +| Param | Type | Required | Description | +|-------|------|----------|-------------| +| `call_id` | string | Yes | The call to transfer | +| `device` | string | Yes | Device ID or type to ring | + +### hangup Hang up an active call. @@ -29,102 +58,117 @@ Hang up an active call. |-------|------|----------|-------------| | `call_id` | string | Yes | The call to hang up | -### send_dtmf - -Send touch-tone digits to an active call (for manual IVR navigation). - -| Param | Type | Required | Description | -|-------|------|----------|-------------| -| `call_id` | string | Yes | The call to send digits to | -| `digits` | string | Yes | DTMF digits to send (e.g., "1", "3#", "1234") | - -### get_call_status - -Check the current state of a call. - -| Param | Type | Required | Description | -|-------|------|----------|-------------| -| `call_id` | string | Yes | The call to check | - -Returns: Status, duration, hold time, audio classification, transcript excerpt. - -### get_call_transcript - -Get the live transcript of a call. - -| Param | Type | Required | Description | -|-------|------|----------|-------------| -| `call_id` | string | Yes | The call to get transcript for | - -Returns: Array of transcript chunks with timestamps and speaker labels. - -### get_call_recording - -Get recording metadata and file path for a call. - -| Param | Type | Required | Description | -|-------|------|----------|-------------| -| `call_id` | string | Yes | The call to get recording for | - -Returns: Recording path, duration, file size. - ### list_active_calls List all calls currently in progress. No parameters. -Returns: Array of active calls with status, number, duration. +### send_dtmf + +Send touch-tone digits on an active call (manual IVR navigation). + +| Param | Type | Required | Description | +|-------|------|----------|-------------| +| `call_id` | string | Yes | The call to send digits on | +| `digits` | string | Yes | DTMF digits (e.g., `"1"`, `"123#"`) | + +### get_call_transcript + +Get the full transcript of an active call. + +| Param | Type | Required | Description | +|-------|------|----------|-------------| +| `call_id` | string | Yes | The call to get the transcript for | + +### get_call_recording + +Get recording metadata (path, duration) for a persisted call. + +| Param | Type | Required | Description | +|-------|------|----------|-------------| +| `call_id` | string | Yes | The call to look up | ### get_call_summary -Get analytics summary — hold times, success rates, call volume. No parameters. +Stored summary, action items, and sentiment for a persisted call. -Returns: Aggregate statistics across all calls. +| Param | Type | Required | Description | +|-------|------|----------|-------------| +| `call_id` | string | Yes | The call to look up | ### search_call_history -Search past calls by number, company, or date range. +Search past call records. | Param | Type | Required | Description | |-------|------|----------|-------------| -| `query` | string | Yes | Search term (phone number, company name) | -| `limit` | int | No | Max results (default: 20) | +| `phone_number` | string | No | Filter by phone number (partial match) | +| `intent` | string | No | Filter by intent text (partial match) | +| `limit` | int | No | Max results (default: 10) | + +### get_call_flow + +Look up the stored IVR call flow for a phone number. + +| Param | Type | Required | Description | +|-------|------|----------|-------------| +| `phone_number` | string | Yes | Number to look up (E.164) | + +### create_call_flow + +Store a new IVR call flow by hand. + +| Param | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Human-readable name | +| `phone_number` | string | Yes | Phone number (E.164) | +| `steps_json` | string | Yes | JSON array of call flow steps | +| `notes` | string | No | General notes | ### learn_call_flow -Build a reusable call flow from a completed exploration call. +Build (or refine) a reusable IVR call flow from a completed hold-slayer +exploration call. Exploration calls record every IVR prompt heard and DTMF +sent; this turns those discoveries into a stored flow so the next call +navigates directly. | Param | Type | Required | Description | |-------|------|----------|-------------| -| `call_id` | string | Yes | The exploration call to learn from | -| `company` | string | No | Company name for the flow | +| `call_id` | string | Yes | A completed call that ran in exploration mode | +| `company_name` | string | No | Company name for labeling a new flow | -Returns: The generated CallFlow object. +### list_devices + +List registered devices and their online/offline status. No parameters. + +### gateway_status + +Trunk registration, device count, active calls, engine mode. No parameters. ## Resources -MCP resources provide read-only data that assistants can reference: - | Resource URI | Description | |-------------|-------------| -| `gateway://status` | Current gateway status — trunk registration, active calls, service health | -| `gateway://calls` | List of all active calls with current status | -| `gateway://calls/{call_id}` | Detailed status for a specific call | -| `gateway://flows` | List of all stored call flows | -| `gateway://analytics` | Call analytics summary | +| `gateway://status` | Current gateway status — trunk registration, active calls | +| `gateway://call-flows` | List of all stored call flows | +| `gateway://active-calls` | All active calls with current status | -## Configuration +## Connecting an AI Assistant -The MCP server is mounted on the FastAPI app at `/mcp`. +Claude Code: -### Connecting an AI Assistant +```bash +claude mcp add hold-slayer --transport http http://localhost:8000/mcp/ \ + --header "Authorization: Bearer $API_TOKEN" +``` -Add to your MCP client configuration (e.g., Claude Desktop, Cline): +Generic MCP client configuration: ```json { "mcpServers": { "hold-slayer": { - "url": "http://localhost:8000/mcp/sse" + "url": "http://localhost:8000/mcp/", + "headers": {"Authorization": "Bearer "} } } } @@ -132,8 +176,6 @@ Add to your MCP client configuration (e.g., Claude Desktop, Cline): ## Example Conversation -Here is how an AI assistant would use the MCP tools to handle a complete call: - **User:** "Call Chase Bank and dispute the Amazon charge from December 15th" **Assistant actions:** @@ -141,15 +183,8 @@ Here is how an AI assistant would use the MCP tools to handle a complete call: 1. Calls `make_call(number="+18005551234", mode="hold_slayer", intent="dispute Amazon charge Dec 15th", call_flow_id="chase-bank-main")` 2. Receives `call_id: "call_abc123"` 3. Polls `get_call_status("call_abc123")` periodically -4. Status progression: `trying` → `ringing` → `connected` → `on_hold` +4. Status progression: `initiating` → `ringing` → `connected` → `on_hold` 5. Tells user: "I'm on hold with Chase Bank. Currently 4 minutes in. I'll let you know when someone picks up." 6. Status changes to `transferring` — human detected! 7. Tells user: "A live agent just picked up. I'm transferring the call to your desk phone now. Pick up!" -8. After the call, calls `learn_call_flow("call_abc123", company="Chase Bank")` to save the IVR path for next time. - -**User:** "How long was I on hold?" - -**Assistant actions:** - -1. Calls `get_call_summary()` -2. Reports: "Your Chase Bank call lasted 12 minutes total, with 8 minutes on hold. The disputes department averages 6 minutes hold time on Tuesdays." +8. After the call, calls `learn_call_flow("call_abc123", company_name="Chase Bank")` to save the IVR path for next time. diff --git a/mcp_server/server.py b/mcp_server/server.py index 3c641d3..82ebab9 100644 --- a/mcp_server/server.py +++ b/mcp_server/server.py @@ -264,6 +264,70 @@ def create_mcp_server( except Exception as e: return f"Error creating call flow: {e}" + @mcp.tool() + async def learn_call_flow(call_id: str, company_name: str = "") -> str: + """ + Build (or refine) a reusable IVR call flow from a completed + hold-slayer exploration call. + + Exploration calls record every IVR prompt heard and DTMF sent; + this turns those discoveries into a stored call flow so the next + call to that number navigates directly instead of exploring. + If a flow already exists for the number, the discoveries refine + it (timeouts averaged, usage counters updated). + + Args: + call_id: A completed call that ran in exploration mode + company_name: Optional company name for labeling a new flow + """ + from db.database import session_scope + from services import call_persistence as store + from services.call_flow_learner import CallFlowLearner + from services.llm_client import get_llm + + try: + async with session_scope() as session: + record = await store.get_record(session, call_id) + if not record: + return f"No record found for call {call_id}." + + steps = (record.metadata_ or {}).get("exploration_steps") or [] + if not steps: + return ( + f"Call {call_id} has no exploration data to learn from. " + "Only hold-slayer calls without a stored flow record " + "IVR discoveries." + ) + + learner = CallFlowLearner(llm_client=get_llm()) + existing = await store.get_flow_by_number( + session, record.remote_number + ) + if existing: + flow = await learner.merge_discoveries( + store.flow_to_model(existing), steps, intent=record.intent + ) + await store.update_flow_from_model(session, existing, flow) + return ( + f"Refined existing flow '{existing.name}' for " + f"{record.remote_number} from {len(steps)} discoveries " + f"({len(flow.steps)} steps, used {flow.times_used}x)." + ) + + flow = await learner.build_flow( + phone_number=record.remote_number, + discovered_steps=steps, + intent=record.intent, + company_name=company_name or None, + ) + await store.save_learned_flow(session, flow) + return ( + f"Learned new flow '{flow.name}' with {len(flow.steps)} " + f"steps from {len(steps)} discoveries (ID: {flow.id})." + ) + except Exception as e: + return f"Error learning call flow: {e}" + @mcp.tool() async def send_dtmf(call_id: str, digits: str) -> str: """ diff --git a/models/call.py b/models/call.py index d9863c4..7511718 100644 --- a/models/call.py +++ b/models/call.py @@ -82,6 +82,9 @@ class ActiveCall(BaseModel): transcript_chunks: list[TranscriptEntry] = Field(default_factory=list) current_step_id: Optional[str] = None # Current position in call flow services: list[str] = Field(default_factory=list) # Active services on this call + # IVR discoveries from hold-slayer exploration mode; persisted with + # the call record so learn_call_flow can build a flow afterwards + exploration_steps: list[dict] = Field(default_factory=list) @property def duration(self) -> int: diff --git a/pyproject.toml b/pyproject.toml index ed8ab45..b201e22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,34 +13,30 @@ dependencies = [ # Web framework "fastapi>=0.115.0", "uvicorn[standard]>=0.32.0", - "websockets>=13.0", - + # Database "sqlalchemy[asyncio]>=2.0.36", "asyncpg>=0.30.0", "alembic>=1.14.0", - + # Settings & validation "pydantic>=2.10.0", "pydantic-settings>=2.6.0", - + # SIP signaling "sippy>=1.2.0", - + # Audio analysis "numpy>=1.26.0", - "librosa>=0.10.0", - "soundfile>=0.12.0", - + # HTTP client (for Speaches STT) "httpx>=0.28.0", - + # MCP server (3.x — http_app + StaticTokenVerifier) "fastmcp>=3.0.0", - + # Utilities "python-slugify>=8.0.0", - "python-multipart>=0.0.12", ] [project.optional-dependencies] diff --git a/services/call_persistence.py b/services/call_persistence.py index 6390eda..8779557 100644 --- a/services/call_persistence.py +++ b/services/call_persistence.py @@ -136,6 +136,36 @@ async def create_flow( return row +async def save_learned_flow(session: AsyncSession, flow: CallFlow) -> StoredCallFlow: + """The one CallFlow-model → row mapping (auto-learned flows).""" + row = StoredCallFlow( + id=flow.id, + name=flow.name, + phone_number=flow.phone_number, + description=flow.description, + steps=[s.model_dump(mode="json") for s in flow.steps], + tags=flow.tags, + notes=flow.notes, + times_used=flow.times_used, + last_used=flow.last_used, + last_verified=datetime.now(), + ) + session.add(row) + await session.flush() + return row + + +async def update_flow_from_model( + session: AsyncSession, row: StoredCallFlow, flow: CallFlow +) -> None: + """Write a refined CallFlow back onto its existing row.""" + row.steps = [s.model_dump(mode="json") for s in flow.steps] + row.times_used = flow.times_used + row.last_used = flow.last_used + row.notes = flow.notes + await session.flush() + + # ================================================================ # Devices # ================================================================ @@ -320,7 +350,10 @@ async def _finalize_call_record(call: ActiveCall, final_status: CallStatus) -> N } for c in call.classification_history ] - record.metadata_ = {"services": list(call.services)} + metadata = {"services": list(call.services)} + if call.exploration_steps: + metadata["exploration_steps"] = call.exploration_steps + record.metadata_ = metadata # Each transcript entry gets its own row with a sequence number # and real offset so the dashboard can render click-to-seek. diff --git a/services/hold_slayer.py b/services/hold_slayer.py index b92f781..c953264 100644 --- a/services/hold_slayer.py +++ b/services/hold_slayer.py @@ -294,7 +294,7 @@ class HoldSlayerService: logger.info(f"🔍 Exploration mode: discovering IVR for {call.remote_number}") await self.call_manager.update_status(call.id, CallStatus.NAVIGATING_IVR) - discovered_steps: list[dict] = [] + discovered_steps = call.exploration_steps # persisted with the record max_time = self.settings.hold_slayer.max_hold_time start_time = time.time() diff --git a/tests/test_learner.py b/tests/test_learner.py new file mode 100644 index 0000000..64d51b7 --- /dev/null +++ b/tests/test_learner.py @@ -0,0 +1,137 @@ +""" +Call-flow learner tests. + +Exploration discoveries become a linked CallFlow, survive with the +persisted call record, and the learn_call_flow MCP tool turns them +into a stored flow (refining on subsequent calls). +""" + +import pytest +from fastmcp import Client +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from sqlalchemy.pool import StaticPool + +import db.database as dbmod +import services.llm_client as llm_mod +from core.call_manager import CallManager +from core.event_bus import EventBus +from db.database import Base +from mcp_server.server import create_mcp_server +from models.call import CallStatus +from models.call_flow import ActionType +from services import call_persistence as store +from services.call_flow_learner import CallFlowLearner + +DISCOVERIES = [ + {"timestamp": 1.0, "audio_type": "ringing", "confidence": 0.9, + "transcript": "", "action_taken": None}, + {"timestamp": 4.0, "audio_type": "ivr_prompt", "confidence": 0.8, + "transcript": "press 1 for english press 2 for french", + "action_taken": {"dtmf": "1"}}, + {"timestamp": 8.0, "audio_type": "ivr_prompt", "confidence": 0.8, + "transcript": "press 1 for billing press 2 for support press 0 for an agent", + "action_taken": {"dtmf": "0"}}, + {"timestamp": 12.0, "audio_type": "music", "confidence": 0.9, + "transcript": "", "action_taken": None}, + {"timestamp": 200.0, "audio_type": "live_human", "confidence": 0.85, + "transcript": "thank you for holding, how can I help", + "action_taken": None}, +] + + +class TestBuildFlow: + async def test_discoveries_become_linked_steps(self): + learner = CallFlowLearner(llm_client=None) + flow = await learner.build_flow( + phone_number="+18005551234", + discovered_steps=DISCOVERIES, + intent="dispute a charge", + ) + + # ringing is skipped; menus/hold/human map to actions in order + assert [s.action for s in flow.steps] == [ + ActionType.DTMF, ActionType.DTMF, ActionType.HOLD, ActionType.TRANSFER, + ] + assert [s.action_value for s in flow.steps[:2]] == ["1", "0"] + assert [s.next_step for s in flow.steps[:-1]] == [s.id for s in flow.steps[1:]] + assert "auto-learned" in flow.tags + assert flow.phone_number == "+18005551234" + + +class TestExplorationPersistence: + async def test_exploration_steps_survive_with_the_record(self, mem_db): + cm = CallManager(EventBus(), on_call_ended=store.persist_call_on_end) + call = await cm.create_call("+18005551234", intent="dispute a charge") + call.exploration_steps.extend(DISCOVERIES) + await cm.end_call(call.id, CallStatus.COMPLETED) + + async with mem_db() as session: + record = await store.get_record(session, call.id) + assert record.metadata_["exploration_steps"] == DISCOVERIES + + +@pytest.fixture +async def mem_db(monkeypatch): + engine = create_async_engine( + "sqlite+aiosqlite:///:memory:", + poolclass=StaticPool, + connect_args={"check_same_thread": False}, + ) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + factory = async_sessionmaker(engine, expire_on_commit=False) + monkeypatch.setattr(dbmod, "_engine", engine) + monkeypatch.setattr(dbmod, "_session_factory", factory) + yield factory + await engine.dispose() + + +@pytest.fixture +def no_llm(monkeypatch): + """learn_call_flow must work without an LLM (labels stay heuristic).""" + monkeypatch.setattr(llm_mod, "_shared_client", None) + monkeypatch.setattr(llm_mod, "_shared_failed", True) + + +class TestLearnCallFlowTool: + async def _completed_exploration_call(self, number: str) -> str: + cm = CallManager(EventBus(), on_call_ended=store.persist_call_on_end) + call = await cm.create_call(number, intent="dispute a charge") + call.exploration_steps.extend(DISCOVERIES) + await cm.end_call(call.id, CallStatus.COMPLETED) + return call.id + + async def test_learns_then_refines(self, mem_db, no_llm): + call_id = await self._completed_exploration_call("+18005551234") + + mcp = create_mcp_server(lambda: None) + async with Client(mcp) as client: + result = await client.call_tool("learn_call_flow", {"call_id": call_id}) + assert "Learned new flow" in result.content[0].text + + async with mem_db() as session: + row = await store.get_flow_by_number(session, "+18005551234") + assert row is not None + assert len(row.steps) == 4 + assert "auto-learned" in row.tags + + result = await client.call_tool("learn_call_flow", {"call_id": call_id}) + assert "Refined existing flow" in result.content[0].text + + async def test_call_without_exploration_data(self, mem_db, no_llm): + cm = CallManager(EventBus(), on_call_ended=store.persist_call_on_end) + call = await cm.create_call("+15550001111") + await cm.end_call(call.id, CallStatus.COMPLETED) + + mcp = create_mcp_server(lambda: None) + async with Client(mcp) as client: + result = await client.call_tool("learn_call_flow", {"call_id": call.id}) + assert "no exploration data" in result.content[0].text + + async def test_unknown_call(self, mem_db, no_llm): + mcp = create_mcp_server(lambda: None) + async with Client(mcp) as client: + result = await client.call_tool( + "learn_call_flow", {"call_id": "call_nope"} + ) + assert "No record found" in result.content[0].text diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 3825f05..0cd1f55 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -25,6 +25,7 @@ EXPECTED_TOOLS = { "get_call_recording", "get_call_summary", "search_call_history", + "learn_call_flow", "list_devices", "gateway_status", } diff --git a/tests/test_websocket.py b/tests/test_websocket.py new file mode 100644 index 0000000..609e55c --- /dev/null +++ b/tests/test_websocket.py @@ -0,0 +1,67 @@ +""" +WebSocket event-stream tests. + +The socket is refused (4401) without the bearer token, and an +authorized client immediately receives the synthetic trunk-status +event followed by the replayed recent history. +""" + +import asyncio + +import pytest +from pydantic import SecretStr +from starlette.testclient import TestClient +from starlette.websockets import WebSocketDisconnect + +import main +from config import Settings, get_settings +from core.gateway import AIPSTNGateway +from models.events import EventType, GatewayEvent + + +@pytest.fixture +def ws_app(monkeypatch): + monkeypatch.setattr(get_settings(), "api_token", SecretStr("tok")) + gateway = AIPSTNGateway(settings=Settings()) + main.app.state.gateway = gateway + yield gateway + del main.app.state.gateway + + +def _publish(gateway, call_id: str) -> None: + asyncio.run(gateway.event_bus.publish(GatewayEvent( + type=EventType.CALL_INITIATED, + call_id=call_id, + data={}, + message=f"call {call_id}", + ))) + + +class TestEventStream: + def test_refused_without_token(self, ws_app): + client = TestClient(main.app) + with pytest.raises(WebSocketDisconnect) as exc: + with client.websocket_connect("/ws/events"): + pass + assert exc.value.code == 4401 + + def test_trunk_status_then_replayed_history(self, ws_app): + _publish(ws_app, "call_ws1") + _publish(ws_app, "call_ws2") + + client = TestClient(main.app) + with client.websocket_connect("/ws/events?token=tok") as ws: + first = ws.receive_json() + assert first["type"] == EventType.SIP_TRUNK_REGISTRATION_FAILED.value + replayed = [ws.receive_json() for _ in range(2)] + assert [m["call_id"] for m in replayed] == ["call_ws1", "call_ws2"] + + def test_per_call_stream_filters(self, ws_app): + client = TestClient(main.app) + with client.websocket_connect( + "/ws/calls/call_target/events?token=tok" + ) as ws: + _publish(ws_app, "call_other") + _publish(ws_app, "call_target") + msg = ws.receive_json() + assert msg["call_id"] == "call_target"