feat(media): implement PJSUA2 audio capture port; document the media-plane refactor
Implements the tap half of the media path and records why the other half requires moving call placement into PJSUA2. MediaPipeline.create_tap was a stub: it logged "🎤 Audio tap created" and returned a tap that nothing ever fed, so the classifier received no audio on a live call. It now builds a real pj.AudioMediaPort subclass whose onFrameReceived converts the SWIG ByteVector to PCM bytes and fans it out to every tap on the stream. One capture port per stream, shared by all taps: a second port on the same stream would be mixed back into the conference bridge and the call would echo. Thread safety is the constraint here. onFrameReceived runs on a PJSUA2 worker thread — a third execution context beside the asyncio loop and the Sippy ED thread — and touches nothing but AudioTap.feed, which hops to the owning loop via call_soon_threadsafe. An exception escaping into PJSUA2's C++ callback would tear down the worker thread and silently kill media for every call, so the handler catches and logs once per port rather than on every 20ms frame. Also fixes a hard crash found while testing this against real PJSUA2: a media port finalised after Endpoint.libDestroy() calls pjmedia_conf_remove_port against a freed conference bridge and aborts the process on a native assertion. Ports are now released in remove_stream while the bridge still exists, and stop() forces a collection before libDestroy — dropping the last Python reference is not sufficient on its own. Verified against the real bindings: frames fan out to multiple taps, cross the thread boundary intact, and shutdown is clean. add_remote_stream remains a stub, and deliberately so. PJSUA2 exposes no standalone RTP media object — every AudioMedia subclass in the Python bindings is a file player, recorder, tone generator or capture port, and RTP is reachable only via pj.Call.getAudioMedia() on a dialog PJSUA2 itself owns. A design where Sippy owns the dialog can never obtain media from PJSUA2, so that function cannot be written against this API. docs/architecture.md now explains this and records the resolution: PJSUA2 places the trunk call while Sippy keeps the SBC roles (device registration, routing, leg bridging), with the emergency guard and concurrency cap staying first in gateway.make_call regardless of which library dials. The architecture doc also had drift unrelated to media: it described the thread boundary as asyncio.run_in_executor() when the real mechanism is run_coroutine_threadsafe / ED2.callFromThread, claimed two execution contexts where there are three, and cited MediaPipeline.add_stream() and SippyEngine.bridge() — neither of which exists. Corrected, with the data flow now showing the emergency guard and concurrency cap in their real positions. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -20,6 +20,7 @@ PJSUA2 runs in its own thread with a dedicated Endpoint.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import AsyncIterator
|
||||
@@ -98,6 +99,72 @@ class AudioTap:
|
||||
self._active = False
|
||||
|
||||
|
||||
def make_capture_port(stream_id: str, sample_rate: int, channels: int, frame_ms: int):
|
||||
"""Build a PJSUA2 media port that forks conference audio into taps.
|
||||
|
||||
Defined as a factory rather than a module-level class because
|
||||
``pj.AudioMediaPort`` can only be subclassed once ``pjsua2`` imports —
|
||||
and the whole pipeline degrades to stub mode when it doesn't.
|
||||
|
||||
The returned port is a *sink*: the conference bridge transmits into it,
|
||||
and every frame is copied to each registered tap. Returns ``None`` when
|
||||
pjsua2 is unavailable.
|
||||
"""
|
||||
try:
|
||||
import pjsua2 as pj
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
class _CapturePort(pj.AudioMediaPort):
|
||||
"""Receives conference-bridge frames and fans them out to taps.
|
||||
|
||||
``onFrameReceived`` is called on a **PJSUA2 worker thread** — a third
|
||||
execution context alongside the asyncio loop and the Sippy ED thread.
|
||||
It must touch nothing but ``AudioTap.feed``, which is explicitly
|
||||
thread-safe (it hops to the owning loop via ``call_soon_threadsafe``).
|
||||
Reaching into pipeline state, the event bus, or a Sippy object from
|
||||
here would be a data race.
|
||||
"""
|
||||
|
||||
def __init__(self, stream_id: str):
|
||||
super().__init__()
|
||||
self.stream_id = stream_id
|
||||
self.taps: list[AudioTap] = []
|
||||
self._logged_error = False
|
||||
|
||||
def onFrameReceived(self, frame): # noqa: N802 — PJSUA2 C++ callback name
|
||||
try:
|
||||
if not self.taps or frame.size <= 0:
|
||||
return
|
||||
# frame.buf is a SWIG ByteVector of signed chars; the tap
|
||||
# contract is raw little-endian 16-bit PCM.
|
||||
pcm = bytes(bytearray(b & 0xFF for b in frame.buf))
|
||||
for tap in self.taps:
|
||||
tap.feed(pcm)
|
||||
except Exception as e:
|
||||
# An exception escaping into PJSUA2's C++ callback would tear
|
||||
# down the worker thread and silently kill media for every
|
||||
# call. Log once per port rather than on every 20ms frame.
|
||||
if not self._logged_error:
|
||||
self._logged_error = True
|
||||
logger.error(
|
||||
f" Audio capture failed for {self.stream_id}: {e}",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
fmt = pj.MediaFormatAudio()
|
||||
fmt.init(
|
||||
pj.PJMEDIA_FORMAT_L16,
|
||||
sample_rate,
|
||||
channels,
|
||||
frame_ms * 1000, # frameTimeUsec
|
||||
16, # bitsPerSample
|
||||
)
|
||||
port = _CapturePort(stream_id)
|
||||
port.createPort(f"tap-{stream_id}", fmt)
|
||||
return port
|
||||
|
||||
|
||||
# ================================================================
|
||||
# Stream Entry — tracks a single media stream in the pipeline
|
||||
# ================================================================
|
||||
@@ -112,6 +179,8 @@ class MediaStream:
|
||||
self.codec = codec
|
||||
self.conf_port: Optional[int] = None # PJSUA2 conference bridge port ID
|
||||
self.transport = None # PJSUA2 SipTransport
|
||||
self.media = None # PJSUA2 AudioMedia for this stream
|
||||
self.capture_port = None # Shared _CapturePort feeding this stream's taps
|
||||
self.rtp_port: Optional[int] = None # Local RTP listen port
|
||||
self.taps: list[AudioTap] = []
|
||||
self.recorder = None # PJSUA2 AudioMediaRecorder
|
||||
@@ -173,6 +242,7 @@ class MediaPipeline:
|
||||
self._next_rtp_port = rtp_start_port
|
||||
self._sample_rate = sample_rate
|
||||
self._channels = channels
|
||||
self._frame_ms = 20 # Must match medConfig.audioFramePtime below
|
||||
self._null_audio = null_audio # Use null audio device (no sound card needed)
|
||||
|
||||
# State
|
||||
@@ -255,11 +325,17 @@ class MediaPipeline:
|
||||
tap.close()
|
||||
self._taps.clear()
|
||||
|
||||
# Remove all streams
|
||||
# Remove all streams (this releases their capture ports)
|
||||
for stream_id in list(self._streams.keys()):
|
||||
self.remove_stream(stream_id)
|
||||
|
||||
# Destroy PJSUA2 endpoint
|
||||
# Destroy PJSUA2 endpoint. Every media port must be collected first:
|
||||
# a port finalised after libDestroy() runs pjmedia_conf_remove_port
|
||||
# against a freed conference bridge and aborts the process. Dropping
|
||||
# the last Python reference is not enough on its own — force the
|
||||
# collection here rather than leaving it to interpreter exit.
|
||||
gc.collect()
|
||||
|
||||
if self._endpoint:
|
||||
try:
|
||||
self._endpoint.libDestroy()
|
||||
@@ -350,6 +426,19 @@ class MediaPipeline:
|
||||
tap.close()
|
||||
self._taps.pop(stream_id, None)
|
||||
|
||||
# Release the capture port while the conference bridge still exists.
|
||||
# A port garbage-collected after Endpoint.libDestroy() calls
|
||||
# pjmedia_conf_remove_port against a freed bridge and aborts the
|
||||
# process on a native assertion — a hard crash, not an exception.
|
||||
if stream.capture_port is not None:
|
||||
try:
|
||||
if stream.media is not None:
|
||||
stream.media.stopTransmit(stream.capture_port)
|
||||
except Exception as e:
|
||||
logger.debug(f" stopTransmit failed for {stream_id}: {e}")
|
||||
stream.capture_port.taps.clear()
|
||||
stream.capture_port = None
|
||||
|
||||
# Stop recording
|
||||
if stream.recorder:
|
||||
try:
|
||||
@@ -426,16 +515,28 @@ class MediaPipeline:
|
||||
self._taps[stream_id] = []
|
||||
self._taps[stream_id].append(tap)
|
||||
|
||||
if self._endpoint and stream and stream.conf_port is not None:
|
||||
if self._endpoint and stream and stream.media is not None:
|
||||
try:
|
||||
import pjsua2 as pj
|
||||
# Create an AudioMediaPort that captures frames
|
||||
# and feeds them to the tap
|
||||
# In PJSUA2, we'd subclass AudioMediaPort and implement
|
||||
# onFrameReceived to call tap.feed(frame_data)
|
||||
logger.info(f" 🎤 Audio tap created for {stream_id} (PJSUA2)")
|
||||
# One capture port per stream, shared by every tap on it:
|
||||
# the bridge would otherwise mix each additional port back
|
||||
# into the conference and the call would echo.
|
||||
if stream.capture_port is None:
|
||||
port = make_capture_port(
|
||||
stream_id, self._sample_rate, self._channels, self._frame_ms
|
||||
)
|
||||
if port is not None:
|
||||
# The stream's media transmits into the capture port,
|
||||
# not the reverse — the port is a sink.
|
||||
stream.media.startTransmit(port)
|
||||
stream.capture_port = port
|
||||
logger.info(f" 🎤 Audio tap created for {stream_id} (PJSUA2)")
|
||||
|
||||
if stream.capture_port is not None:
|
||||
stream.capture_port.taps.append(tap)
|
||||
except Exception as e:
|
||||
logger.error(f" Failed to create PJSUA2 tap for {stream_id}: {e}")
|
||||
logger.error(
|
||||
f" Failed to create PJSUA2 tap for {stream_id}: {e}", exc_info=True
|
||||
)
|
||||
else:
|
||||
logger.info(f" 🎤 Audio tap created for {stream_id} (virtual)")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user