Stage 5: Alembic migrations, durable call rows, one transcript truth

Alembic replaces create_all as the schema authority: async env.py
against Base.metadata (CLI and in-app entry paths share it via
config.attributes["connection"]), an autogenerated baseline of the
create_all-era schema, and init_db now runs upgrade head — stamping
the baseline first on a pre-Alembic database so existing deployments
adopt cleanly. create_all remains for tests only.

Calls are durable from the start: CallManager gains an
on_call_created hook (wired to persist_call_on_create) that inserts
an in_progress CallRecord the moment a call is created;
persist_call_on_end finalizes that same row. A SIGKILL mid-call now
leaves an in_progress row instead of erasing the call from history
(verified live against the dev database).

One transcript representation: ActiveCall.transcript_chunks holds
TranscriptEntry (t_offset_ms, speaker, text) — add_transcript stamps
real offsets from connect time, receptionist passes speaker instead
of encoding it into "caller: ..." strings, persisted chunks carry
real seek offsets, and the dead CallRecord.transcript Text column is
dropped by migration. Device.is_online migrates String → Boolean
(with a USING cast for existing rows).

Model de-triplication: CallResponse/CallStatusResponse build via
from_call classmethods (one ActiveCall→response mapping);
DeviceStatus deleted — can_receive_call is a computed field on
Device and the list endpoint returns the domain model; all row↔dict
and row↔domain mapping now lives in call_persistence.py
(record_summary/record_detail/chunk_to_dict + device row functions).

New tests/test_data_layer.py: upgrade-head-matches-models,
pre-Alembic adoption, durable in_progress rows, end-without-create
fallback, transcript offsets, consolidated response models.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

services/call_persistence.py

@@ -22,8 +22,10 @@ from db.database import (
     TranscriptChunk,
     session_scope,
 )
+from db.database import Device as DeviceRow
 from models.call import ActiveCall, CallStatus
 from models.call_flow import CallFlow, CallFlowStep
+from models.device import Device
 
 logger = logging.getLogger(__name__)
 
@@ -45,6 +47,45 @@ def flow_to_model(row: StoredCallFlow) -> CallFlow:
     )
 
 
+def record_summary(row: CallRecord) -> dict:
+    """The one CallRecord-row → list-item mapping."""
+    return {
+        "id": row.id,
+        "direction": row.direction,
+        "remote_number": row.remote_number,
+        "status": row.status,
+        "mode": row.mode,
+        "intent": row.intent,
+        "started_at": row.started_at.isoformat() if row.started_at else None,
+        "ended_at": row.ended_at.isoformat() if row.ended_at else None,
+        "duration": row.duration,
+        "hold_time": row.hold_time,
+        "device_used": row.device_used,
+        "summary": row.summary,
+    }
+
+
+def record_detail(row: CallRecord) -> dict:
+    """Full CallRecord-row mapping, superset of record_summary."""
+    return record_summary(row) | {
+        "action_items": row.action_items,
+        "sentiment": row.sentiment,
+        "call_flow_id": row.call_flow_id,
+        "classification_timeline": row.classification_timeline,
+    }
+
+
+def chunk_to_dict(row: TranscriptChunk) -> dict:
+    """The one TranscriptChunk-row → dict mapping."""
+    return {
+        "seq": row.seq,
+        "t_offset_ms": row.t_offset_ms,
+        "speaker": row.speaker,
+        "text": row.text,
+        "confidence": row.confidence,
+    }
+
+
 # ================================================================
 # Call flows
 # ================================================================
@@ -95,6 +136,49 @@ async def create_flow(
     return row
 
 
+# ================================================================
+# Devices
+# ================================================================
+
+async def create_device_row(session: AsyncSession, device: Device) -> None:
+    """The one Device-model → row mapping."""
+    session.add(DeviceRow(
+        id=device.id,
+        name=device.name,
+        type=device.type.value,
+        sip_uri=device.sip_uri,
+        phone_number=device.phone_number,
+        priority=device.priority,
+        capabilities=device.capabilities,
+        is_online=device.is_online,
+    ))
+    await session.flush()
+
+
+async def update_device_row(
+    session: AsyncSession, device_id: str, values: dict
+) -> None:
+    result = await session.execute(
+        select(DeviceRow).where(DeviceRow.id == device_id)
+    )
+    row = result.scalar_one_or_none()
+    if row is None:
+        return
+    for key, value in values.items():
+        if key == "type" and value is not None:
+            value = value.value if hasattr(value, "value") else value
+        setattr(row, key, value)
+
+
+async def delete_device_row(session: AsyncSession, device_id: str) -> None:
+    result = await session.execute(
+        select(DeviceRow).where(DeviceRow.id == device_id)
+    )
+    row = result.scalar_one_or_none()
+    if row is not None:
+        await session.delete(row)
+
+
 # ================================================================
 # Call history / records
 # ================================================================
@@ -156,71 +240,96 @@ async def latest_recording(
     return result.scalars().first()
 
 
+async def persist_call_on_create(call: ActiveCall) -> None:
+    """Insert an in_progress CallRecord the moment a call starts.
+
+    Wired into CallManager as its on_call_created hook — a crash
+    mid-call leaves this row behind instead of erasing the call from
+    history. persist_call_on_end updates it to the terminal state.
+    """
+    await _with_retry(_insert_in_progress_record, call)
+
+
 async def persist_call_on_end(call: ActiveCall, final_status: CallStatus) -> None:
-    """Insert a CallRecord and any transcript chunks for `call`.
+    """Finalize the CallRecord and write transcript chunks for `call`.
 
     Wired into CallManager as its on_call_ended hook by the
-    composition root in main.py. Retries briefly — losing the row
-    means the call never happened as far as history is concerned, so
-    the final failure logs at ERROR with the payload identifiers.
+    composition root in main.py.
     """
+    await _with_retry(_finalize_call_record, call, final_status)
+
+
+async def _with_retry(write, call: ActiveCall, *args) -> None:
+    """Losing the row means the call never happened as far as history
+    is concerned, so the final failure logs at ERROR with identifiers."""
     for attempt in range(3):
         try:
-            await _write_call_record(call, final_status)
+            await write(call, *args)
             return
         except Exception as e:
             if attempt == 2:
                 logger.error(
-                    f"Call record lost: id={call.id} number={call.remote_number} "
-                    f"status={final_status.value}: {e}"
+                    f"Call record lost ({write.__name__}): id={call.id} "
+                    f"number={call.remote_number}: {e}"
                 )
                 return
             await asyncio.sleep(2**attempt)
 
 
-async def _write_call_record(call: ActiveCall, final_status: CallStatus) -> None:
+async def _insert_in_progress_record(call: ActiveCall) -> None:
     async with session_scope() as session:
-        record = CallRecord(
+        session.add(CallRecord(
             id=call.id,
             direction=call.direction,
             remote_number=call.remote_number,
-            status=final_status.value,
+            status="in_progress",
             mode=call.mode.value,
             intent=call.intent,
             started_at=call.started_at,
-            ended_at=datetime.now(),
-            duration=int(call.duration),
-            hold_time=int(call.hold_time),
             device_used=call.device,
             call_flow_id=call.call_flow_id,
-            classification_timeline=[
-                {
-                    "timestamp": c.timestamp,
-                    "audio_type": c.audio_type.value,
-                    "confidence": c.confidence,
-                }
-                for c in call.classification_history
-            ],
             metadata_={"services": list(call.services)},
-        )
-        session.add(record)
+        ))
 
-        # Each transcript chunk gets its own row with a sequence number
-        # so the dashboard can render them in order with click-to-seek.
-        for seq, text in enumerate(call.transcript_chunks):
-            speaker = "unknown"
-            payload = text
-            if ":" in text:
-                head, rest = text.split(":", 1)
-                head = head.strip().lower()
-                if head in {"caller", "agent", "receptionist", "caller_message"}:
-                    speaker = head if head != "caller_message" else "caller"
-                    payload = rest.strip()
+
+async def _finalize_call_record(call: ActiveCall, final_status: CallStatus) -> None:
+    async with session_scope() as session:
+        record = await get_record(session, call.id)
+        if record is None:
+            # The create-time insert failed (or predates the hook);
+            # write the whole row now instead.
+            record = CallRecord(id=call.id)
+            session.add(record)
+
+        record.direction = call.direction
+        record.remote_number = call.remote_number
+        record.status = final_status.value
+        record.mode = call.mode.value
+        record.intent = call.intent
+        record.started_at = call.started_at
+        record.ended_at = datetime.now()
+        record.duration = int(call.duration)
+        record.hold_time = int(call.hold_time)
+        record.device_used = call.device
+        record.call_flow_id = call.call_flow_id
+        record.classification_timeline = [
+            {
+                "timestamp": c.timestamp,
+                "audio_type": c.audio_type.value,
+                "confidence": c.confidence,
+            }
+            for c in call.classification_history
+        ]
+        record.metadata_ = {"services": list(call.services)}
+
+        # Each transcript entry gets its own row with a sequence number
+        # and real offset so the dashboard can render click-to-seek.
+        for seq, entry in enumerate(call.transcript_chunks):
             session.add(TranscriptChunk(
                 id=f"tc_{uuid.uuid4().hex[:10]}",
                 call_id=call.id,
                 seq=seq,
-                t_offset_ms=0,
-                speaker=speaker,
-                text=payload,
+                t_offset_ms=entry.t_offset_ms,
+                speaker=entry.speaker,
+                text=entry.text,
             ))

services/receptionist.py

@@ -134,13 +134,9 @@ class ReceptionistService:
 
             transcript = await self._listen(call, sip_leg_id)
             if transcript:
-                call.transcript_chunks.append(f"caller: {transcript}")
-                await self.gateway.event_bus.publish(GatewayEvent(
-                    type=EventType.TRANSCRIPT_CHUNK,
-                    call_id=call.id,
-                    data={"text": transcript, "speaker": "caller"},
-                    message=f"📝 caller: {transcript[:80]}",
-                ))
+                await self.gateway.call_manager.add_transcript(
+                    call.id, transcript, speaker="caller"
+                )
 
             classification = await self._classify(call, transcript, routing_decision)
             call.intent = classification.get("intent")
@@ -390,7 +386,9 @@ class ReceptionistService:
                     await self._service_error(call.id, "transcription", e)
 
             if message_text:
-                call.transcript_chunks.append(f"caller_message: {message_text}")
+                await self.gateway.call_manager.add_transcript(
+                    call.id, message_text, speaker="caller"
+                )
 
             await self.gateway.event_bus.publish(GatewayEvent(
                 type=EventType.RECEPTIONIST_MESSAGE_SAVED,