Stage 5: Alembic migrations, durable call rows, one transcript truth

Alembic replaces create_all as the schema authority: async env.py
against Base.metadata (CLI and in-app entry paths share it via
config.attributes["connection"]), an autogenerated baseline of the
create_all-era schema, and init_db now runs upgrade head — stamping
the baseline first on a pre-Alembic database so existing deployments
adopt cleanly. create_all remains for tests only.

Calls are durable from the start: CallManager gains an
on_call_created hook (wired to persist_call_on_create) that inserts
an in_progress CallRecord the moment a call is created;
persist_call_on_end finalizes that same row. A SIGKILL mid-call now
leaves an in_progress row instead of erasing the call from history
(verified live against the dev database).

One transcript representation: ActiveCall.transcript_chunks holds
TranscriptEntry (t_offset_ms, speaker, text) — add_transcript stamps
real offsets from connect time, receptionist passes speaker instead
of encoding it into "caller: ..." strings, persisted chunks carry
real seek offsets, and the dead CallRecord.transcript Text column is
dropped by migration. Device.is_online migrates String → Boolean
(with a USING cast for existing rows).

Model de-triplication: CallResponse/CallStatusResponse build via
from_call classmethods (one ActiveCall→response mapping);
DeviceStatus deleted — can_receive_call is a computed field on
Device and the list endpoint returns the domain model; all row↔dict
and row↔domain mapping now lives in call_persistence.py
(record_summary/record_detail/chunk_to_dict + device row functions).

New tests/test_data_layer.py: upgrade-head-matches-models,
pre-Alembic adoption, durable in_progress rows, end-without-create
fallback, transcript offsets, consolidated response models.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

db/database.py

@@ -6,7 +6,7 @@ PostgreSQL via asyncpg + SQLAlchemy async.
 
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
-from datetime import datetime
+from pathlib import Path
 
 from sqlalchemy import (
     JSON,
@@ -51,7 +51,6 @@ class CallRecord(Base):
     hold_time = Column(Integer, default=0)  # seconds spent on hold
     device_used = Column(String)
     recording_path = Column(String, nullable=True)
-    transcript = Column(Text, nullable=True)
     summary = Column(Text, nullable=True)
     action_items = Column(JSON, nullable=True)
     sentiment = Column(String, nullable=True)
@@ -94,7 +93,7 @@ class Device(Base):
     sip_uri = Column(String, nullable=True)  # sip:robert@gateway.helu.ca
     phone_number = Column(String, nullable=True)  # For PSTN devices
     priority = Column(Integer, default=10)  # Routing priority (lower = higher priority)
-    is_online = Column(String, default="false")
+    is_online = Column(Boolean, default=False, nullable=False)
     capabilities = Column(JSON, default=list)  # ["voice", "video", "sms"]
     dnd = Column(Boolean, default=False, nullable=False)
     last_seen = Column(DateTime, nullable=True)
@@ -211,11 +210,31 @@ async def get_db() -> AsyncIterator[AsyncSession]:
         yield session
 
 
+# The autogenerated baseline revision — a schema created by the old
+# create_all path is identical to it, so such databases are stamped
+# here and then migrated forward like any other.
+_BASELINE_REVISION = "1173a71329ed"
+
+
+def _upgrade_to_head(connection) -> None:
+    from alembic import command
+    from alembic.config import Config
+    from sqlalchemy import inspect
+
+    cfg = Config(str(Path(__file__).resolve().parent.parent / "alembic.ini"))
+    cfg.attributes["connection"] = connection
+
+    inspector = inspect(connection)
+    if not inspector.has_table("alembic_version") and inspector.has_table("call_records"):
+        command.stamp(cfg, _BASELINE_REVISION)
+    command.upgrade(cfg, "head")
+
+
 async def init_db():
-    """Create all tables. For development; use Alembic migrations in production."""
+    """Bring the schema to Alembic head (tests create tables directly)."""
     engine = get_engine()
     async with engine.begin() as conn:
-        await conn.run_sync(Base.metadata.create_all)
+        await conn.run_sync(_upgrade_to_head)
 
 
 async def close_db():

db/migrations/env.py

@@ -0,0 +1,68 @@
+"""
+Alembic environment — async engine against Base.metadata.
+
+Two entry paths:
+- CLI (``alembic upgrade head``): builds an async engine from
+  Settings.database_url and runs migrations on it.
+- App startup (db.database.init_db): passes an already-open
+  connection via ``config.attributes["connection"]`` so migrations
+  run inside the app's engine instead of opening a second one.
+"""
+
+import asyncio
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import pool
+from sqlalchemy.ext.asyncio import create_async_engine
+
+from config import get_settings
+from db.database import Base
+
+config = context.config
+
+# Only configure logging on standalone CLI runs — inside the app this
+# would clobber uvicorn's logger setup.
+if config.config_file_name is not None and config.attributes.get("connection") is None:
+    fileConfig(config.config_file_name, disable_existing_loggers=False)
+
+target_metadata = Base.metadata
+
+
+def run_migrations_offline() -> None:
+    """Emit SQL to stdout without a live connection (--sql mode)."""
+    context.configure(
+        url=get_settings().database_url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def do_run_migrations(connection) -> None:
+    context.configure(connection=connection, target_metadata=target_metadata)
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+    engine = create_async_engine(get_settings().database_url, poolclass=pool.NullPool)
+    async with engine.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+    await engine.dispose()
+
+
+def run_migrations_online() -> None:
+    connection = config.attributes.get("connection")
+    if connection is not None:
+        do_run_migrations(connection)
+    else:
+        asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

db/migrations/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

db/migrations/versions/1173a71329ed_baseline_schema.py

@@ -0,0 +1,129 @@
+"""baseline schema
+
+Revision ID: 1173a71329ed
+Revises: 
+Create Date: 2026-07-10 07:19:08.321778
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = '1173a71329ed'
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('call_flows',
+    sa.Column('id', sa.String(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('phone_number', sa.String(), nullable=False),
+    sa.Column('description', sa.Text(), nullable=True),
+    sa.Column('steps', sa.JSON(), nullable=False),
+    sa.Column('last_verified', sa.DateTime(), nullable=True),
+    sa.Column('avg_hold_time', sa.Integer(), nullable=True),
+    sa.Column('success_rate', sa.Float(), nullable=True),
+    sa.Column('times_used', sa.Integer(), nullable=True),
+    sa.Column('last_used', sa.DateTime(), nullable=True),
+    sa.Column('notes', sa.Text(), nullable=True),
+    sa.Column('tags', sa.JSON(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('updated_at', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_call_flows_phone_number'), 'call_flows', ['phone_number'], unique=False)
+    op.create_table('call_records',
+    sa.Column('id', sa.String(), nullable=False),
+    sa.Column('direction', sa.String(), nullable=False),
+    sa.Column('remote_number', sa.String(), nullable=False),
+    sa.Column('status', sa.String(), nullable=False),
+    sa.Column('mode', sa.String(), nullable=False),
+    sa.Column('intent', sa.Text(), nullable=True),
+    sa.Column('started_at', sa.DateTime(), nullable=True),
+    sa.Column('ended_at', sa.DateTime(), nullable=True),
+    sa.Column('duration', sa.Integer(), nullable=True),
+    sa.Column('hold_time', sa.Integer(), nullable=True),
+    sa.Column('device_used', sa.String(), nullable=True),
+    sa.Column('recording_path', sa.String(), nullable=True),
+    sa.Column('transcript', sa.Text(), nullable=True),
+    sa.Column('summary', sa.Text(), nullable=True),
+    sa.Column('action_items', sa.JSON(), nullable=True),
+    sa.Column('sentiment', sa.String(), nullable=True),
+    sa.Column('call_flow_id', sa.String(), nullable=True),
+    sa.Column('classification_timeline', sa.JSON(), nullable=True),
+    sa.Column('metadata', sa.JSON(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_call_records_remote_number'), 'call_records', ['remote_number'], unique=False)
+    op.create_table('devices',
+    sa.Column('id', sa.String(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('type', sa.String(), nullable=False),
+    sa.Column('sip_uri', sa.String(), nullable=True),
+    sa.Column('phone_number', sa.String(), nullable=True),
+    sa.Column('priority', sa.Integer(), nullable=True),
+    sa.Column('is_online', sa.String(), nullable=True),
+    sa.Column('capabilities', sa.JSON(), nullable=True),
+    sa.Column('dnd', sa.Boolean(), nullable=False),
+    sa.Column('last_seen', sa.DateTime(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('updated_at', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('recordings',
+    sa.Column('id', sa.String(), nullable=False),
+    sa.Column('call_id', sa.String(), nullable=False),
+    sa.Column('path', sa.String(), nullable=False),
+    sa.Column('format', sa.String(), nullable=True),
+    sa.Column('duration_s', sa.Float(), nullable=True),
+    sa.Column('size_bytes', sa.Integer(), nullable=True),
+    sa.Column('channels', sa.Integer(), nullable=True),
+    sa.Column('started_at', sa.DateTime(), nullable=True),
+    sa.Column('ended_at', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_recordings_call_id'), 'recordings', ['call_id'], unique=False)
+    op.create_table('routing_rules',
+    sa.Column('id', sa.String(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('priority', sa.Integer(), nullable=False),
+    sa.Column('enabled', sa.Boolean(), nullable=False),
+    sa.Column('match', sa.JSON(), nullable=False),
+    sa.Column('action', sa.JSON(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('updated_at', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('transcript_chunks',
+    sa.Column('id', sa.String(), nullable=False),
+    sa.Column('call_id', sa.String(), nullable=False),
+    sa.Column('seq', sa.Integer(), nullable=False),
+    sa.Column('t_offset_ms', sa.Integer(), nullable=True),
+    sa.Column('speaker', sa.String(), nullable=True),
+    sa.Column('text', sa.Text(), nullable=False),
+    sa.Column('confidence', sa.Float(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_transcript_chunks_call_id'), 'transcript_chunks', ['call_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_transcript_chunks_call_id'), table_name='transcript_chunks')
+    op.drop_table('transcript_chunks')
+    op.drop_table('routing_rules')
+    op.drop_index(op.f('ix_recordings_call_id'), table_name='recordings')
+    op.drop_table('recordings')
+    op.drop_table('devices')
+    op.drop_index(op.f('ix_call_records_remote_number'), table_name='call_records')
+    op.drop_table('call_records')
+    op.drop_index(op.f('ix_call_flows_phone_number'), table_name='call_flows')
+    op.drop_table('call_flows')
+    # ### end Alembic commands ###

db/migrations/versions/5187577efc23_drop_dead_transcript_column_boolean_is_.py

@@ -0,0 +1,45 @@
+"""drop dead transcript column, boolean is_online
+
+Revision ID: 5187577efc23
+Revises: 1173a71329ed
+Create Date: 2026-07-10 07:19:40.741327
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = '5187577efc23'
+down_revision: Union[str, None] = '1173a71329ed'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Transcript text lives solely in transcript_chunks rows now.
+    op.drop_column('call_records', 'transcript')
+
+    # String "true"/"false" (or NULL) -> real boolean; NULLs become false.
+    # batch mode so the table-recreate path works on SQLite too.
+    with op.batch_alter_table('devices') as batch_op:
+        batch_op.alter_column(
+            'is_online',
+            existing_type=sa.VARCHAR(),
+            type_=sa.Boolean(),
+            nullable=False,
+            postgresql_using="coalesce(lower(is_online) in ('true', 't', '1'), false)",
+        )
+
+
+def downgrade() -> None:
+    with op.batch_alter_table('devices') as batch_op:
+        batch_op.alter_column(
+            'is_online',
+            existing_type=sa.Boolean(),
+            type_=sa.VARCHAR(),
+            nullable=True,
+            postgresql_using="case when is_online then 'true' else 'false' end",
+        )
+    op.add_column('call_records', sa.Column('transcript', sa.TEXT(), nullable=True))