palladium/tests/test_notebooks.py

"""Repo-level notebook validation — structure only, kernel-free.

Layer 0 of the testing story (see CLAUDE.md and the pattern docs): runs
from the ROOT venv against every master's committed ``.ipynb``. Content
and engine pins live in each master's own ``tests/``.

Tiers (defined in ``nbcheck.py``):
  structural — every notebook: parses, python3 kernel, no error outputs,
               cleanly executed top-to-bottom, no duplicate claimed-unique
               tags.
  notebook-first — NOTEBOOK_FIRST masters only: the tagged-cell taxonomy
               (topic-bank / engagement-data / gate / data-appendix) with
               its uniqueness and position rules. GRANDFATHERED notebooks
               skip this tier with their recorded reason (``pytest -rs``).
"""

from __future__ import annotations

import pytest

from .nbcheck import (
    ALL_CLASSIFIED,
    GRANDFATHERED,
    NOTEBOOK_FIRST,
    UNIQUE_TAGS,
    cell_tags,
    cells_tagged,
    discover,
    error_outputs,
    execution_problem,
    load,
)


def notebook_first_only(rel: str) -> None:
    if rel in GRANDFATHERED:
        pytest.skip(GRANDFATHERED[rel])


# ── Classification is complete — a new master cannot dodge the suite ──
def test_every_notebook_on_disk_is_classified():
    on_disk = set(discover())
    classified = set(ALL_CLASSIFIED)
    assert on_disk == classified, (
        f"unclassified notebooks: {sorted(on_disk - classified)}; "
        f"classified but missing from disk: {sorted(classified - on_disk)} — "
        "add each notebook to NOTEBOOK_FIRST or GRANDFATHERED (with a reason) "
        "in tests/nbcheck.py"
    )
    assert not (NOTEBOOK_FIRST & set(GRANDFATHERED)), "a notebook is in both tiers"


# ── Structural tier — every notebook ─────────────────────────────────
@pytest.mark.parametrize("rel", ALL_CLASSIFIED)
def test_parses_as_nbformat_4(rel):
    nb = load(rel)
    assert nb.nbformat == 4


@pytest.mark.parametrize("rel", ALL_CLASSIFIED)
def test_kernel_is_python3(rel):
    nb = load(rel)
    assert nb.metadata.get("kernelspec", {}).get("name") == "python3"


@pytest.mark.parametrize("rel", ALL_CLASSIFIED)
def test_no_error_outputs(rel):
    assert error_outputs(load(rel)) == []


@pytest.mark.parametrize("rel", ALL_CLASSIFIED)
def test_executed_cleanly_top_to_bottom(rel):
    problem = execution_problem(load(rel))
    assert problem is None, problem


@pytest.mark.parametrize("rel", ALL_CLASSIFIED)
def test_no_duplicate_unique_tags(rel):
    nb = load(rel)
    for tag in UNIQUE_TAGS:
        hits = cells_tagged(nb, tag)
        assert len(hits) <= 1, f"tag {tag!r} appears in cells {hits}"


# ── Notebook-first tier — the Assessment Pattern tagged-cell taxonomy ─
@pytest.mark.parametrize("rel", ALL_CLASSIFIED)
def test_has_exactly_one_topic_bank(rel):
    notebook_first_only(rel)
    assert len(cells_tagged(load(rel), "topic-bank")) == 1


@pytest.mark.parametrize("rel", ALL_CLASSIFIED)
def test_engagement_data_present_and_above_widgets(rel):
    notebook_first_only(rel)
    nb = load(rel)
    hits = cells_tagged(nb, "engagement-data")
    assert len(hits) == 1
    # Position: client data must never re-run on a sidebar change, so the
    # cell precedes the widget-defining presentation cell (identified by a
    # widget constructor in its source; if a master builds widgets another
    # way, its own tests carry the precise rule).
    widget_cells = [
        i
        for i, c in enumerate(nb.cells)
        if c.cell_type == "code"
        and "presentation" in cell_tags(c)
        and "mr.Select(" in c.source
    ]
    if widget_cells:
        assert hits[0] < min(widget_cells), (
            "engagement-data cell must sit above the widget cell"
        )


@pytest.mark.parametrize("rel", ALL_CLASSIFIED)
def test_has_one_gate_with_asserts(rel):
    notebook_first_only(rel)
    nb = load(rel)
    hits = cells_tagged(nb, "gate")
    assert len(hits) == 1
    assert "assert" in nb.cells[hits[0]].source, "gate cell carries no assert"


@pytest.mark.parametrize("rel", ALL_CLASSIFIED)
def test_data_appendix_is_last_code_cell(rel):
    notebook_first_only(rel)
    nb = load(rel)
    hits = cells_tagged(nb, "data-appendix")
    assert len(hits) == 1
    last_code = max(
        i for i, c in enumerate(nb.cells) if c.cell_type == "code"
    )
    assert hits[0] == last_code, "data-appendix must be the last code cell"


@pytest.mark.parametrize("rel", ALL_CLASSIFIED)
def test_presentation_cells_do_not_print(rel):
    notebook_first_only(rel)
    nb = load(rel)
    for i in cells_tagged(nb, "presentation"):
        streams = [
            o for o in nb.cells[i].get("outputs", [])
            if o.get("output_type") == "stream"
        ]
        assert not streams, (
            f"presentation cell {i} emits stream output — route diagnostics "
            "through backstage()"
        )


@pytest.mark.parametrize("rel", ALL_CLASSIFIED)
def test_content_cells_are_exec_safe(rel):
    # topic-bank / engagement-data are exec'd by tag in per-master tests
    # and by scripts — magics or shell escapes would break that contract.
    notebook_first_only(rel)
    nb = load(rel)
    for tag in ("topic-bank", "engagement-data"):
        for i in cells_tagged(nb, tag):
            for line in nb.cells[i].source.splitlines():
                stripped = line.lstrip()
                assert not stripped.startswith(("%", "!")), (
                    f"{tag} cell {i} uses a magic/shell escape: {line!r}"
                )