"""Repo-level notebook validation — structure only, kernel-free. Layer 0 of the testing story (see CLAUDE.md and the pattern docs): runs from the ROOT venv against every master's committed ``.ipynb``. Content and engine pins live in each master's own ``tests/``. Tiers (defined in ``nbcheck.py``): structural — every notebook: parses, python3 kernel, no error outputs, cleanly executed top-to-bottom, no duplicate claimed-unique tags. notebook-first — NOTEBOOK_FIRST masters only: the tagged-cell taxonomy (topic-bank / engagement-data / gate / data-appendix) with its uniqueness and position rules. GRANDFATHERED notebooks skip this tier with their recorded reason (``pytest -rs``). """ from __future__ import annotations import pytest from .nbcheck import ( ALL_CLASSIFIED, GRANDFATHERED, NOTEBOOK_FIRST, UNIQUE_TAGS, cell_tags, cells_tagged, discover, error_outputs, execution_problem, load, ) def notebook_first_only(rel: str) -> None: if rel in GRANDFATHERED: pytest.skip(GRANDFATHERED[rel]) # ── Classification is complete — a new master cannot dodge the suite ── def test_every_notebook_on_disk_is_classified(): on_disk = set(discover()) classified = set(ALL_CLASSIFIED) assert on_disk == classified, ( f"unclassified notebooks: {sorted(on_disk - classified)}; " f"classified but missing from disk: {sorted(classified - on_disk)} — " "add each notebook to NOTEBOOK_FIRST or GRANDFATHERED (with a reason) " "in tests/nbcheck.py" ) assert not (NOTEBOOK_FIRST & set(GRANDFATHERED)), "a notebook is in both tiers" # ── Structural tier — every notebook ───────────────────────────────── @pytest.mark.parametrize("rel", ALL_CLASSIFIED) def test_parses_as_nbformat_4(rel): nb = load(rel) assert nb.nbformat == 4 @pytest.mark.parametrize("rel", ALL_CLASSIFIED) def test_kernel_is_python3(rel): nb = load(rel) assert nb.metadata.get("kernelspec", {}).get("name") == "python3" @pytest.mark.parametrize("rel", ALL_CLASSIFIED) def test_no_error_outputs(rel): assert error_outputs(load(rel)) == [] @pytest.mark.parametrize("rel", ALL_CLASSIFIED) def test_executed_cleanly_top_to_bottom(rel): problem = execution_problem(load(rel)) assert problem is None, problem @pytest.mark.parametrize("rel", ALL_CLASSIFIED) def test_no_duplicate_unique_tags(rel): nb = load(rel) for tag in UNIQUE_TAGS: hits = cells_tagged(nb, tag) assert len(hits) <= 1, f"tag {tag!r} appears in cells {hits}" # ── Notebook-first tier — the Assessment Pattern tagged-cell taxonomy ─ @pytest.mark.parametrize("rel", ALL_CLASSIFIED) def test_has_exactly_one_topic_bank(rel): notebook_first_only(rel) assert len(cells_tagged(load(rel), "topic-bank")) == 1 @pytest.mark.parametrize("rel", ALL_CLASSIFIED) def test_engagement_data_present_and_above_widgets(rel): notebook_first_only(rel) nb = load(rel) hits = cells_tagged(nb, "engagement-data") assert len(hits) == 1 # Position: client data must never re-run on a sidebar change, so the # cell precedes the widget-defining presentation cell (identified by a # widget constructor in its source; if a master builds widgets another # way, its own tests carry the precise rule). widget_cells = [ i for i, c in enumerate(nb.cells) if c.cell_type == "code" and "presentation" in cell_tags(c) and "mr.Select(" in c.source ] if widget_cells: assert hits[0] < min(widget_cells), ( "engagement-data cell must sit above the widget cell" ) @pytest.mark.parametrize("rel", ALL_CLASSIFIED) def test_has_one_gate_with_asserts(rel): notebook_first_only(rel) nb = load(rel) hits = cells_tagged(nb, "gate") assert len(hits) == 1 assert "assert" in nb.cells[hits[0]].source, "gate cell carries no assert" @pytest.mark.parametrize("rel", ALL_CLASSIFIED) def test_data_appendix_is_last_code_cell(rel): notebook_first_only(rel) nb = load(rel) hits = cells_tagged(nb, "data-appendix") assert len(hits) == 1 last_code = max( i for i, c in enumerate(nb.cells) if c.cell_type == "code" ) assert hits[0] == last_code, "data-appendix must be the last code cell" @pytest.mark.parametrize("rel", ALL_CLASSIFIED) def test_presentation_cells_do_not_print(rel): notebook_first_only(rel) nb = load(rel) for i in cells_tagged(nb, "presentation"): streams = [ o for o in nb.cells[i].get("outputs", []) if o.get("output_type") == "stream" ] assert not streams, ( f"presentation cell {i} emits stream output — route diagnostics " "through backstage()" ) @pytest.mark.parametrize("rel", ALL_CLASSIFIED) def test_content_cells_are_exec_safe(rel): # topic-bank / engagement-data are exec'd by tag in per-master tests # and by scripts — magics or shell escapes would break that contract. notebook_first_only(rel) nb = load(rel) for tag in ("topic-bank", "engagement-data"): for i in cells_tagged(nb, tag): for line in nb.cells[i].source.splitlines(): stripped = line.lstrip() assert not stripped.startswith(("%", "!")), ( f"{tag} cell {i} uses a magic/shell escape: {line!r}" )