🐾 fix(parsers): rasterize SVG instead of failing ingest

"svg" was in both IMAGE_EXTENSIONS and PYMUPDF_EXTENSIONS, and the image
check ran first, so every SVG reached PIL.Image.open — which cannot decode
vector XML. Ingest raised UnidentifiedImageError, logged "Failed to read
image file_type=svg", and counted documents_parsed_total{status="error"}.
SVG ingest has never worked.

Rasterize to PNG instead, via PyMuPDF (already a dependency). This also
unblocks the vision stage downstream: it sends images to a vision LLM as a
data: URI, which cannot carry image/svg+xml either, so the description and
OCR fields were unreachable for SVG regardless of the parse fix.

Route SVG explicitly before both extension sets. Falling through to
PYMUPDF_EXTENSIONS would parse but not split, and a multi-page Write note
rendered as one document is either blank (no root width/height, so MuPDF
falls back to US-Letter and emits the top-left corner) or an illegible tall
strip (root sized across every stacked page). Splitting per write-page
element is correct for both formats, and since Write never rewrites
existing files both persist indefinitely.

svg_raster is a deliberate twin of daedalus's extraction/svg.py — the repos
share no common package, so the duplication is noted in both docstrings and
fixes belong in both.

lxml is a new dependency: recover=True is needed for the real files that
aren't well-formed XML, and stdlib ElementTree has no equivalent.

.env.example

@@ -91,6 +91,19 @@ SPELUNKER_S3_REGION_NAME=us-east-1
 SPELUNKER_S3_USE_SSL=True
 SPELUNKER_S3_VERIFY=True
 
+# --- Kairos S3 (cross-bucket reads for ingest, source="kairos-mail") ---
+# Consumed by: worker only
+# Kairos renders synced mail to text documents in its own bucket and posts
+# ingest requests with source="kairos-mail". These creds should be scoped
+# read-only to the Kairos bucket in your secret manager.
+KAIROS_S3_ENDPOINT_URL=https://nyx.helu.ca:8555
+KAIROS_S3_ACCESS_KEY_ID=
+KAIROS_S3_SECRET_ACCESS_KEY=
+KAIROS_S3_BUCKET_NAME=kairos
+KAIROS_S3_REGION_NAME=us-east-1
+KAIROS_S3_USE_SSL=True
+KAIROS_S3_VERIFY=True
+
 # --- Celery / RabbitMQ (Oberon) ---------------------------------------------
 # Consumed by: app (producer), worker (consumer). NOT mcp.
 # Remember to percent-encode any password characters that have meaning in a

.gitea/workflows/cve-scan-docker-build.yml

@@ -73,8 +73,6 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
-            type=ref,event=branch
-            type=ref,event=pr
             type=sha,prefix=
             type=raw,value=latest,enable=${{ gitea.ref == 'refs/heads/main' }}
 

docker-compose.yaml

@@ -346,6 +346,13 @@ services:
       - SPELUNKER_S3_REGION_NAME=${SPELUNKER_S3_REGION_NAME}
       - SPELUNKER_S3_USE_SSL=${SPELUNKER_S3_USE_SSL}
       - SPELUNKER_S3_VERIFY=${SPELUNKER_S3_VERIFY}
+      - KAIROS_S3_ENDPOINT_URL=${KAIROS_S3_ENDPOINT_URL}
+      - KAIROS_S3_ACCESS_KEY_ID=${KAIROS_S3_ACCESS_KEY_ID}
+      - KAIROS_S3_SECRET_ACCESS_KEY=${KAIROS_S3_SECRET_ACCESS_KEY}
+      - KAIROS_S3_BUCKET_NAME=${KAIROS_S3_BUCKET_NAME}
+      - KAIROS_S3_REGION_NAME=${KAIROS_S3_REGION_NAME}
+      - KAIROS_S3_USE_SSL=${KAIROS_S3_USE_SSL}
+      - KAIROS_S3_VERIFY=${KAIROS_S3_VERIFY}
       # Celery / RabbitMQ
       - CELERY_BROKER_URL=${CELERY_BROKER_URL}
       - CELERY_RESULT_BACKEND=${CELERY_RESULT_BACKEND}
@@ -371,7 +378,12 @@ services:
     volumes:
       - media:/mnt/media
     healthcheck:
-      test: ["CMD", "celery", "-A", "mnemosyne", "inspect", "ping", "-d", "celery@$$HOSTNAME"]
+      # No -d destination: exec-form CMD has no shell, so $$HOSTNAME never
+      # expanded and the literal "celery@$HOSTNAME" matched no node → every
+      # check failed. There's one worker per container, so an unfiltered ping
+      # (any node replies = healthy) is correct. -t gives the reply room to
+      # round-trip through the broker on Oberon (~450ms observed) under jitter.
+      test: ["CMD", "celery", "-A", "mnemosyne", "inspect", "ping", "-t", "8"]
       interval: 60s
       timeout: 10s
       retries: 3

mnemosyne/library/api/serializers.py

@@ -15,6 +15,7 @@ LIBRARY_TYPE_CHOICES = [
     "film",
     "art",
     "journal",
+    "email",
     "business",
     "finance",
 ]

mnemosyne/library/content_types.py

@@ -241,6 +241,38 @@ LIBRARY_TYPE_DEFAULTS = {
             "4) The commercial purpose — positioning, pricing, capability demonstration."
         ),
     },
+    "email": {
+        "chunking_config": {
+            "strategy": "entry_level",
+            "chunk_size": 512,
+            "chunk_overlap": 32,
+            "respect_boundaries": ["message", "quote", "paragraph"],
+        },
+        "embedding_instruction": (
+            "Represent this email message for retrieval. "
+            "Focus on the sender, recipients, subject, dates, requests and "
+            "commitments made, and the people, organizations, and events discussed."
+        ),
+        "reranker_instruction": (
+            "Re-rank email messages based on relevance to the query. "
+            "Prioritize messages matching the correspondents, subject matter, "
+            "time period, and any specific commitments or requests mentioned."
+        ),
+        "llm_context_prompt": (
+            "The following excerpts are from personal email correspondence. "
+            "This is private content — answer with discretion. Attribute "
+            "statements to their senders, note dates, and distinguish what was "
+            "asked from what was agreed. Quoted text below a reply is earlier "
+            "context, not the sender's own words."
+        ),
+        "vision_prompt": (
+            "Analyze this image from an email message. Identify:\n"
+            "1) Image type (photograph, screenshot, scanned document, chart, signature graphic).\n"
+            "2) What it depicts — people, places, documents, data.\n"
+            "3) Any visible text, dates, or figures.\n"
+            "4) Its role in the message — attachment content, inline illustration, or boilerplate."
+        ),
+    },
     "finance": {
         "chunking_config": {
             "strategy": "section_aware",
@@ -282,7 +314,7 @@ def get_library_type_config(library_type):
 
     Args:
         library_type: One of 'fiction', 'nonfiction', 'technical', 'music',
-                      'film', 'art', 'journal', 'business', 'finance'
+                      'film', 'art', 'journal', 'email', 'business', 'finance'
 
     Returns:
         dict with keys: chunking_config, embedding_instruction,

mnemosyne/library/models.py

@@ -77,6 +77,7 @@ class Library(StructuredNode):
             "film": "Film",
             "art": "Art",
             "journal": "Journal",
+            "email": "Email",
             "business": "Business",
             "finance": "Finance",
         },

mnemosyne/library/services/parsers.py

@@ -31,8 +31,11 @@ PYMUPDF_EXTENSIONS = {
 # Plain text extensions — read directly, no PyMuPDF needed
 PLAINTEXT_EXTENSIONS = {"txt", "md", "csv", "tsv", "log", "json", "yaml", "yml", "xml"}
 
-# Image extensions — store as Image nodes directly
+# Image extensions — store as Image nodes directly.
-IMAGE_EXTENSIONS = {"jpg", "jpeg", "png", "gif", "bmp", "tiff", "tif", "webp", "svg"}
+# SVG is deliberately absent: it is vector XML that Pillow cannot decode and
+# that the vision stage cannot send as a data URI, so it gets rasterized by
+# _parse_svg_file instead.
+IMAGE_EXTENSIONS = {"jpg", "jpeg", "png", "gif", "bmp", "tiff", "tif", "webp"}
 
 # Minimum image dimensions to extract (skip tiny icons/bullets)
 MIN_IMAGE_WIDTH = 50
@@ -98,6 +101,12 @@ class DocumentParser:
         if file_type in PLAINTEXT_EXTENSIONS:
             return self._parse_plaintext(file_path, file_type)
 
+        # Checked before PYMUPDF_EXTENSIONS: PyMuPDF can open an SVG, but
+        # rendering a multi-page Write note as one document yields a blank or
+        # illegible image (see svg_raster), so it needs page-aware handling.
+        if file_type == "svg":
+            return self._parse_svg_file(file_path, file_type)
+
         if file_type in IMAGE_EXTENSIONS:
             return self._parse_image_file(file_path, file_type)
 
@@ -309,6 +318,61 @@ class DocumentParser:
                 file_type=file_type,
             )
 
+    def _parse_svg_file(self, file_path: str, file_type: str) -> ParseResult:
+        """
+        Rasterize an SVG into one ExtractedImage per page.
+
+        SVG is vector XML: Pillow cannot decode it and the vision stage cannot
+        put it in a data URI, so it is rendered to PNG here. Multi-page Write
+        notes become one image per page so each page reaches vision/OCR at a
+        legible size.
+
+        :param file_path: Path to the SVG file.
+        :param file_type: Normalized file extension ("svg").
+        :returns: ParseResult with one image per rendered page.
+        """
+        with DOCUMENT_PARSE_DURATION.labels(file_type=file_type).time():
+            try:
+                from library.services.svg_raster import render_svg_pages
+
+                with open(file_path, "rb") as f:
+                    data = f.read()
+
+                pages = render_svg_pages(data)
+            except Exception as exc:
+                DOCUMENTS_PARSED_TOTAL.labels(file_type=file_type, status="error").inc()
+                logger.error("Failed to rasterize SVG file_type=%s: %s", file_type, exc)
+                raise
+
+            images = [
+                ExtractedImage(
+                    data=png,
+                    ext="png",
+                    width=width,
+                    height=height,
+                    source_page=index,
+                    source_index=0,
+                )
+                for index, (png, width, height) in enumerate(pages)
+            ]
+
+            DOCUMENTS_PARSED_TOTAL.labels(file_type=file_type, status="success").inc()
+            IMAGES_EXTRACTED_TOTAL.labels(file_type=file_type).inc(len(images))
+
+            logger.info(
+                "Parsed SVG file_type=%s pages=%d bytes=%d",
+                file_type,
+                len(images),
+                len(data),
+            )
+
+            return ParseResult(
+                text_blocks=[],
+                images=images,
+                metadata={"page_count": len(images)},
+                file_type=file_type,
+            )
+
     def _parse_image_file(self, file_path: str, file_type: str) -> ParseResult:
         """
         Handle a standalone image file — store as a single ExtractedImage.

mnemosyne/library/services/svg_raster.py

@@ -0,0 +1,263 @@
+"""SVG rasterization for the ingest pipeline.
+
+The vision stage sends each extracted image to a vision LLM as a ``data:`` URI,
+which cannot carry ``image/svg+xml`` — so an SVG has to become raster before it
+can be described, OCR'd, or embedded. PyMuPDF (already a dependency for PDF
+parsing) renders SVG natively, so this needs no cairo/rsvg system libraries.
+
+The non-obvious part is page splitting. Handwritten notes from the Write
+(Stylus Labs) app are a single SVG document holding one or more
+``<svg class="write-page">`` children stacked vertically via x/y offsets. Two
+root formats exist in the wild and *both* must be split per page:
+
+- Older files carry no width/height on the root ``<svg>``. Rendering the
+  document as-is makes PyMuPDF fall back to US-Letter and emit the top-left
+  corner only — ruled lines and no handwriting, in a perfectly valid PNG.
+- Newer files (Write commit eeab021) do carry root width/height, spanning the
+  full stacked extent. Rendering those as-is yields one very tall strip; capped
+  to a sane longest edge, a 5-page note squashes to ~209px wide, well past
+  illegible.
+
+Write never rewrites existing files, so the old format is permanent rather than
+a migration window. Page geometry lives on the ``write-page`` element and is
+identical across both formats, so splitting ignores the root dimensions
+entirely and needs no format detection.
+
+.. note::
+   Daedalus carries a twin of this module at
+   ``backend/daedalus/extraction/svg.py``, which returns base64 for direct
+   chat attachment. The two are deliberately duplicated rather than shared —
+   the repos ship no common package — so fixes belong in both.
+"""
+
+from __future__ import annotations
+
+import copy
+import logging
+import re
+
+from lxml import etree
+
+logger = logging.getLogger(__name__)
+
+SVG_NS = "http://www.w3.org/2000/svg"
+XLINK_NS = "http://www.w3.org/1999/xlink"
+
+_SVG = f"{{{SVG_NS}}}"
+_PAGE_CLASS = "write-page"
+
+# Write documents render on a grey backdrop; pages themselves are transparent,
+# so each rendered page needs an explicit white underlay or strokes land on
+# black in the flattened PNG.
+_PAGE_BACKGROUND = "#ffffff"
+
+#: Longest edge of a rendered page, in pixels. Enough for a vision model to
+#: read handwriting without spending tokens on unusable resolution.
+DEFAULT_TARGET_PX = 1568
+
+#: Cap on pages rendered from one document.
+DEFAULT_MAX_PAGES = 20
+
+# Unquoted attribute value in the root tag, e.g. Write's `width=auto`. Matches
+# only bare alphabetic values so numeric or already-quoted attributes are left
+# alone.
+_UNQUOTED_ATTR = re.compile(rb"(\s[-\w:]+)=([A-Za-z][-\w]*)(?=[\s>])")
+_ROOT_TAG = re.compile(rb"<svg[^>]*>")
+
+
+class SvgRenderError(Exception):
+    """Raised when an SVG cannot be parsed or contains no renderable page."""
+
+
+def _repair_root_tag(data: bytes) -> bytes:
+    """Quote unquoted attribute values in the root ``<svg>`` tag.
+
+    Some Write files emit ``width=auto height=auto``, which is not valid XML.
+    A malformation *inside* the root tag defeats ``recover=True`` differently
+    from one in the body: rather than dropping a subtree, libxml2 abandons the
+    whole document and yields a bare root, so every page becomes invisible.
+    Quoting the values first recovers the full tree.
+    """
+    match = _ROOT_TAG.search(data)
+    if not match:
+        return data
+    repaired = _UNQUOTED_ATTR.sub(rb'\1="\2"', match.group(0))
+    if repaired == match.group(0):
+        return data
+    return data[: match.start()] + repaired + data[match.end() :]
+
+
+def _parser() -> etree.XMLParser:
+    """Build the hardened parser used for all untrusted SVG input.
+
+    ``resolve_entities=False`` blocks XXE — ingest content is untrusted.
+    ``huge_tree`` is required because handwriting path data runs to megabytes.
+    ``recover`` salvages the handful of Write files that emit unescaped
+    attribute content and are not well-formed XML.
+    """
+    return etree.XMLParser(
+        huge_tree=True,
+        resolve_entities=False,
+        no_network=True,
+        recover=True,
+    )
+
+
+def _dimension(element: etree._Element, name: str) -> float | None:
+    """Read a CSS-pixel dimension attribute, tolerating a ``px`` suffix."""
+    raw = element.get(name)
+    if not raw:
+        return None
+    try:
+        return float(raw.strip().removesuffix("px"))
+    except ValueError:
+        return None
+
+
+def _viewbox_size(element: etree._Element) -> tuple[float, float] | None:
+    """Derive width/height from a viewBox extent."""
+    raw = element.get("viewBox")
+    if not raw:
+        return None
+    parts = raw.replace(",", " ").split()
+    if len(parts) != 4:
+        return None
+    try:
+        width, height = float(parts[2]), float(parts[3])
+    except ValueError:
+        return None
+    return (width, height) if width > 0 and height > 0 else None
+
+
+def _element_size(element: etree._Element) -> tuple[float, float] | None:
+    """Resolve an element's rendered size from width/height, else viewBox."""
+    width = _dimension(element, "width")
+    height = _dimension(element, "height")
+    if width and height:
+        return width, height
+    return _viewbox_size(element)
+
+
+def _standalone_page(
+    page: etree._Element,
+    defs: etree._Element | None,
+    width: float,
+    height: float,
+) -> bytes:
+    """Wrap one ``write-page`` element as its own renderable SVG document.
+
+    The page is repositioned to the origin (its x/y place it within the stacked
+    parent) and given an explicit viewBox so the renderer has an unambiguous
+    size. ``defs`` is copied in because pen and ruling definitions live on the
+    root and are referenced by page content.
+    """
+    root = etree.Element(f"{_SVG}svg", nsmap={None: SVG_NS, "xlink": XLINK_NS})
+    root.set("width", str(width))
+    root.set("height", str(height))
+    root.set("viewBox", f"0 0 {width} {height}")
+
+    background = etree.SubElement(root, f"{_SVG}rect")
+    background.set("width", "100%")
+    background.set("height", "100%")
+    background.set("fill", _PAGE_BACKGROUND)
+
+    if defs is not None:
+        root.append(copy.deepcopy(defs))
+
+    element = copy.deepcopy(page)
+    for positional in ("x", "y"):
+        element.attrib.pop(positional, None)
+    element.set("viewBox", f"0 0 {width} {height}")
+    root.append(element)
+
+    return etree.tostring(root)
+
+
+def split_svg_pages(data: bytes) -> list[bytes]:
+    """Split an SVG into one standalone document per renderable page.
+
+    Write multi-page notes yield one document per ``write-page`` child. Any
+    other SVG (a diagram, a logo) yields a single document — the input itself,
+    which the renderer sizes from its own width/height or viewBox.
+
+    :param data: Raw SVG bytes.
+    :returns: One or more standalone SVG documents, in page order.
+    :raises SvgRenderError: If the SVG cannot be parsed or has no usable size.
+    """
+    try:
+        root = etree.fromstring(_repair_root_tag(data), _parser())
+    except etree.XMLSyntaxError as exc:
+        raise SvgRenderError(f"Could not parse SVG: {exc}") from exc
+
+    if root is None:
+        raise SvgRenderError("Could not parse SVG: no root element.")
+
+    defs = root.find(f"{_SVG}defs")
+
+    pages: list[bytes] = []
+    for element in root.iter(f"{_SVG}svg"):
+        if element is root:
+            continue
+        if _PAGE_CLASS not in (element.get("class") or "").split():
+            continue
+        size = _element_size(element)
+        if not size:
+            continue
+        pages.append(_standalone_page(element, defs, *size))
+
+    if pages:
+        return pages
+
+    # Generic SVG: render as-is. It must still be sizeable, or the renderer
+    # would silently substitute a default page box.
+    if not _element_size(root):
+        raise SvgRenderError(
+            "SVG has no width/height or viewBox, so its size is undefined."
+        )
+    return [data]
+
+
+def render_svg_pages(
+    data: bytes,
+    max_pages: int = DEFAULT_MAX_PAGES,
+    target_px: int = DEFAULT_TARGET_PX,
+) -> list[tuple[bytes, int, int]]:
+    """Rasterize an SVG to one PNG per page.
+
+    :param data: Raw SVG bytes.
+    :param max_pages: Cap on pages rendered.
+    :param target_px: Longest edge of each rendered page, in pixels.
+    :returns: One ``(png_bytes, width, height)`` tuple per page, in page order.
+    :raises SvgRenderError: If nothing could be rendered.
+    """
+    import fitz
+
+    pages = split_svg_pages(data)
+
+    rendered: list[tuple[bytes, int, int]] = []
+    for index, page_svg in enumerate(pages[:max_pages]):
+        try:
+            with fitz.open(stream=page_svg, filetype="svg") as document:
+                page = document[0]
+                longest = max(page.rect.width, page.rect.height)
+                zoom = target_px / longest if longest else 1.0
+                pixmap = page.get_pixmap(matrix=fitz.Matrix(zoom, zoom))
+                rendered.append(
+                    (pixmap.tobytes("png"), pixmap.width, pixmap.height)
+                )
+        except Exception as exc:
+            raise SvgRenderError(
+                f"Could not render SVG page {index + 1}: {exc}"
+            ) from exc
+
+    if not rendered:
+        raise SvgRenderError("No renderable pages in SVG.")
+
+    logger.info(
+        "Rasterized SVG pages_rendered=%d total_pages=%d target_px=%d",
+        len(rendered),
+        len(pages),
+        target_px,
+    )
+
+    return rendered

mnemosyne/library/templates/library/library_list.html

@@ -15,6 +15,18 @@
     </div>
 </div>
 
+<form method="get" class="mb-4 flex flex-wrap gap-3 items-end">
+    <div class="form-control">
+        <label class="label"><span class="label-text">Scope</span></label>
+        <select name="scope" class="select select-bordered select-sm">
+            <option value="all"      {% if scope == "all" %}selected{% endif %}>All libraries</option>
+            <option value="global"   {% if scope == "global" %}selected{% endif %}>Global only</option>
+            <option value="daedalus" {% if scope == "daedalus" %}selected{% endif %}>Daedalus workspaces only</option>
+        </select>
+    </div>
+    <button type="submit" class="btn btn-sm btn-outline">Filter</button>
+</form>
+
 {% if error %}
 <div class="alert alert-warning mb-4">
     <span>{{ error }}</span>
@@ -53,8 +65,12 @@
 {% else %}
     {% if not error %}
     <div class="text-center py-12 opacity-60">
+        {% if scope == "all" %}
         <p class="text-lg">No libraries yet.</p>
         <p class="mt-2">Create your first library to get started.</p>
+        {% else %}
+        <p class="text-lg">No libraries match this filter.</p>
+        {% endif %}
     </div>
     {% endif %}
 {% endif %}

mnemosyne/library/tests/test_content_types.py

@@ -21,6 +21,7 @@ class LibraryTypeDefaultsTests(TestCase):
         "film",
         "art",
         "journal",
+        "email",
         "business",
         "finance",
     }

mnemosyne/library/tests/test_svg_raster.py

@@ -0,0 +1,195 @@
+"""
+Tests for SVG rasterization in the ingest pipeline.
+
+SVG is vector XML: Pillow cannot decode it and the vision stage cannot send it
+as a data URI, so it is rendered to PNG at parse time. The cases that matter
+are the ones that fail *silently* — Write notes stack their pages inside one
+document, and rendering that document whole yields a plausible-looking PNG that
+is either blank or an illegible tall strip.
+"""
+
+import io
+import os
+import tempfile
+
+from django.test import TestCase
+from PIL import Image
+
+from library.services.parsers import (
+    IMAGE_EXTENSIONS,
+    DocumentParser,
+)
+from library.services.svg_raster import (
+    SvgRenderError,
+    render_svg_pages,
+    split_svg_pages,
+)
+
+# Geometry copied from a real Write note; the page element is identical across
+# both root formats, which is what makes splitting format-agnostic.
+PAGE_WIDTH = 1094
+PAGE_HEIGHT = 1654
+PAGE_PITCH = 1674  # page height + inter-page gap
+
+
+def write_document(pages: int, root_size: bool = False) -> bytes:
+    """
+    Build a Write-style document.
+
+    :param pages: Number of stacked pages.
+    :param root_size: Emit width/height on the root <svg>. False models
+        pre-eeab021 files (the permanent majority), True models newer saves.
+    """
+    root_attrs = ""
+    if root_size:
+        total = 10 + pages * PAGE_PITCH
+        root_attrs = f' width="{PAGE_WIDTH + 20}" height="{total}"'
+
+    body = "".join(
+        f'<svg class="write-page" x="10" y="{10 + i * PAGE_PITCH}" '
+        f'width="{PAGE_WIDTH}px" height="{PAGE_HEIGHT}px" '
+        f'xmlns="http://www.w3.org/2000/svg">'
+        f'<path d="M 100 {100 + i * 40} L {600 + i * 120} {700 + i * 40}" '
+        f'stroke="#000000" stroke-width="12" fill="none"/></svg>'
+        for i in range(pages)
+    )
+    return (
+        f'<svg id="write-document"{root_attrs} '
+        f'xmlns="http://www.w3.org/2000/svg" '
+        f'xmlns:xlink="http://www.w3.org/1999/xlink">'
+        f'<rect id="write-doc-background" width="100%" height="100%" fill="#808080"/>'
+        f'<defs id="write-defs"><style>.write-flat-pen{{fill:none}}</style></defs>'
+        f"{body}</svg>"
+    ).encode()
+
+
+GENERIC_SVG = (
+    b'<svg xmlns="http://www.w3.org/2000/svg" width="400" height="200" '
+    b'viewBox="0 0 400 200">'
+    b'<path d="M 20 20 L 380 180" stroke="#000" stroke-width="10"/></svg>'
+)
+
+
+def ink_fraction(png: bytes) -> float:
+    """Fraction of non-white pixels — the blank-render canary."""
+    grey = Image.open(io.BytesIO(png)).convert("L")
+    histogram = grey.histogram()
+    return sum(histogram[:200]) / sum(histogram)
+
+
+class SvgSplitTests(TestCase):
+    """Splitting a Write document into per-page SVGs."""
+
+    def test_splits_one_document_per_page(self):
+        for pages in (1, 2, 5):
+            for root_size in (False, True):
+                with self.subTest(pages=pages, root_size=root_size):
+                    document = write_document(pages, root_size=root_size)
+                    self.assertEqual(len(split_svg_pages(document)), pages)
+
+    def test_both_root_formats_split_identically(self):
+        """Root width/height (Write eeab021) must not change the outcome.
+
+        Existing notes are never rewritten, so both formats persist
+        indefinitely and have to render the same.
+        """
+        without = split_svg_pages(write_document(3, root_size=False))
+        with_size = split_svg_pages(write_document(3, root_size=True))
+        self.assertEqual(len(without), len(with_size))
+        self.assertEqual(len(without), 3)
+
+    def test_generic_svg_is_a_single_page(self):
+        self.assertEqual(len(split_svg_pages(GENERIC_SVG)), 1)
+
+    def test_unsized_svg_is_rejected_rather_than_guessed(self):
+        unsized = b'<svg xmlns="http://www.w3.org/2000/svg"><rect/></svg>'
+        with self.assertRaises(SvgRenderError):
+            split_svg_pages(unsized)
+
+    def test_script_inside_defs_does_not_desync_the_split(self):
+        """Regression: a non-greedy <defs>...</defs> regex mis-parses these."""
+        document = write_document(2).replace(
+            b'<defs id="write-defs">',
+            b'<defs id="write-defs"><script><float value="770" /></script>',
+        )
+        self.assertEqual(len(split_svg_pages(document)), 2)
+
+    def test_unquoted_root_attributes_are_repaired(self):
+        """Write can emit ``width=auto``, which is not valid XML.
+
+        A malformation inside the root tag defeats recovery differently from
+        one in the body: libxml2 abandons the whole document and returns a
+        bare root, so every page silently disappears.
+        """
+        document = write_document(2).replace(
+            b'<svg id="write-document"',
+            b'<svg width=auto height=auto id="write-document"',
+        )
+        self.assertEqual(len(split_svg_pages(document)), 2)
+
+
+class SvgRenderTests(TestCase):
+    """Rasterizing pages to PNG."""
+
+    def test_renders_one_png_per_page(self):
+        pages = render_svg_pages(write_document(3))
+        self.assertEqual(len(pages), 3)
+        for png, width, height in pages:
+            self.assertEqual(Image.open(io.BytesIO(png)).format, "PNG")
+            self.assertEqual(max(width, height), 1568)
+
+    def test_rendered_pages_are_not_blank(self):
+        """The pre-patch trap: a whole-document render emits ruling, no ink."""
+        for root_size in (False, True):
+            with self.subTest(root_size=root_size):
+                for png, _, _ in render_svg_pages(
+                    write_document(3, root_size=root_size)
+                ):
+                    self.assertGreater(ink_fraction(png), 0.001)
+
+    def test_pages_are_page_shaped_not_a_stacked_strip(self):
+        """The post-patch trap: root dimensions span every stacked page.
+
+        Rendering that whole gives one tall strip which, once capped, squashes
+        a 5-page note to ~209px wide.
+        """
+        expected = PAGE_HEIGHT / PAGE_WIDTH
+        for root_size in (False, True):
+            with self.subTest(root_size=root_size):
+                for _, width, height in render_svg_pages(
+                    write_document(5, root_size=root_size)
+                ):
+                    self.assertAlmostEqual(height / width, expected, delta=0.05)
+
+    def test_page_cap_is_respected(self):
+        self.assertEqual(len(render_svg_pages(write_document(8), max_pages=5)), 5)
+
+
+class SvgParserIntegrationTests(TestCase):
+    """The parser dispatch — SVG must not reach the Pillow image path."""
+
+    def setUp(self):
+        self.parser = DocumentParser()
+
+    def test_svg_is_not_in_image_extensions(self):
+        # It was, and PIL.Image.open cannot decode SVG, so ingest always failed.
+        self.assertNotIn("svg", IMAGE_EXTENSIONS)
+
+    def test_parse_multipage_svg_yields_one_image_per_page(self):
+        with tempfile.NamedTemporaryFile(suffix=".svg", delete=False) as f:
+            f.write(write_document(3))
+            f.flush()
+            path = f.name
+        try:
+            result = self.parser.parse(path, "svg")
+        finally:
+            os.unlink(path)
+
+        self.assertEqual(len(result.images), 3)
+        self.assertEqual(result.metadata["page_count"], 3)
+        self.assertEqual(result.text_blocks, [])
+        for index, image in enumerate(result.images):
+            # PNG, not svg — the vision stage needs raster for its data URI.
+            self.assertEqual(image.ext, "png")
+            self.assertEqual(image.source_page, index)
+            self.assertGreater(ink_fraction(image.data), 0.001)

mnemosyne/library/tests/test_views.py

@@ -0,0 +1,92 @@
+"""Tests for the library CRUD HTML views.
+
+Currently covers ``library_list``'s Daedalus-workspace scope filter. The
+view loads every ``Library`` node from Neo4j and narrows it by a ``scope``
+GET param (``all`` / ``global`` / ``daedalus``). These tests stub out
+Neo4j entirely — patching ``neo4j_available`` and injecting a fake
+``Library`` class via ``sys.modules`` — so they assert on the queryset
+``.filter(...)`` call the view makes and the context it renders, not on
+real graph behaviour. Mirrors the mocking style in
+``test_search_views_admin_scope.py``.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+from django.contrib.auth import get_user_model
+from django.test import TestCase
+from django.urls import reverse
+
+User = get_user_model()
+
+
+class LibraryListScopeFilterTests(TestCase):
+    """Cover the ``scope`` filter branches of ``library_list``."""
+
+    def setUp(self):
+        self.user = User.objects.create_user(
+            username="op", email="op@example.com", password="pw"
+        )
+        self.client.force_login(self.user)
+        self.url = reverse("library:library-list")
+
+    def _fake_library_cls(self):
+        """Return (Library stub, nodes mock) where ``nodes`` chains fluently.
+
+        ``Library.nodes`` → ``.filter(...)`` → ``.order_by(...)`` all return
+        the same MagicMock so the view's queryset building works regardless
+        of which branch it takes, and ``.filter`` records its kwargs.
+        """
+        fake_nodes = MagicMock()
+        fake_nodes.filter.return_value = fake_nodes
+        fake_nodes.order_by.return_value = []
+        return SimpleNamespace(nodes=fake_nodes), fake_nodes
+
+    def _get(self, fake_library_cls, **params):
+        with patch("library.views.neo4j_available", return_value=True), \
+             patch.dict(
+                 "sys.modules",
+                 {"library.models": SimpleNamespace(Library=fake_library_cls)},
+             ):
+            return self.client.get(self.url, params)
+
+    def test_default_scope_is_all_and_does_not_filter(self):
+        fake_cls, fake_nodes = self._fake_library_cls()
+        response = self._get(fake_cls)
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.context["scope"], "all")
+        fake_nodes.filter.assert_not_called()
+        fake_nodes.order_by.assert_called_once_with("name")
+
+    def test_global_scope_filters_workspace_isnull_true(self):
+        fake_cls, fake_nodes = self._fake_library_cls()
+        response = self._get(fake_cls, scope="global")
+
+        self.assertEqual(response.context["scope"], "global")
+        fake_nodes.filter.assert_called_once_with(workspace_id__isnull=True)
+
+    def test_daedalus_scope_filters_workspace_isnull_false(self):
+        fake_cls, fake_nodes = self._fake_library_cls()
+        response = self._get(fake_cls, scope="daedalus")
+
+        self.assertEqual(response.context["scope"], "daedalus")
+        fake_nodes.filter.assert_called_once_with(workspace_id__isnull=False)
+
+    def test_unknown_scope_does_not_filter(self):
+        """An unexpected scope value degrades to the unfiltered list."""
+        fake_cls, fake_nodes = self._fake_library_cls()
+        response = self._get(fake_cls, scope="bogus")
+
+        self.assertEqual(response.context["scope"], "bogus")
+        fake_nodes.filter.assert_not_called()
+
+    def test_neo4j_unavailable_sets_error_and_empty_list(self):
+        with patch("library.views.neo4j_available", return_value=False):
+            response = self.client.get(self.url)
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(list(response.context["libraries"]), [])
+        self.assertEqual(response.context["error"], "Neo4j is not available.")

mnemosyne/library/views.py

@@ -31,14 +31,20 @@ logger = logging.getLogger(__name__)
 
 @login_required
 def library_list(request):
-    """List all libraries."""
+    """List libraries, optionally filtered by Daedalus-workspace scope."""
+    scope = request.GET.get("scope", "all")
     libraries = []
     error = None
     if neo4j_available():
         try:
             from .models import Library
 
-            libraries = Library.nodes.order_by("name")
+            qs = Library.nodes
+            if scope == "daedalus":
+                qs = qs.filter(workspace_id__isnull=False)
+            elif scope == "global":
+                qs = qs.filter(workspace_id__isnull=True)
+            libraries = qs.order_by("name")
         except Exception as e:
             error = f"Could not connect to Neo4j: {e}"
             logger.error(error)
@@ -47,7 +53,7 @@ def library_list(request):
     return render(
         request,
         "library/library_list.html",
-        {"libraries": libraries, "error": error},
+        {"libraries": libraries, "error": error, "scope": scope},
     )
 
 

mnemosyne/mnemosyne/settings.py

@@ -23,8 +23,9 @@ env = environ.Env(
     DEBUG=(bool, True),
 )
 
-# Read .env file if it exists
+# Read .env file if it exists. Lives at the repo root (one level above the
-environ.Env.read_env(BASE_DIR / ".env")
+# Django project package), shared with the docker-compose stack.
+environ.Env.read_env(BASE_DIR.parent / ".env")
 
 # --- Security ---
 SECRET_KEY = env("SECRET_KEY", default="django-insecure-change-me-in-production")
@@ -267,6 +268,15 @@ SPELUNKER_S3_REGION_NAME = env("SPELUNKER_S3_REGION_NAME", default="us-east-1")
 SPELUNKER_S3_USE_SSL = env.bool("SPELUNKER_S3_USE_SSL", default=False)
 SPELUNKER_S3_VERIFY = env.bool("SPELUNKER_S3_VERIFY", default=True)
 
+# Kairos renders synced mail to text documents in its own bucket.
+KAIROS_S3_ENDPOINT_URL = env("KAIROS_S3_ENDPOINT_URL", default="")
+KAIROS_S3_ACCESS_KEY_ID = env("KAIROS_S3_ACCESS_KEY_ID", default="")
+KAIROS_S3_SECRET_ACCESS_KEY = env("KAIROS_S3_SECRET_ACCESS_KEY", default="")
+KAIROS_S3_BUCKET_NAME = env("KAIROS_S3_BUCKET_NAME", default="kairos")
+KAIROS_S3_REGION_NAME = env("KAIROS_S3_REGION_NAME", default="us-east-1")
+KAIROS_S3_USE_SSL = env.bool("KAIROS_S3_USE_SSL", default=False)
+KAIROS_S3_VERIFY = env.bool("KAIROS_S3_VERIFY", default=True)
+
 # Registry keyed by the ingest `source` field. Unknown/blank sources fall
 # back to "daedalus" for backwards compatibility.
 SOURCE_S3_BUCKETS = {
@@ -288,6 +298,15 @@ SOURCE_S3_BUCKETS = {
         "use_ssl": SPELUNKER_S3_USE_SSL,
         "verify": SPELUNKER_S3_VERIFY,
     },
+    "kairos-mail": {
+        "endpoint_url": KAIROS_S3_ENDPOINT_URL,
+        "access_key_id": KAIROS_S3_ACCESS_KEY_ID,
+        "secret_access_key": KAIROS_S3_SECRET_ACCESS_KEY,
+        "bucket_name": KAIROS_S3_BUCKET_NAME,
+        "region_name": KAIROS_S3_REGION_NAME,
+        "use_ssl": KAIROS_S3_USE_SSL,
+        "verify": KAIROS_S3_VERIFY,
+    },
 }
 
 # --- Celery / RabbitMQ ---

pyproject.toml

@@ -30,6 +30,9 @@ dependencies = [
     "semantic-text-splitter>=0.20,<1.0",
     "tokenizers>=0.20,<1.0",
     "Pillow>=10.0,<12.0",
+    # SVG page splitting — needs recover=True for Write notes that emit
+    # unescaped attribute content and aren't well-formed XML
+    "lxml>=5.3,<7",
     "requests>=2.31,<3.0",
     # Phase 5: MCP Server
     "fastmcp>=2.0,<3.0",