diff --git a/mnemosyne/library/services/parsers.py b/mnemosyne/library/services/parsers.py index b74872a..6a44fb6 100644 --- a/mnemosyne/library/services/parsers.py +++ b/mnemosyne/library/services/parsers.py @@ -31,8 +31,11 @@ PYMUPDF_EXTENSIONS = { # Plain text extensions — read directly, no PyMuPDF needed PLAINTEXT_EXTENSIONS = {"txt", "md", "csv", "tsv", "log", "json", "yaml", "yml", "xml"} -# Image extensions — store as Image nodes directly -IMAGE_EXTENSIONS = {"jpg", "jpeg", "png", "gif", "bmp", "tiff", "tif", "webp", "svg"} +# Image extensions — store as Image nodes directly. +# SVG is deliberately absent: it is vector XML that Pillow cannot decode and +# that the vision stage cannot send as a data URI, so it gets rasterized by +# _parse_svg_file instead. +IMAGE_EXTENSIONS = {"jpg", "jpeg", "png", "gif", "bmp", "tiff", "tif", "webp"} # Minimum image dimensions to extract (skip tiny icons/bullets) MIN_IMAGE_WIDTH = 50 @@ -98,6 +101,12 @@ class DocumentParser: if file_type in PLAINTEXT_EXTENSIONS: return self._parse_plaintext(file_path, file_type) + # Checked before PYMUPDF_EXTENSIONS: PyMuPDF can open an SVG, but + # rendering a multi-page Write note as one document yields a blank or + # illegible image (see svg_raster), so it needs page-aware handling. + if file_type == "svg": + return self._parse_svg_file(file_path, file_type) + if file_type in IMAGE_EXTENSIONS: return self._parse_image_file(file_path, file_type) @@ -309,6 +318,61 @@ class DocumentParser: file_type=file_type, ) + def _parse_svg_file(self, file_path: str, file_type: str) -> ParseResult: + """ + Rasterize an SVG into one ExtractedImage per page. + + SVG is vector XML: Pillow cannot decode it and the vision stage cannot + put it in a data URI, so it is rendered to PNG here. Multi-page Write + notes become one image per page so each page reaches vision/OCR at a + legible size. + + :param file_path: Path to the SVG file. + :param file_type: Normalized file extension ("svg"). + :returns: ParseResult with one image per rendered page. + """ + with DOCUMENT_PARSE_DURATION.labels(file_type=file_type).time(): + try: + from library.services.svg_raster import render_svg_pages + + with open(file_path, "rb") as f: + data = f.read() + + pages = render_svg_pages(data) + except Exception as exc: + DOCUMENTS_PARSED_TOTAL.labels(file_type=file_type, status="error").inc() + logger.error("Failed to rasterize SVG file_type=%s: %s", file_type, exc) + raise + + images = [ + ExtractedImage( + data=png, + ext="png", + width=width, + height=height, + source_page=index, + source_index=0, + ) + for index, (png, width, height) in enumerate(pages) + ] + + DOCUMENTS_PARSED_TOTAL.labels(file_type=file_type, status="success").inc() + IMAGES_EXTRACTED_TOTAL.labels(file_type=file_type).inc(len(images)) + + logger.info( + "Parsed SVG file_type=%s pages=%d bytes=%d", + file_type, + len(images), + len(data), + ) + + return ParseResult( + text_blocks=[], + images=images, + metadata={"page_count": len(images)}, + file_type=file_type, + ) + def _parse_image_file(self, file_path: str, file_type: str) -> ParseResult: """ Handle a standalone image file — store as a single ExtractedImage. diff --git a/mnemosyne/library/services/svg_raster.py b/mnemosyne/library/services/svg_raster.py new file mode 100644 index 0000000..ed63cd0 --- /dev/null +++ b/mnemosyne/library/services/svg_raster.py @@ -0,0 +1,263 @@ +"""SVG rasterization for the ingest pipeline. + +The vision stage sends each extracted image to a vision LLM as a ``data:`` URI, +which cannot carry ``image/svg+xml`` — so an SVG has to become raster before it +can be described, OCR'd, or embedded. PyMuPDF (already a dependency for PDF +parsing) renders SVG natively, so this needs no cairo/rsvg system libraries. + +The non-obvious part is page splitting. Handwritten notes from the Write +(Stylus Labs) app are a single SVG document holding one or more +```` children stacked vertically via x/y offsets. Two +root formats exist in the wild and *both* must be split per page: + +- Older files carry no width/height on the root ````. Rendering the + document as-is makes PyMuPDF fall back to US-Letter and emit the top-left + corner only — ruled lines and no handwriting, in a perfectly valid PNG. +- Newer files (Write commit eeab021) do carry root width/height, spanning the + full stacked extent. Rendering those as-is yields one very tall strip; capped + to a sane longest edge, a 5-page note squashes to ~209px wide, well past + illegible. + +Write never rewrites existing files, so the old format is permanent rather than +a migration window. Page geometry lives on the ``write-page`` element and is +identical across both formats, so splitting ignores the root dimensions +entirely and needs no format detection. + +.. note:: + Daedalus carries a twin of this module at + ``backend/daedalus/extraction/svg.py``, which returns base64 for direct + chat attachment. The two are deliberately duplicated rather than shared — + the repos ship no common package — so fixes belong in both. +""" + +from __future__ import annotations + +import copy +import logging +import re + +from lxml import etree + +logger = logging.getLogger(__name__) + +SVG_NS = "http://www.w3.org/2000/svg" +XLINK_NS = "http://www.w3.org/1999/xlink" + +_SVG = f"{{{SVG_NS}}}" +_PAGE_CLASS = "write-page" + +# Write documents render on a grey backdrop; pages themselves are transparent, +# so each rendered page needs an explicit white underlay or strokes land on +# black in the flattened PNG. +_PAGE_BACKGROUND = "#ffffff" + +#: Longest edge of a rendered page, in pixels. Enough for a vision model to +#: read handwriting without spending tokens on unusable resolution. +DEFAULT_TARGET_PX = 1568 + +#: Cap on pages rendered from one document. +DEFAULT_MAX_PAGES = 20 + +# Unquoted attribute value in the root tag, e.g. Write's `width=auto`. Matches +# only bare alphabetic values so numeric or already-quoted attributes are left +# alone. +_UNQUOTED_ATTR = re.compile(rb"(\s[-\w:]+)=([A-Za-z][-\w]*)(?=[\s>])") +_ROOT_TAG = re.compile(rb"]*>") + + +class SvgRenderError(Exception): + """Raised when an SVG cannot be parsed or contains no renderable page.""" + + +def _repair_root_tag(data: bytes) -> bytes: + """Quote unquoted attribute values in the root ```` tag. + + Some Write files emit ``width=auto height=auto``, which is not valid XML. + A malformation *inside* the root tag defeats ``recover=True`` differently + from one in the body: rather than dropping a subtree, libxml2 abandons the + whole document and yields a bare root, so every page becomes invisible. + Quoting the values first recovers the full tree. + """ + match = _ROOT_TAG.search(data) + if not match: + return data + repaired = _UNQUOTED_ATTR.sub(rb'\1="\2"', match.group(0)) + if repaired == match.group(0): + return data + return data[: match.start()] + repaired + data[match.end() :] + + +def _parser() -> etree.XMLParser: + """Build the hardened parser used for all untrusted SVG input. + + ``resolve_entities=False`` blocks XXE — ingest content is untrusted. + ``huge_tree`` is required because handwriting path data runs to megabytes. + ``recover`` salvages the handful of Write files that emit unescaped + attribute content and are not well-formed XML. + """ + return etree.XMLParser( + huge_tree=True, + resolve_entities=False, + no_network=True, + recover=True, + ) + + +def _dimension(element: etree._Element, name: str) -> float | None: + """Read a CSS-pixel dimension attribute, tolerating a ``px`` suffix.""" + raw = element.get(name) + if not raw: + return None + try: + return float(raw.strip().removesuffix("px")) + except ValueError: + return None + + +def _viewbox_size(element: etree._Element) -> tuple[float, float] | None: + """Derive width/height from a viewBox extent.""" + raw = element.get("viewBox") + if not raw: + return None + parts = raw.replace(",", " ").split() + if len(parts) != 4: + return None + try: + width, height = float(parts[2]), float(parts[3]) + except ValueError: + return None + return (width, height) if width > 0 and height > 0 else None + + +def _element_size(element: etree._Element) -> tuple[float, float] | None: + """Resolve an element's rendered size from width/height, else viewBox.""" + width = _dimension(element, "width") + height = _dimension(element, "height") + if width and height: + return width, height + return _viewbox_size(element) + + +def _standalone_page( + page: etree._Element, + defs: etree._Element | None, + width: float, + height: float, +) -> bytes: + """Wrap one ``write-page`` element as its own renderable SVG document. + + The page is repositioned to the origin (its x/y place it within the stacked + parent) and given an explicit viewBox so the renderer has an unambiguous + size. ``defs`` is copied in because pen and ruling definitions live on the + root and are referenced by page content. + """ + root = etree.Element(f"{_SVG}svg", nsmap={None: SVG_NS, "xlink": XLINK_NS}) + root.set("width", str(width)) + root.set("height", str(height)) + root.set("viewBox", f"0 0 {width} {height}") + + background = etree.SubElement(root, f"{_SVG}rect") + background.set("width", "100%") + background.set("height", "100%") + background.set("fill", _PAGE_BACKGROUND) + + if defs is not None: + root.append(copy.deepcopy(defs)) + + element = copy.deepcopy(page) + for positional in ("x", "y"): + element.attrib.pop(positional, None) + element.set("viewBox", f"0 0 {width} {height}") + root.append(element) + + return etree.tostring(root) + + +def split_svg_pages(data: bytes) -> list[bytes]: + """Split an SVG into one standalone document per renderable page. + + Write multi-page notes yield one document per ``write-page`` child. Any + other SVG (a diagram, a logo) yields a single document — the input itself, + which the renderer sizes from its own width/height or viewBox. + + :param data: Raw SVG bytes. + :returns: One or more standalone SVG documents, in page order. + :raises SvgRenderError: If the SVG cannot be parsed or has no usable size. + """ + try: + root = etree.fromstring(_repair_root_tag(data), _parser()) + except etree.XMLSyntaxError as exc: + raise SvgRenderError(f"Could not parse SVG: {exc}") from exc + + if root is None: + raise SvgRenderError("Could not parse SVG: no root element.") + + defs = root.find(f"{_SVG}defs") + + pages: list[bytes] = [] + for element in root.iter(f"{_SVG}svg"): + if element is root: + continue + if _PAGE_CLASS not in (element.get("class") or "").split(): + continue + size = _element_size(element) + if not size: + continue + pages.append(_standalone_page(element, defs, *size)) + + if pages: + return pages + + # Generic SVG: render as-is. It must still be sizeable, or the renderer + # would silently substitute a default page box. + if not _element_size(root): + raise SvgRenderError( + "SVG has no width/height or viewBox, so its size is undefined." + ) + return [data] + + +def render_svg_pages( + data: bytes, + max_pages: int = DEFAULT_MAX_PAGES, + target_px: int = DEFAULT_TARGET_PX, +) -> list[tuple[bytes, int, int]]: + """Rasterize an SVG to one PNG per page. + + :param data: Raw SVG bytes. + :param max_pages: Cap on pages rendered. + :param target_px: Longest edge of each rendered page, in pixels. + :returns: One ``(png_bytes, width, height)`` tuple per page, in page order. + :raises SvgRenderError: If nothing could be rendered. + """ + import fitz + + pages = split_svg_pages(data) + + rendered: list[tuple[bytes, int, int]] = [] + for index, page_svg in enumerate(pages[:max_pages]): + try: + with fitz.open(stream=page_svg, filetype="svg") as document: + page = document[0] + longest = max(page.rect.width, page.rect.height) + zoom = target_px / longest if longest else 1.0 + pixmap = page.get_pixmap(matrix=fitz.Matrix(zoom, zoom)) + rendered.append( + (pixmap.tobytes("png"), pixmap.width, pixmap.height) + ) + except Exception as exc: + raise SvgRenderError( + f"Could not render SVG page {index + 1}: {exc}" + ) from exc + + if not rendered: + raise SvgRenderError("No renderable pages in SVG.") + + logger.info( + "Rasterized SVG pages_rendered=%d total_pages=%d target_px=%d", + len(rendered), + len(pages), + target_px, + ) + + return rendered diff --git a/mnemosyne/library/tests/test_svg_raster.py b/mnemosyne/library/tests/test_svg_raster.py new file mode 100644 index 0000000..58a33a3 --- /dev/null +++ b/mnemosyne/library/tests/test_svg_raster.py @@ -0,0 +1,195 @@ +""" +Tests for SVG rasterization in the ingest pipeline. + +SVG is vector XML: Pillow cannot decode it and the vision stage cannot send it +as a data URI, so it is rendered to PNG at parse time. The cases that matter +are the ones that fail *silently* — Write notes stack their pages inside one +document, and rendering that document whole yields a plausible-looking PNG that +is either blank or an illegible tall strip. +""" + +import io +import os +import tempfile + +from django.test import TestCase +from PIL import Image + +from library.services.parsers import ( + IMAGE_EXTENSIONS, + DocumentParser, +) +from library.services.svg_raster import ( + SvgRenderError, + render_svg_pages, + split_svg_pages, +) + +# Geometry copied from a real Write note; the page element is identical across +# both root formats, which is what makes splitting format-agnostic. +PAGE_WIDTH = 1094 +PAGE_HEIGHT = 1654 +PAGE_PITCH = 1674 # page height + inter-page gap + + +def write_document(pages: int, root_size: bool = False) -> bytes: + """ + Build a Write-style document. + + :param pages: Number of stacked pages. + :param root_size: Emit width/height on the root . False models + pre-eeab021 files (the permanent majority), True models newer saves. + """ + root_attrs = "" + if root_size: + total = 10 + pages * PAGE_PITCH + root_attrs = f' width="{PAGE_WIDTH + 20}" height="{total}"' + + body = "".join( + f'' + f'' + for i in range(pages) + ) + return ( + f'' + f'' + f'' + f"{body}" + ).encode() + + +GENERIC_SVG = ( + b'' + b'' +) + + +def ink_fraction(png: bytes) -> float: + """Fraction of non-white pixels — the blank-render canary.""" + grey = Image.open(io.BytesIO(png)).convert("L") + histogram = grey.histogram() + return sum(histogram[:200]) / sum(histogram) + + +class SvgSplitTests(TestCase): + """Splitting a Write document into per-page SVGs.""" + + def test_splits_one_document_per_page(self): + for pages in (1, 2, 5): + for root_size in (False, True): + with self.subTest(pages=pages, root_size=root_size): + document = write_document(pages, root_size=root_size) + self.assertEqual(len(split_svg_pages(document)), pages) + + def test_both_root_formats_split_identically(self): + """Root width/height (Write eeab021) must not change the outcome. + + Existing notes are never rewritten, so both formats persist + indefinitely and have to render the same. + """ + without = split_svg_pages(write_document(3, root_size=False)) + with_size = split_svg_pages(write_document(3, root_size=True)) + self.assertEqual(len(without), len(with_size)) + self.assertEqual(len(without), 3) + + def test_generic_svg_is_a_single_page(self): + self.assertEqual(len(split_svg_pages(GENERIC_SVG)), 1) + + def test_unsized_svg_is_rejected_rather_than_guessed(self): + unsized = b'' + with self.assertRaises(SvgRenderError): + split_svg_pages(unsized) + + def test_script_inside_defs_does_not_desync_the_split(self): + """Regression: a non-greedy ... regex mis-parses these.""" + document = write_document(2).replace( + b'', + b'', + ) + self.assertEqual(len(split_svg_pages(document)), 2) + + def test_unquoted_root_attributes_are_repaired(self): + """Write can emit ``width=auto``, which is not valid XML. + + A malformation inside the root tag defeats recovery differently from + one in the body: libxml2 abandons the whole document and returns a + bare root, so every page silently disappears. + """ + document = write_document(2).replace( + b'=0.20,<1.0", "tokenizers>=0.20,<1.0", "Pillow>=10.0,<12.0", + # SVG page splitting — needs recover=True for Write notes that emit + # unescaped attribute content and aren't well-formed XML + "lxml>=5.3,<7", "requests>=2.31,<3.0", # Phase 5: MCP Server "fastmcp>=2.0,<3.0",