Robert Helewka
224541a4ce
"svg" was in both IMAGE_EXTENSIONS and PYMUPDF_EXTENSIONS, and the image
check ran first, so every SVG reached PIL.Image.open — which cannot decode
vector XML. Ingest raised UnidentifiedImageError, logged "Failed to read
image file_type=svg", and counted documents_parsed_total{status="error"}.
SVG ingest has never worked.
Rasterize to PNG instead, via PyMuPDF (already a dependency). This also
unblocks the vision stage downstream: it sends images to a vision LLM as a
data: URI, which cannot carry image/svg+xml either, so the description and
OCR fields were unreachable for SVG regardless of the parse fix.
Route SVG explicitly before both extension sets. Falling through to
PYMUPDF_EXTENSIONS would parse but not split, and a multi-page Write note
rendered as one document is either blank (no root width/height, so MuPDF
falls back to US-Letter and emits the top-left corner) or an illegible tall
strip (root sized across every stacked page). Splitting per write-page
element is correct for both formats, and since Write never rewrites
existing files both persist indefinitely.
svg_raster is a deliberate twin of daedalus's extraction/svg.py — the repos
share no common package, so the duplication is noted in both docstrings and
fixes belong in both.
lxml is a new dependency: recover=True is needed for the real files that
aren't well-formed XML, and stdlib ElementTree has no equivalent.
74 lines
1.9 KiB
TOML
74 lines
1.9 KiB
TOML
[project]
|
|
name = "mnemosyne"
|
|
version = "0.1.0"
|
|
description = "Content-type-aware, multimodal personal knowledge management system"
|
|
readme = "README.md"
|
|
license = {text = "MIT"}
|
|
requires-python = ">=3.12"
|
|
dependencies = [
|
|
"Django>=5.2,<6.0",
|
|
"djangorestframework>=3.14,<4.0",
|
|
"django-neomodel>=0.1,<1.0",
|
|
"neomodel>=5.3,<6.0",
|
|
"neo4j>=5.0,<6.0",
|
|
"celery>=5.3,<6.0",
|
|
"django-storages[boto3]>=1.14,<2.0",
|
|
"django-environ>=0.11,<1.0",
|
|
"psycopg[binary]>=3.1,<4.0",
|
|
"dj-database-url>=2.1,<3.0",
|
|
"shortuuid>=1.0,<2.0",
|
|
"gunicorn>=21.0,<24.0",
|
|
"cryptography>=41.0,<45.0",
|
|
"flower>=2.0,<3.0",
|
|
"pymemcache>=4.0,<5.0",
|
|
"openai>=1.0,<2.0",
|
|
"django-prometheus>=2.3,<3.0",
|
|
"python-json-logger>=3.0,<4.0",
|
|
# Phase 2: Embedding Pipeline
|
|
"PyMuPDF>=1.24,<2.0",
|
|
"pymupdf4llm>=0.0.17,<1.0",
|
|
"semantic-text-splitter>=0.20,<1.0",
|
|
"tokenizers>=0.20,<1.0",
|
|
"Pillow>=10.0,<12.0",
|
|
# SVG page splitting — needs recover=True for Write notes that emit
|
|
# unescaped attribute content and aren't well-formed XML
|
|
"lxml>=5.3,<7",
|
|
"requests>=2.31,<3.0",
|
|
# Phase 5: MCP Server
|
|
"fastmcp>=2.0,<3.0",
|
|
"uvicorn[standard]>=0.30,<1.0",
|
|
# Phase 6: Per-turn signed JWTs from Daedalus
|
|
"PyJWT>=2.8,<3.0",
|
|
# SSO
|
|
"django-allauth[openid_connect]>=65.0,<66.0",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
dev = [
|
|
"django-debug-toolbar>=4.0,<5.0",
|
|
"docker>=7.0,<8.0",
|
|
]
|
|
test = [
|
|
"pytest>=8.0,<9.0",
|
|
"pytest-django>=4.8,<5.0",
|
|
]
|
|
lint = [
|
|
"ruff>=0.6,<1.0",
|
|
]
|
|
docs = [
|
|
"sphinx>=8.0,<10.0",
|
|
"sphinx-rtd-theme>=3.0,<4.0",
|
|
"sphinx-autodoc-typehints>=2.0,<4.0",
|
|
"sphinx-autobuild>=2024.0,<2026.0",
|
|
"sphinxcontrib-httpdomain>=1.8,<2.0",
|
|
"sphinxcontrib-mermaid>=1.0,<2.0",
|
|
"myst-parser>=4.0,<5.0",
|
|
]
|
|
|
|
[build-system]
|
|
requires = ["setuptools>=68.0"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[tool.setuptools.packages.find]
|
|
where = ["mnemosyne"]
|