Compare commits
16 Commits
4dde063299
...
feat/email
| Author | SHA1 | Date | |
|---|---|---|---|
| 9f20110f56 | |||
| d6b541636a | |||
| d01afd6203 | |||
| 840b9435a3 | |||
| 6120e9cd1f | |||
| 31a98b4f3a | |||
| 3394726ca1 | |||
| 03e3155bd6 | |||
| 929a3c8c3c | |||
| 2af72d6e82 | |||
| 70b1fc510b | |||
| 46ca2a934d | |||
| dd06f923cd | |||
| 539d9b6c34 | |||
| 142e9675b5 | |||
| a90c6e7479 |
13
.env.example
13
.env.example
@@ -91,6 +91,19 @@ SPELUNKER_S3_REGION_NAME=us-east-1
|
||||
SPELUNKER_S3_USE_SSL=True
|
||||
SPELUNKER_S3_VERIFY=True
|
||||
|
||||
# --- Kairos S3 (cross-bucket reads for ingest, source="kairos-mail") ---
|
||||
# Consumed by: worker only
|
||||
# Kairos renders synced mail to text documents in its own bucket and posts
|
||||
# ingest requests with source="kairos-mail". These creds should be scoped
|
||||
# read-only to the Kairos bucket in your secret manager.
|
||||
KAIROS_S3_ENDPOINT_URL=https://nyx.helu.ca:8555
|
||||
KAIROS_S3_ACCESS_KEY_ID=
|
||||
KAIROS_S3_SECRET_ACCESS_KEY=
|
||||
KAIROS_S3_BUCKET_NAME=kairos
|
||||
KAIROS_S3_REGION_NAME=us-east-1
|
||||
KAIROS_S3_USE_SSL=True
|
||||
KAIROS_S3_VERIFY=True
|
||||
|
||||
# --- Celery / RabbitMQ (Oberon) ---------------------------------------------
|
||||
# Consumed by: app (producer), worker (consumer). NOT mcp.
|
||||
# Remember to percent-encode any password characters that have meaning in a
|
||||
|
||||
@@ -3,8 +3,6 @@ name: CVE Scan & Docker Build
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
REGISTRY: git.helu.ca
|
||||
@@ -75,8 +73,6 @@ jobs:
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=sha,prefix=
|
||||
type=raw,value=latest,enable=${{ gitea.ref == 'refs/heads/main' }}
|
||||
|
||||
|
||||
@@ -346,6 +346,13 @@ services:
|
||||
- SPELUNKER_S3_REGION_NAME=${SPELUNKER_S3_REGION_NAME}
|
||||
- SPELUNKER_S3_USE_SSL=${SPELUNKER_S3_USE_SSL}
|
||||
- SPELUNKER_S3_VERIFY=${SPELUNKER_S3_VERIFY}
|
||||
- KAIROS_S3_ENDPOINT_URL=${KAIROS_S3_ENDPOINT_URL}
|
||||
- KAIROS_S3_ACCESS_KEY_ID=${KAIROS_S3_ACCESS_KEY_ID}
|
||||
- KAIROS_S3_SECRET_ACCESS_KEY=${KAIROS_S3_SECRET_ACCESS_KEY}
|
||||
- KAIROS_S3_BUCKET_NAME=${KAIROS_S3_BUCKET_NAME}
|
||||
- KAIROS_S3_REGION_NAME=${KAIROS_S3_REGION_NAME}
|
||||
- KAIROS_S3_USE_SSL=${KAIROS_S3_USE_SSL}
|
||||
- KAIROS_S3_VERIFY=${KAIROS_S3_VERIFY}
|
||||
# Celery / RabbitMQ
|
||||
- CELERY_BROKER_URL=${CELERY_BROKER_URL}
|
||||
- CELERY_RESULT_BACKEND=${CELERY_RESULT_BACKEND}
|
||||
@@ -371,7 +378,12 @@ services:
|
||||
volumes:
|
||||
- media:/mnt/media
|
||||
healthcheck:
|
||||
test: ["CMD", "celery", "-A", "mnemosyne", "inspect", "ping", "-d", "celery@$$HOSTNAME"]
|
||||
# No -d destination: exec-form CMD has no shell, so $$HOSTNAME never
|
||||
# expanded and the literal "celery@$HOSTNAME" matched no node → every
|
||||
# check failed. There's one worker per container, so an unfiltered ping
|
||||
# (any node replies = healthy) is correct. -t gives the reply room to
|
||||
# round-trip through the broker on Oberon (~450ms observed) under jitter.
|
||||
test: ["CMD", "celery", "-A", "mnemosyne", "inspect", "ping", "-t", "8"]
|
||||
interval: 60s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
@@ -15,6 +15,7 @@ LIBRARY_TYPE_CHOICES = [
|
||||
"film",
|
||||
"art",
|
||||
"journal",
|
||||
"email",
|
||||
"business",
|
||||
"finance",
|
||||
]
|
||||
|
||||
@@ -17,12 +17,14 @@ across users.
|
||||
import logging
|
||||
|
||||
from neomodel import db
|
||||
from neomodel.exceptions import UniqueProperty
|
||||
from rest_framework import status
|
||||
from rest_framework.decorators import api_view, permission_classes
|
||||
from rest_framework.permissions import IsAuthenticated
|
||||
from rest_framework.response import Response
|
||||
|
||||
from library.content_types import get_library_type_config
|
||||
from library.services.library_delete import delete_library_cascade
|
||||
|
||||
from .serializers import WorkspaceCreateSerializer, WorkspaceStatusSerializer
|
||||
|
||||
@@ -84,7 +86,10 @@ def workspace_create(request):
|
||||
data["workspace_id"], request.user.username,
|
||||
)
|
||||
return Response(
|
||||
{"detail": "Workspace id is already in use."},
|
||||
{
|
||||
"detail": "Workspace id is already in use.",
|
||||
"code": "owner_conflict",
|
||||
},
|
||||
status=status.HTTP_409_CONFLICT,
|
||||
)
|
||||
if existing.library_type != data["library_type"]:
|
||||
@@ -94,7 +99,8 @@ def workspace_create(request):
|
||||
"library_type is immutable for an existing workspace "
|
||||
f"(have '{existing.library_type}', "
|
||||
f"got '{data['library_type']}')."
|
||||
)
|
||||
),
|
||||
"code": "library_type_immutable",
|
||||
},
|
||||
status=status.HTTP_409_CONFLICT,
|
||||
)
|
||||
@@ -119,7 +125,29 @@ def workspace_create(request):
|
||||
reranker_instruction=defaults["reranker_instruction"],
|
||||
llm_context_prompt=defaults["llm_context_prompt"],
|
||||
)
|
||||
lib.save()
|
||||
try:
|
||||
lib.save()
|
||||
except UniqueProperty:
|
||||
# Library.name is globally unique. A name collision here almost always
|
||||
# means an orphaned Library survived a failed Daedalus workspace delete
|
||||
# (the old node kept the name), and the recreate under a new
|
||||
# workspace_id now clashes. Surface a clean 409 instead of a 500 so
|
||||
# Daedalus can record + report it; the operator clears the orphan
|
||||
# (admin delete) or renames the workspace.
|
||||
logger.warning(
|
||||
"workspace_create name_conflict workspace_id=%s name=%s",
|
||||
data["workspace_id"], data["name"],
|
||||
)
|
||||
return Response(
|
||||
{
|
||||
"detail": (
|
||||
f"A library named '{data['name']}' already exists in "
|
||||
"Mnemosyne."
|
||||
),
|
||||
"code": "name_conflict",
|
||||
},
|
||||
status=status.HTTP_409_CONFLICT,
|
||||
)
|
||||
logger.info(
|
||||
"Workspace created workspace_id=%s library_uid=%s library_type=%s",
|
||||
data["workspace_id"], lib.uid, lib.library_type,
|
||||
@@ -165,74 +193,15 @@ def workspace_detail_or_delete(request, workspace_id):
|
||||
if lib is None:
|
||||
return Response(status=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
library_uid = lib.uid
|
||||
library_name = lib.name
|
||||
|
||||
# Step 1-4: delete chunks, items, collections, then the library itself.
|
||||
# We collect Item s3_keys first so the caller can clean up S3
|
||||
# asynchronously (a future enhancement — for now, the keys are logged).
|
||||
s3_rows, _ = db.cypher_query(
|
||||
"MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(:Collection)"
|
||||
"-[:CONTAINS]->(i:Item) RETURN i.uid, i.s3_key",
|
||||
{"wsid": workspace_id},
|
||||
)
|
||||
item_s3_keys = [(r[0], r[1]) for r in s3_rows if r[1]]
|
||||
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(:Collection)
|
||||
-[:CONTAINS]->(i:Item)-[:HAS_CHUNK]->(c:Chunk)
|
||||
DETACH DELETE c
|
||||
""",
|
||||
{"wsid": workspace_id},
|
||||
)
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(:Collection)
|
||||
-[:CONTAINS]->(i:Item)-[:HAS_IMAGE]->(img:Image)
|
||||
OPTIONAL MATCH (img)-[:HAS_EMBEDDING]->(emb:ImageEmbedding)
|
||||
DETACH DELETE img, emb
|
||||
""",
|
||||
{"wsid": workspace_id},
|
||||
)
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(:Collection)
|
||||
-[:CONTAINS]->(i:Item)
|
||||
DETACH DELETE i
|
||||
""",
|
||||
{"wsid": workspace_id},
|
||||
)
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(col:Collection)
|
||||
DETACH DELETE col
|
||||
""",
|
||||
{"wsid": workspace_id},
|
||||
)
|
||||
db.cypher_query(
|
||||
"MATCH (l:Library {workspace_id: $wsid}) DETACH DELETE l",
|
||||
{"wsid": workspace_id},
|
||||
)
|
||||
|
||||
# Step 5: orphan Concept garbage collection.
|
||||
orphan_result, _ = db.cypher_query(
|
||||
"""
|
||||
MATCH (con:Concept)
|
||||
WHERE NOT (con)<-[:REFERENCES]-() AND NOT (con)<-[:MENTIONS]-()
|
||||
AND NOT (con)<-[:DEPICTS]-()
|
||||
WITH con
|
||||
DETACH DELETE con
|
||||
RETURN count(con) AS deleted
|
||||
"""
|
||||
)
|
||||
orphans_deleted = orphan_result[0][0] if orphan_result else 0
|
||||
# Delete the Library and everything reachable + unique to it, plus
|
||||
# orphan-Concept GC. Shared with the admin/HTML delete path.
|
||||
result = delete_library_cascade(lib)
|
||||
|
||||
logger.info(
|
||||
"Workspace deleted workspace_id=%s library_uid=%s name=%s "
|
||||
"items=%d orphans_deleted=%d",
|
||||
workspace_id, library_uid, library_name,
|
||||
len(item_s3_keys), orphans_deleted,
|
||||
workspace_id, result["library_uid"], result["name"],
|
||||
result["item_count"], result["orphans_deleted"],
|
||||
)
|
||||
|
||||
return Response(status=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
@@ -88,6 +88,29 @@ def _should_skip_probe() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _is_web_process() -> bool:
|
||||
"""
|
||||
True when running inside the web (gunicorn / runserver) process.
|
||||
|
||||
The reachability collector must only register here: ``/metrics`` is served
|
||||
by the web process, and registering in the Celery worker would both probe
|
||||
the GPU endpoints from a process whose metrics nobody scrapes and risk
|
||||
duplicate registration. Celery launches via ``celery`` argv; management
|
||||
commands are excluded above.
|
||||
"""
|
||||
argv0 = sys.argv[0]
|
||||
if "celery" in argv0 or (len(sys.argv) >= 2 and sys.argv[1] == "celery"):
|
||||
return False
|
||||
if "pytest" in argv0 or "PYTEST_CURRENT_TEST" in os.environ:
|
||||
return False
|
||||
# gunicorn (prod) or runserver (dev).
|
||||
if "gunicorn" in argv0:
|
||||
return True
|
||||
if len(sys.argv) >= 2 and sys.argv[1] == "runserver":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _run_startup_probe():
|
||||
"""
|
||||
Emit ERROR/WARNING logs if the stack is misconfigured for search.
|
||||
@@ -199,4 +222,7 @@ class LibraryConfig(AppConfig):
|
||||
verbose_name = "Library"
|
||||
|
||||
def ready(self):
|
||||
pass
|
||||
if _is_web_process():
|
||||
from library.health_collector import register
|
||||
|
||||
register()
|
||||
|
||||
@@ -241,6 +241,38 @@ LIBRARY_TYPE_DEFAULTS = {
|
||||
"4) The commercial purpose — positioning, pricing, capability demonstration."
|
||||
),
|
||||
},
|
||||
"email": {
|
||||
"chunking_config": {
|
||||
"strategy": "entry_level",
|
||||
"chunk_size": 512,
|
||||
"chunk_overlap": 32,
|
||||
"respect_boundaries": ["message", "quote", "paragraph"],
|
||||
},
|
||||
"embedding_instruction": (
|
||||
"Represent this email message for retrieval. "
|
||||
"Focus on the sender, recipients, subject, dates, requests and "
|
||||
"commitments made, and the people, organizations, and events discussed."
|
||||
),
|
||||
"reranker_instruction": (
|
||||
"Re-rank email messages based on relevance to the query. "
|
||||
"Prioritize messages matching the correspondents, subject matter, "
|
||||
"time period, and any specific commitments or requests mentioned."
|
||||
),
|
||||
"llm_context_prompt": (
|
||||
"The following excerpts are from personal email correspondence. "
|
||||
"This is private content — answer with discretion. Attribute "
|
||||
"statements to their senders, note dates, and distinguish what was "
|
||||
"asked from what was agreed. Quoted text below a reply is earlier "
|
||||
"context, not the sender's own words."
|
||||
),
|
||||
"vision_prompt": (
|
||||
"Analyze this image from an email message. Identify:\n"
|
||||
"1) Image type (photograph, screenshot, scanned document, chart, signature graphic).\n"
|
||||
"2) What it depicts — people, places, documents, data.\n"
|
||||
"3) Any visible text, dates, or figures.\n"
|
||||
"4) Its role in the message — attachment content, inline illustration, or boilerplate."
|
||||
),
|
||||
},
|
||||
"finance": {
|
||||
"chunking_config": {
|
||||
"strategy": "section_aware",
|
||||
@@ -282,7 +314,7 @@ def get_library_type_config(library_type):
|
||||
|
||||
Args:
|
||||
library_type: One of 'fiction', 'nonfiction', 'technical', 'music',
|
||||
'film', 'art', 'journal', 'business', 'finance'
|
||||
'film', 'art', 'journal', 'email', 'business', 'finance'
|
||||
|
||||
Returns:
|
||||
dict with keys: chunking_config, embedding_instruction,
|
||||
|
||||
99
mnemosyne/library/health_collector.py
Normal file
99
mnemosyne/library/health_collector.py
Normal file
@@ -0,0 +1,99 @@
|
||||
"""
|
||||
Scrape-time Prometheus collector for system-default model reachability.
|
||||
|
||||
The ingest-pipeline counters in ``library/metrics.py`` live in the Celery
|
||||
worker process and only move during an active ingest, so they cannot signal
|
||||
"models down" on an idle queue. This collector runs in the **web** process
|
||||
(where ``/metrics`` is served by ``django_prometheus``) and probes the four
|
||||
system-default models at scrape time, emitting an up/down gauge that is
|
||||
present regardless of queue activity.
|
||||
|
||||
Probe results are cached for a short TTL so rapid scrapes — or multiple
|
||||
gunicorn workers each scraped in turn — cannot hammer the GPU endpoints.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
from prometheus_client.core import GaugeMetricFamily
|
||||
|
||||
from library.services.model_health import probe_system_models
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Cache probe results so repeated scrapes don't re-probe the router. The
|
||||
# value is comfortably above a 15s scrape_interval but bounded so a recovered
|
||||
# model shows green within a minute.
|
||||
_CACHE_TTL_SECONDS = 55
|
||||
|
||||
_lock = threading.Lock()
|
||||
_cache: dict = {"ts": 0.0, "results": None}
|
||||
|
||||
|
||||
def _cached_probe() -> list[dict]:
|
||||
"""Return probe results, re-probing only when the cache has expired."""
|
||||
now = time.monotonic()
|
||||
with _lock:
|
||||
if _cache["results"] is not None and (now - _cache["ts"]) < _CACHE_TTL_SECONDS:
|
||||
return _cache["results"]
|
||||
try:
|
||||
results = probe_system_models()
|
||||
except Exception as exc: # never let a probe failure break /metrics
|
||||
logger.warning("Model health probe failed during scrape: %s", exc)
|
||||
# Serve the stale cache if we have one; otherwise report nothing.
|
||||
return _cache["results"] or []
|
||||
_cache["ts"] = now
|
||||
_cache["results"] = results
|
||||
return results
|
||||
|
||||
|
||||
class SystemModelHealthCollector:
|
||||
"""prometheus_client custom collector for system-default model health."""
|
||||
|
||||
def collect(self):
|
||||
results = _cached_probe()
|
||||
|
||||
up = GaugeMetricFamily(
|
||||
"mnemosyne_system_default_model_up",
|
||||
"System-default model endpoint reachable (1) or not (0)",
|
||||
labels=["role", "model", "api"],
|
||||
)
|
||||
configured = GaugeMetricFamily(
|
||||
"mnemosyne_system_default_model_configured",
|
||||
"A system-default model is configured for this role (1) or not (0)",
|
||||
labels=["role"],
|
||||
)
|
||||
latency = GaugeMetricFamily(
|
||||
"mnemosyne_system_default_model_probe_latency_seconds",
|
||||
"Latency of the last reachability probe for this role",
|
||||
labels=["role"],
|
||||
)
|
||||
|
||||
for r in results:
|
||||
role = r["role"]
|
||||
configured.add_metric([role], 1 if r["configured"] else 0)
|
||||
if not r["configured"]:
|
||||
continue
|
||||
up.add_metric(
|
||||
[role, r["model_name"] or "", r["api_name"] or ""],
|
||||
1 if r["ok"] else 0,
|
||||
)
|
||||
if r["latency_ms"] is not None:
|
||||
latency.add_metric([role], r["latency_ms"] / 1000.0)
|
||||
|
||||
yield configured
|
||||
yield up
|
||||
yield latency
|
||||
|
||||
|
||||
def register():
|
||||
"""Register the collector against the default registry (idempotent)."""
|
||||
from prometheus_client import REGISTRY
|
||||
|
||||
# Guard against duplicate registration (autoreload, repeated ready()).
|
||||
for collector in list(getattr(REGISTRY, "_collector_to_names", {})):
|
||||
if isinstance(collector, SystemModelHealthCollector):
|
||||
return
|
||||
REGISTRY.register(SystemModelHealthCollector())
|
||||
logger.info("Registered SystemModelHealthCollector on Prometheus default registry")
|
||||
@@ -77,6 +77,7 @@ class Library(StructuredNode):
|
||||
"film": "Film",
|
||||
"art": "Art",
|
||||
"journal": "Journal",
|
||||
"email": "Email",
|
||||
"business": "Business",
|
||||
"finance": "Finance",
|
||||
},
|
||||
|
||||
108
mnemosyne/library/services/library_delete.py
Normal file
108
mnemosyne/library/services/library_delete.py
Normal file
@@ -0,0 +1,108 @@
|
||||
"""
|
||||
Shared Library deletion cascade.
|
||||
|
||||
Deletes a Library node and everything reachable AND unique to it
|
||||
(Collections, Items, Chunks, Images + ImageEmbeddings), then garbage-collects
|
||||
Concepts that are no longer referenced by any other Library.
|
||||
|
||||
Keyed on the Library ``uid`` so it works for *both* global libraries
|
||||
(``workspace_id`` is null) and workspace-scoped libraries. This is the single
|
||||
source of truth used by:
|
||||
|
||||
* the Daedalus integration API (``DELETE /library/api/workspaces/{id}/``), and
|
||||
* the admin/HTML delete view (``library_delete``).
|
||||
|
||||
Concept-safe: orphan-only Concept GC happens at the end. Concepts still
|
||||
referenced by another library (workspace or global) are preserved.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from neomodel import db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def delete_library_cascade(lib) -> dict:
|
||||
"""Delete ``lib`` and all content reachable and unique to it.
|
||||
|
||||
:param lib: A ``library.models.Library`` node instance.
|
||||
:returns: Dict with ``library_uid``, ``name``, ``item_count``,
|
||||
``item_s3_keys`` (list of ``(uid, s3_key)`` for async S3 cleanup),
|
||||
and ``orphans_deleted`` (Concept GC count).
|
||||
"""
|
||||
library_uid = lib.uid
|
||||
library_name = lib.name
|
||||
|
||||
# Collect Item s3_keys first so the caller can clean up S3 asynchronously
|
||||
# (a future enhancement — for now, the keys are returned/logged).
|
||||
s3_rows, _ = db.cypher_query(
|
||||
"MATCH (l:Library {uid: $uid})-[:CONTAINS]->(:Collection)"
|
||||
"-[:CONTAINS]->(i:Item) RETURN i.uid, i.s3_key",
|
||||
{"uid": library_uid},
|
||||
)
|
||||
item_s3_keys = [(r[0], r[1]) for r in s3_rows if r[1]]
|
||||
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {uid: $uid})-[:CONTAINS]->(:Collection)
|
||||
-[:CONTAINS]->(i:Item)-[:HAS_CHUNK]->(c:Chunk)
|
||||
DETACH DELETE c
|
||||
""",
|
||||
{"uid": library_uid},
|
||||
)
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {uid: $uid})-[:CONTAINS]->(:Collection)
|
||||
-[:CONTAINS]->(i:Item)-[:HAS_IMAGE]->(img:Image)
|
||||
OPTIONAL MATCH (img)-[:HAS_EMBEDDING]->(emb:ImageEmbedding)
|
||||
DETACH DELETE img, emb
|
||||
""",
|
||||
{"uid": library_uid},
|
||||
)
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {uid: $uid})-[:CONTAINS]->(:Collection)
|
||||
-[:CONTAINS]->(i:Item)
|
||||
DETACH DELETE i
|
||||
""",
|
||||
{"uid": library_uid},
|
||||
)
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {uid: $uid})-[:CONTAINS]->(col:Collection)
|
||||
DETACH DELETE col
|
||||
""",
|
||||
{"uid": library_uid},
|
||||
)
|
||||
db.cypher_query(
|
||||
"MATCH (l:Library {uid: $uid}) DETACH DELETE l",
|
||||
{"uid": library_uid},
|
||||
)
|
||||
|
||||
# Orphan Concept garbage collection: drop Concepts no longer referenced
|
||||
# by any Item (REFERENCES/MENTIONS) or Image (DEPICTS).
|
||||
orphan_result, _ = db.cypher_query(
|
||||
"""
|
||||
MATCH (con:Concept)
|
||||
WHERE NOT (con)<-[:REFERENCES]-() AND NOT (con)<-[:MENTIONS]-()
|
||||
AND NOT (con)<-[:DEPICTS]-()
|
||||
WITH con
|
||||
DETACH DELETE con
|
||||
RETURN count(con) AS deleted
|
||||
"""
|
||||
)
|
||||
orphans_deleted = orphan_result[0][0] if orphan_result else 0
|
||||
|
||||
logger.info(
|
||||
"Library cascade-deleted library_uid=%s name=%s items=%d orphans_deleted=%d",
|
||||
library_uid, library_name, len(item_s3_keys), orphans_deleted,
|
||||
)
|
||||
|
||||
return {
|
||||
"library_uid": library_uid,
|
||||
"name": library_name,
|
||||
"item_count": len(item_s3_keys),
|
||||
"item_s3_keys": item_s3_keys,
|
||||
"orphans_deleted": orphans_deleted,
|
||||
}
|
||||
119
mnemosyne/library/services/model_health.py
Normal file
119
mnemosyne/library/services/model_health.py
Normal file
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
System-default model reachability probes.
|
||||
|
||||
Provides a cheap, bounded liveness check for the four system-default models
|
||||
(embedding, chat, vision, reranker) so the embedding dashboard and the
|
||||
scrape-time Prometheus collector can surface "model not responding" without
|
||||
running an ingest.
|
||||
|
||||
The probe deliberately hits ``GET {base_url}/models`` as its primary check:
|
||||
on an OpenAI-compatible router (e.g. the llama-router) this answers instantly
|
||||
without loading a model, so repeated probes never burn GPU time. This mirrors
|
||||
the GPU-avoidance principle in ``mcp_server/tools/health.py``.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# api_type values whose endpoints expose an OpenAI-compatible ``/models`` list.
|
||||
_OPENAI_COMPATIBLE = {"openai", "azure", "ollama", "llama-cpp", "vllm"}
|
||||
|
||||
# (role, getter method name) pairs — order is the dashboard/metrics order.
|
||||
ROLE_GETTERS = [
|
||||
("embedding", "get_system_embedding_model"),
|
||||
("chat", "get_system_chat_model"),
|
||||
("vision", "get_system_vision_model"),
|
||||
("reranker", "get_system_reranker_model"),
|
||||
]
|
||||
|
||||
|
||||
def probe_api(api, timeout: int = 5) -> tuple[bool, str]:
|
||||
"""Check whether an ``LLMApi`` endpoint is responding.
|
||||
|
||||
Args:
|
||||
api: ``LLMApi`` instance (provides base_url, api_key, api_type).
|
||||
timeout: Per-request timeout in seconds.
|
||||
|
||||
Returns:
|
||||
``(ok, detail)`` — ok is True if the endpoint answered acceptably;
|
||||
detail is a short human-readable status (HTTP code, error, or "ok").
|
||||
"""
|
||||
base_url = api.base_url.rstrip("/")
|
||||
headers = {}
|
||||
if api.api_key:
|
||||
headers["Authorization"] = f"Bearer {api.api_key}"
|
||||
|
||||
if api.api_type not in _OPENAI_COMPATIBLE:
|
||||
# bedrock / anthropic have no equivalent cheap unauthenticated list;
|
||||
# treat a reachable host as the liveness signal via a HEAD on base_url.
|
||||
try:
|
||||
resp = requests.head(base_url, headers=headers, timeout=timeout)
|
||||
return True, f"reachable (HTTP {resp.status_code})"
|
||||
except requests.RequestException as exc:
|
||||
return False, type(exc).__name__
|
||||
|
||||
url = f"{base_url}/models"
|
||||
try:
|
||||
resp = requests.get(url, headers=headers, timeout=timeout)
|
||||
except requests.Timeout:
|
||||
return False, f"timeout after {timeout}s"
|
||||
except requests.RequestException as exc:
|
||||
return False, type(exc).__name__
|
||||
|
||||
if resp.status_code == 200:
|
||||
return True, "ok"
|
||||
return False, f"HTTP {resp.status_code}"
|
||||
|
||||
|
||||
def probe_system_models(timeout: int = 5) -> list[dict]:
|
||||
"""Probe all four system-default models for reachability.
|
||||
|
||||
Returns:
|
||||
One dict per role with keys: ``role``, ``configured``, ``model_name``,
|
||||
``api_name``, ``base_url``, ``ok``, ``detail``, ``latency_ms``.
|
||||
For an unconfigured role, ``configured`` is False and the probe is
|
||||
skipped (``ok`` is None).
|
||||
"""
|
||||
from llm_manager.models import LLMModel
|
||||
|
||||
results: list[dict] = []
|
||||
for role, getter_name in ROLE_GETTERS:
|
||||
model = getattr(LLMModel, getter_name)()
|
||||
if model is None:
|
||||
results.append(
|
||||
{
|
||||
"role": role,
|
||||
"configured": False,
|
||||
"model_name": None,
|
||||
"api_name": None,
|
||||
"base_url": None,
|
||||
"ok": None,
|
||||
"detail": "not configured",
|
||||
"latency_ms": None,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
api = model.api
|
||||
start = time.monotonic()
|
||||
ok, detail = probe_api(api, timeout=timeout)
|
||||
latency_ms = round((time.monotonic() - start) * 1000, 1)
|
||||
results.append(
|
||||
{
|
||||
"role": role,
|
||||
"configured": True,
|
||||
"model_name": model.name,
|
||||
"api_name": api.name,
|
||||
"base_url": api.base_url,
|
||||
"ok": ok,
|
||||
"detail": detail,
|
||||
"latency_ms": latency_ms,
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
18
mnemosyne/library/templates/library/_model_health_badge.html
Normal file
18
mnemosyne/library/templates/library/_model_health_badge.html
Normal file
@@ -0,0 +1,18 @@
|
||||
{% comment %}
|
||||
Reachability badge for a system-default model. Expects `h` = one entry from
|
||||
the `model_health` dict (keys: configured, ok, detail, latency_ms). Renders
|
||||
nothing when the role is absent from model_health (probe failed entirely).
|
||||
Text-only badges to match the existing dashboard palette (no emoji per house
|
||||
HTML rule).
|
||||
{% endcomment %}
|
||||
{% if h %}
|
||||
{% if not h.configured %}
|
||||
<span class="badge badge-ghost badge-sm ml-2" title="No system-default model set for this role">NOT CONFIGURED</span>
|
||||
{% elif h.ok %}
|
||||
<span class="badge badge-success badge-sm ml-2" title="{{ h.detail }}">REACHABLE</span>
|
||||
{% if h.latency_ms is not None %}<span class="text-xs opacity-50 ml-1">{{ h.latency_ms }} ms</span>{% endif %}
|
||||
{% else %}
|
||||
<span class="badge badge-error badge-sm ml-2" title="Probe detail: {{ h.detail }}">NOT RESPONDING</span>
|
||||
<span class="text-xs opacity-60 ml-1">{{ h.detail }}</span>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
@@ -28,6 +28,7 @@
|
||||
{% if system_embedding_model.supports_multimodal %}
|
||||
<span class="badge badge-accent badge-sm ml-1">Multimodal</span>
|
||||
{% endif %}
|
||||
{% include "library/_model_health_badge.html" with h=model_health.embedding %}
|
||||
{% else %}
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="badge badge-error">NOT CONFIGURED</span>
|
||||
@@ -41,6 +42,7 @@
|
||||
<td>
|
||||
{% if system_chat_model %}
|
||||
<span class="font-semibold">{{ system_chat_model.api.name }}: {{ system_chat_model.name }}</span>
|
||||
{% include "library/_model_health_badge.html" with h=model_health.chat %}
|
||||
{% else %}
|
||||
<span class="text-sm opacity-60">Not configured — concept extraction disabled</span>
|
||||
{% endif %}
|
||||
@@ -51,6 +53,7 @@
|
||||
<td>
|
||||
{% if system_reranker_model %}
|
||||
<span class="font-semibold">{{ system_reranker_model.api.name }}: {{ system_reranker_model.name }}</span>
|
||||
{% include "library/_model_health_badge.html" with h=model_health.reranker %}
|
||||
{% else %}
|
||||
<span class="text-sm opacity-60">Not configured — Phase 3</span>
|
||||
{% endif %}
|
||||
@@ -64,6 +67,7 @@
|
||||
{% if system_vision_model.supports_vision %}
|
||||
<span class="badge badge-accent badge-sm ml-1">Vision</span>
|
||||
{% endif %}
|
||||
{% include "library/_model_health_badge.html" with h=model_health.vision %}
|
||||
{% else %}
|
||||
<span class="text-sm opacity-60">Not configured — image analysis disabled</span>
|
||||
{% endif %}
|
||||
|
||||
@@ -12,6 +12,18 @@
|
||||
<div class="alert alert-warning mb-6">
|
||||
<span>Are you sure you want to delete <strong>{{ library.name }}</strong>? This action cannot be undone.</span>
|
||||
</div>
|
||||
{% if library.workspace_id %}
|
||||
<div class="alert alert-error mb-6">
|
||||
<span>
|
||||
<strong>This Library is managed by Daedalus</strong>
|
||||
(workspace <code>{{ library.workspace_id }}</code>).
|
||||
Deleting it here removes its embedded content from Mnemosyne, but the
|
||||
source files still live in Daedalus — it will be <strong>recreated and
|
||||
re-embedded on the next Daedalus sync</strong>. Use this to clear an
|
||||
orphaned Library that is blocking workspace re-registration.
|
||||
</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
<form method="post">
|
||||
{% csrf_token %}
|
||||
<div class="flex gap-2">
|
||||
|
||||
@@ -25,14 +25,7 @@
|
||||
</div>
|
||||
<div class="flex gap-2">
|
||||
<a href="{% url 'library:library-edit' uid=library.uid %}" class="btn btn-sm btn-outline">Edit</a>
|
||||
{% if library.workspace_id %}
|
||||
<button type="button" class="btn btn-sm btn-error btn-outline" disabled
|
||||
title="This library is managed by Daedalus. Delete it from the Daedalus workspace, not here.">
|
||||
Delete
|
||||
</button>
|
||||
{% else %}
|
||||
<a href="{% url 'library:library-delete' uid=library.uid %}" class="btn btn-sm btn-error btn-outline">Delete</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -43,8 +36,11 @@
|
||||
<div class="text-sm opacity-80">
|
||||
This library was created for Daedalus workspace
|
||||
<code class="font-mono">{{ library.workspace_id }}</code>.
|
||||
Items here are owned by the workspace; deleting the workspace in
|
||||
Daedalus will remove this library. Do not delete it manually.
|
||||
Normally you manage it from Daedalus. Deleting it here removes its
|
||||
embedded content from Mnemosyne, but the source files still live in
|
||||
Daedalus — it will be recreated and re-embedded on the next sync.
|
||||
Use Delete to clear an orphaned library that is blocking workspace
|
||||
re-registration.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -15,6 +15,18 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<form method="get" class="mb-4 flex flex-wrap gap-3 items-end">
|
||||
<div class="form-control">
|
||||
<label class="label"><span class="label-text">Scope</span></label>
|
||||
<select name="scope" class="select select-bordered select-sm">
|
||||
<option value="all" {% if scope == "all" %}selected{% endif %}>All libraries</option>
|
||||
<option value="global" {% if scope == "global" %}selected{% endif %}>Global only</option>
|
||||
<option value="daedalus" {% if scope == "daedalus" %}selected{% endif %}>Daedalus workspaces only</option>
|
||||
</select>
|
||||
</div>
|
||||
<button type="submit" class="btn btn-sm btn-outline">Filter</button>
|
||||
</form>
|
||||
|
||||
{% if error %}
|
||||
<div class="alert alert-warning mb-4">
|
||||
<span>{{ error }}</span>
|
||||
@@ -53,8 +65,12 @@
|
||||
{% else %}
|
||||
{% if not error %}
|
||||
<div class="text-center py-12 opacity-60">
|
||||
{% if scope == "all" %}
|
||||
<p class="text-lg">No libraries yet.</p>
|
||||
<p class="mt-2">Create your first library to get started.</p>
|
||||
{% else %}
|
||||
<p class="text-lg">No libraries match this filter.</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
@@ -48,30 +48,3 @@ class ConceptExtractionParsingTests(TestCase):
|
||||
result = self.extractor._parse_concept_response(response)
|
||||
self.assertEqual(len(result), 1)
|
||||
self.assertEqual(result[0]["name"], "valid")
|
||||
|
||||
|
||||
class SampleIndexSelectionTests(TestCase):
|
||||
"""Tests for sample index selection."""
|
||||
|
||||
def setUp(self):
|
||||
self.extractor = ConceptExtractor(MagicMock())
|
||||
|
||||
def test_small_total_returns_all(self):
|
||||
indices = self.extractor._select_sample_indices(5, max_samples=10)
|
||||
self.assertEqual(indices, [0, 1, 2, 3, 4])
|
||||
|
||||
def test_equal_total_returns_all(self):
|
||||
indices = self.extractor._select_sample_indices(10, max_samples=10)
|
||||
self.assertEqual(indices, list(range(10)))
|
||||
|
||||
def test_large_total_returns_max_samples(self):
|
||||
indices = self.extractor._select_sample_indices(100, max_samples=10)
|
||||
self.assertEqual(len(indices), 10)
|
||||
# Should be evenly spaced
|
||||
self.assertEqual(indices[0], 0)
|
||||
self.assertEqual(indices[-1], 90)
|
||||
|
||||
def test_returns_integers(self):
|
||||
indices = self.extractor._select_sample_indices(50, max_samples=7)
|
||||
for idx in indices:
|
||||
self.assertIsInstance(idx, int)
|
||||
|
||||
@@ -21,6 +21,7 @@ class LibraryTypeDefaultsTests(TestCase):
|
||||
"film",
|
||||
"art",
|
||||
"journal",
|
||||
"email",
|
||||
"business",
|
||||
"finance",
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ class EmbeddingPipelineInitTests(TestCase):
|
||||
class PipelineItemNotFoundTests(TestCase):
|
||||
"""Tests for handling missing items."""
|
||||
|
||||
@patch("library.services.pipeline.Item")
|
||||
@patch("library.models.Item")
|
||||
def test_process_nonexistent_item_raises(self, mock_item_cls):
|
||||
mock_item_cls.nodes.get.side_effect = Exception("Not found")
|
||||
|
||||
@@ -57,7 +57,7 @@ class PipelineItemNotFoundTests(TestCase):
|
||||
pipeline.process_item("nonexistent-uid")
|
||||
self.assertIn("Item not found", str(ctx.exception))
|
||||
|
||||
@patch("library.services.pipeline.Item")
|
||||
@patch("library.models.Item")
|
||||
def test_reprocess_nonexistent_item_raises(self, mock_item_cls):
|
||||
mock_item_cls.nodes.get.side_effect = Exception("Not found")
|
||||
|
||||
@@ -69,9 +69,9 @@ class PipelineItemNotFoundTests(TestCase):
|
||||
class PipelineNoEmbeddingModelTests(TestCase):
|
||||
"""Tests for handling missing system embedding model."""
|
||||
|
||||
@patch("library.services.pipeline.LLMModel")
|
||||
@patch("llm_manager.models.LLMModel")
|
||||
@patch("library.services.pipeline.default_storage")
|
||||
@patch("library.services.pipeline.DocumentParser")
|
||||
@patch("library.services.parsers.DocumentParser")
|
||||
def test_no_embedding_model_raises(self, mock_parser, mock_storage, mock_llm):
|
||||
"""Pipeline raises ValueError if no system embedding model is configured."""
|
||||
mock_llm.get_system_embedding_model.return_value = None
|
||||
@@ -86,7 +86,7 @@ class PipelineNoEmbeddingModelTests(TestCase):
|
||||
mock_item.chunks.all.return_value = []
|
||||
mock_item.images.all.return_value = []
|
||||
|
||||
with patch("library.services.pipeline.Item") as mock_item_cls:
|
||||
with patch("library.models.Item") as mock_item_cls:
|
||||
mock_item_cls.nodes.get.return_value = mock_item
|
||||
|
||||
# Mock S3 read
|
||||
@@ -166,11 +166,11 @@ class PipelineVisionStageTests(TestCase):
|
||||
item.images.all.return_value = []
|
||||
return item
|
||||
|
||||
@patch("library.services.pipeline.ConceptExtractor")
|
||||
@patch("library.services.pipeline.EmbeddingClient")
|
||||
@patch("library.services.pipeline.ContentTypeChunker")
|
||||
@patch("library.services.pipeline.DocumentParser")
|
||||
@patch("library.services.pipeline.LLMModel")
|
||||
@patch("library.services.concepts.ConceptExtractor")
|
||||
@patch("library.services.embedding_client.EmbeddingClient")
|
||||
@patch("library.services.chunker.ContentTypeChunker")
|
||||
@patch("library.services.parsers.DocumentParser")
|
||||
@patch("llm_manager.models.LLMModel")
|
||||
@patch("library.services.pipeline.default_storage")
|
||||
def test_no_vision_model_marks_images_skipped(
|
||||
self, mock_storage, mock_llm, mock_parser_cls,
|
||||
@@ -227,12 +227,12 @@ class PipelineVisionStageTests(TestCase):
|
||||
img_node.save.assert_called()
|
||||
self.assertEqual(result["images_analyzed"], 0)
|
||||
|
||||
@patch("library.services.pipeline.VisionAnalyzer")
|
||||
@patch("library.services.pipeline.ConceptExtractor")
|
||||
@patch("library.services.pipeline.EmbeddingClient")
|
||||
@patch("library.services.pipeline.ContentTypeChunker")
|
||||
@patch("library.services.pipeline.DocumentParser")
|
||||
@patch("library.services.pipeline.LLMModel")
|
||||
@patch("library.services.vision.VisionAnalyzer")
|
||||
@patch("library.services.concepts.ConceptExtractor")
|
||||
@patch("library.services.embedding_client.EmbeddingClient")
|
||||
@patch("library.services.chunker.ContentTypeChunker")
|
||||
@patch("library.services.parsers.DocumentParser")
|
||||
@patch("llm_manager.models.LLMModel")
|
||||
@patch("library.services.pipeline.default_storage")
|
||||
def test_vision_model_triggers_analysis(
|
||||
self, mock_storage, mock_llm, mock_parser_cls,
|
||||
@@ -287,7 +287,7 @@ class PipelineVisionStageTests(TestCase):
|
||||
mock_vision_cls.assert_called_once_with(mock_vision_model, user=None)
|
||||
mock_analyzer.analyze_images.assert_called_once()
|
||||
|
||||
@patch("library.services.pipeline.LLMModel")
|
||||
@patch("llm_manager.models.LLMModel")
|
||||
def test_no_images_skips_vision_entirely(self, mock_llm):
|
||||
"""When there are no images, vision stage is a no-op regardless of model."""
|
||||
mock_vision_model = MagicMock()
|
||||
@@ -309,10 +309,10 @@ class PipelineVisionStageTests(TestCase):
|
||||
patch.object(pipeline, "_store_chunks", return_value=[]), \
|
||||
patch.object(pipeline, "_store_images", return_value=[]), \
|
||||
patch.object(pipeline, "_associate_images_with_chunks"), \
|
||||
patch("library.services.pipeline.DocumentParser") as mock_parser_cls, \
|
||||
patch("library.services.pipeline.ContentTypeChunker") as mock_chunker_cls, \
|
||||
patch("library.services.pipeline.EmbeddingClient"), \
|
||||
patch("library.services.pipeline.VisionAnalyzer") as mock_vision_cls:
|
||||
patch("library.services.parsers.DocumentParser") as mock_parser_cls, \
|
||||
patch("library.services.chunker.ContentTypeChunker") as mock_chunker_cls, \
|
||||
patch("library.services.embedding_client.EmbeddingClient"), \
|
||||
patch("library.services.vision.VisionAnalyzer") as mock_vision_cls:
|
||||
|
||||
mock_parser = MagicMock()
|
||||
mock_parser.parse_bytes.return_value = MagicMock(images=[], text_blocks=[])
|
||||
|
||||
@@ -100,7 +100,7 @@ class SearchAPIResponseTest(TestCase):
|
||||
self.client = APIClient()
|
||||
self.client.force_authenticate(user=self.user)
|
||||
|
||||
@patch("library.api.views.SearchService")
|
||||
@patch("library.services.search.SearchService")
|
||||
def test_successful_search_response_format(self, MockService):
|
||||
"""Successful search returns expected JSON structure."""
|
||||
mock_response = SearchResponse(
|
||||
@@ -159,7 +159,7 @@ class SearchAPIResponseTest(TestCase):
|
||||
self.assertEqual(image["image_uid"], "img1")
|
||||
self.assertEqual(image["image_type"], "diagram")
|
||||
|
||||
@patch("library.api.views.SearchService")
|
||||
@patch("library.services.search.SearchService")
|
||||
def test_vector_only_endpoint(self, MockService):
|
||||
"""Vector-only endpoint sets correct search types."""
|
||||
mock_response = SearchResponse(
|
||||
@@ -184,7 +184,7 @@ class SearchAPIResponseTest(TestCase):
|
||||
self.assertEqual(call_args.search_types, ["vector"])
|
||||
self.assertFalse(call_args.rerank)
|
||||
|
||||
@patch("library.api.views.SearchService")
|
||||
@patch("library.services.search.SearchService")
|
||||
def test_fulltext_only_endpoint(self, MockService):
|
||||
"""Fulltext-only endpoint sets correct search types."""
|
||||
mock_response = SearchResponse(
|
||||
@@ -208,7 +208,7 @@ class SearchAPIResponseTest(TestCase):
|
||||
self.assertEqual(call_args.search_types, ["fulltext"])
|
||||
self.assertFalse(call_args.rerank)
|
||||
|
||||
@patch("library.api.views.SearchService")
|
||||
@patch("library.services.search.SearchService")
|
||||
def test_reranker_skip_reason_surfaced_in_json(self, MockService):
|
||||
"""``reranker_skip_reason`` propagates through the JSON API."""
|
||||
mock_response = SearchResponse(
|
||||
|
||||
@@ -48,7 +48,7 @@ class AllLibraryUidsHelperTests(TestCase):
|
||||
|
||||
def test_returns_empty_when_neo4j_unavailable(self):
|
||||
"""Helper must not touch ``Library.nodes`` if Neo4j is down."""
|
||||
with patch("library.views.neo4j_available", return_value=False):
|
||||
with patch("library.utils.neo4j_available", return_value=False):
|
||||
self.assertEqual(views._all_library_uids(), [])
|
||||
|
||||
def test_returns_every_library_uid(self):
|
||||
@@ -62,7 +62,7 @@ class AllLibraryUidsHelperTests(TestCase):
|
||||
fake_nodes.all.return_value = fake_libs
|
||||
fake_library_cls = SimpleNamespace(nodes=fake_nodes)
|
||||
|
||||
with patch("library.views.neo4j_available", return_value=True), \
|
||||
with patch("library.utils.neo4j_available", return_value=True), \
|
||||
patch.dict("sys.modules", {"library.models": SimpleNamespace(Library=fake_library_cls)}):
|
||||
result = views._all_library_uids()
|
||||
|
||||
@@ -83,7 +83,7 @@ class AllLibraryUidsHelperTests(TestCase):
|
||||
fake_nodes.all.return_value = fake_libs
|
||||
fake_library_cls = SimpleNamespace(nodes=fake_nodes)
|
||||
|
||||
with patch("library.views.neo4j_available", return_value=True), \
|
||||
with patch("library.utils.neo4j_available", return_value=True), \
|
||||
patch.dict("sys.modules", {"library.models": SimpleNamespace(Library=fake_library_cls)}):
|
||||
result = views._all_library_uids()
|
||||
|
||||
@@ -95,7 +95,7 @@ class AllLibraryUidsHelperTests(TestCase):
|
||||
fake_nodes.all.side_effect = RuntimeError("neo4j blew up")
|
||||
fake_library_cls = SimpleNamespace(nodes=fake_nodes)
|
||||
|
||||
with patch("library.views.neo4j_available", return_value=True), \
|
||||
with patch("library.utils.neo4j_available", return_value=True), \
|
||||
patch.dict("sys.modules", {"library.models": SimpleNamespace(Library=fake_library_cls)}):
|
||||
self.assertEqual(views._all_library_uids(), [])
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from django.test import TestCase, override_settings
|
||||
class EmbedItemTaskTests(TestCase):
|
||||
"""Tests for the embed_item task."""
|
||||
|
||||
@patch("library.tasks.EmbeddingPipeline")
|
||||
@patch("library.services.pipeline.EmbeddingPipeline")
|
||||
def test_embed_item_success(self, mock_pipeline_cls):
|
||||
from library.tasks import embed_item
|
||||
|
||||
@@ -31,7 +31,7 @@ class EmbedItemTaskTests(TestCase):
|
||||
self.assertEqual(result["item_uid"], "test-uid-123")
|
||||
mock_pipeline.process_item.assert_called_once()
|
||||
|
||||
@patch("library.tasks.EmbeddingPipeline")
|
||||
@patch("library.services.pipeline.EmbeddingPipeline")
|
||||
def test_embed_item_failure(self, mock_pipeline_cls):
|
||||
from library.tasks import embed_item
|
||||
|
||||
@@ -49,7 +49,7 @@ class EmbedItemTaskTests(TestCase):
|
||||
class ReembedItemTaskTests(TestCase):
|
||||
"""Tests for the reembed_item task."""
|
||||
|
||||
@patch("library.tasks.EmbeddingPipeline")
|
||||
@patch("library.services.pipeline.EmbeddingPipeline")
|
||||
def test_reembed_item_success(self, mock_pipeline_cls):
|
||||
from library.tasks import reembed_item
|
||||
|
||||
|
||||
92
mnemosyne/library/tests/test_views.py
Normal file
92
mnemosyne/library/tests/test_views.py
Normal file
@@ -0,0 +1,92 @@
|
||||
"""Tests for the library CRUD HTML views.
|
||||
|
||||
Currently covers ``library_list``'s Daedalus-workspace scope filter. The
|
||||
view loads every ``Library`` node from Neo4j and narrows it by a ``scope``
|
||||
GET param (``all`` / ``global`` / ``daedalus``). These tests stub out
|
||||
Neo4j entirely — patching ``neo4j_available`` and injecting a fake
|
||||
``Library`` class via ``sys.modules`` — so they assert on the queryset
|
||||
``.filter(...)`` call the view makes and the context it renders, not on
|
||||
real graph behaviour. Mirrors the mocking style in
|
||||
``test_search_views_admin_scope.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.test import TestCase
|
||||
from django.urls import reverse
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
class LibraryListScopeFilterTests(TestCase):
|
||||
"""Cover the ``scope`` filter branches of ``library_list``."""
|
||||
|
||||
def setUp(self):
|
||||
self.user = User.objects.create_user(
|
||||
username="op", email="op@example.com", password="pw"
|
||||
)
|
||||
self.client.force_login(self.user)
|
||||
self.url = reverse("library:library-list")
|
||||
|
||||
def _fake_library_cls(self):
|
||||
"""Return (Library stub, nodes mock) where ``nodes`` chains fluently.
|
||||
|
||||
``Library.nodes`` → ``.filter(...)`` → ``.order_by(...)`` all return
|
||||
the same MagicMock so the view's queryset building works regardless
|
||||
of which branch it takes, and ``.filter`` records its kwargs.
|
||||
"""
|
||||
fake_nodes = MagicMock()
|
||||
fake_nodes.filter.return_value = fake_nodes
|
||||
fake_nodes.order_by.return_value = []
|
||||
return SimpleNamespace(nodes=fake_nodes), fake_nodes
|
||||
|
||||
def _get(self, fake_library_cls, **params):
|
||||
with patch("library.views.neo4j_available", return_value=True), \
|
||||
patch.dict(
|
||||
"sys.modules",
|
||||
{"library.models": SimpleNamespace(Library=fake_library_cls)},
|
||||
):
|
||||
return self.client.get(self.url, params)
|
||||
|
||||
def test_default_scope_is_all_and_does_not_filter(self):
|
||||
fake_cls, fake_nodes = self._fake_library_cls()
|
||||
response = self._get(fake_cls)
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(response.context["scope"], "all")
|
||||
fake_nodes.filter.assert_not_called()
|
||||
fake_nodes.order_by.assert_called_once_with("name")
|
||||
|
||||
def test_global_scope_filters_workspace_isnull_true(self):
|
||||
fake_cls, fake_nodes = self._fake_library_cls()
|
||||
response = self._get(fake_cls, scope="global")
|
||||
|
||||
self.assertEqual(response.context["scope"], "global")
|
||||
fake_nodes.filter.assert_called_once_with(workspace_id__isnull=True)
|
||||
|
||||
def test_daedalus_scope_filters_workspace_isnull_false(self):
|
||||
fake_cls, fake_nodes = self._fake_library_cls()
|
||||
response = self._get(fake_cls, scope="daedalus")
|
||||
|
||||
self.assertEqual(response.context["scope"], "daedalus")
|
||||
fake_nodes.filter.assert_called_once_with(workspace_id__isnull=False)
|
||||
|
||||
def test_unknown_scope_does_not_filter(self):
|
||||
"""An unexpected scope value degrades to the unfiltered list."""
|
||||
fake_cls, fake_nodes = self._fake_library_cls()
|
||||
response = self._get(fake_cls, scope="bogus")
|
||||
|
||||
self.assertEqual(response.context["scope"], "bogus")
|
||||
fake_nodes.filter.assert_not_called()
|
||||
|
||||
def test_neo4j_unavailable_sets_error_and_empty_list(self):
|
||||
with patch("library.views.neo4j_available", return_value=False):
|
||||
response = self.client.get(self.url)
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(list(response.context["libraries"]), [])
|
||||
self.assertEqual(response.context["error"], "Neo4j is not available.")
|
||||
@@ -31,14 +31,20 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
@login_required
|
||||
def library_list(request):
|
||||
"""List all libraries."""
|
||||
"""List libraries, optionally filtered by Daedalus-workspace scope."""
|
||||
scope = request.GET.get("scope", "all")
|
||||
libraries = []
|
||||
error = None
|
||||
if neo4j_available():
|
||||
try:
|
||||
from .models import Library
|
||||
|
||||
libraries = Library.nodes.order_by("name")
|
||||
qs = Library.nodes
|
||||
if scope == "daedalus":
|
||||
qs = qs.filter(workspace_id__isnull=False)
|
||||
elif scope == "global":
|
||||
qs = qs.filter(workspace_id__isnull=True)
|
||||
libraries = qs.order_by("name")
|
||||
except Exception as e:
|
||||
error = f"Could not connect to Neo4j: {e}"
|
||||
logger.error(error)
|
||||
@@ -47,7 +53,7 @@ def library_list(request):
|
||||
return render(
|
||||
request,
|
||||
"library/library_list.html",
|
||||
{"libraries": libraries, "error": error},
|
||||
{"libraries": libraries, "error": error, "scope": scope},
|
||||
)
|
||||
|
||||
|
||||
@@ -319,20 +325,20 @@ def library_delete(request, uid):
|
||||
messages.error(request, f"Library not found: {e}")
|
||||
return redirect("library:library-list")
|
||||
|
||||
# Daedalus owns the lifecycle of workspace-scoped libraries — they can
|
||||
# only be deleted via DELETE /library/api/workspaces/{workspace_id}/.
|
||||
# Block the human delete path so a stray click can't desync state.
|
||||
if lib.workspace_id:
|
||||
messages.error(
|
||||
request,
|
||||
f'"{lib.name}" is managed by Daedalus workspace '
|
||||
f"{lib.workspace_id}. Delete it from Daedalus, not here.",
|
||||
)
|
||||
return redirect("library:library-detail", uid=uid)
|
||||
|
||||
# Daedalus owns the lifecycle of workspace-scoped libraries. Deleting one
|
||||
# here is allowed but discouraged: the confirm page warns that Daedalus
|
||||
# still holds the source content and will recreate + re-embed it on the
|
||||
# next sync. The risk is low (no data loss — only re-embedding cost), and
|
||||
# this is the supported escape hatch for clearing an orphaned Library that
|
||||
# blocks workspace re-registration.
|
||||
if request.method == "POST":
|
||||
name = lib.name
|
||||
lib.delete()
|
||||
# Use the shared cascade so child nodes (Collections/Items/Chunks/
|
||||
# Images) and orphan Concepts are removed too — a bare lib.delete()
|
||||
# would leak them.
|
||||
from .services.library_delete import delete_library_cascade
|
||||
|
||||
delete_library_cascade(lib)
|
||||
messages.success(request, f'Library "{name}" deleted.')
|
||||
return redirect("library:library-list")
|
||||
return render(request, "library/library_confirm_delete.html", {"library": lib})
|
||||
@@ -729,6 +735,16 @@ def embedding_dashboard(request):
|
||||
except Exception as exc:
|
||||
logger.warning("Could not load system models: %s", exc)
|
||||
|
||||
# Reachability of the system-default models (keyed by role for the
|
||||
# template). A probe failure must never 500 the dashboard.
|
||||
context["model_health"] = {}
|
||||
try:
|
||||
from library.services.model_health import probe_system_models
|
||||
|
||||
context["model_health"] = {r["role"]: r for r in probe_system_models()}
|
||||
except Exception as exc:
|
||||
logger.warning("Could not probe system model health: %s", exc)
|
||||
|
||||
# Get item status counts and node counts from Neo4j
|
||||
if neo4j_available():
|
||||
context["neo4j_available"] = True
|
||||
|
||||
@@ -23,8 +23,9 @@ env = environ.Env(
|
||||
DEBUG=(bool, True),
|
||||
)
|
||||
|
||||
# Read .env file if it exists
|
||||
environ.Env.read_env(BASE_DIR / ".env")
|
||||
# Read .env file if it exists. Lives at the repo root (one level above the
|
||||
# Django project package), shared with the docker-compose stack.
|
||||
environ.Env.read_env(BASE_DIR.parent / ".env")
|
||||
|
||||
# --- Security ---
|
||||
SECRET_KEY = env("SECRET_KEY", default="django-insecure-change-me-in-production")
|
||||
@@ -267,6 +268,15 @@ SPELUNKER_S3_REGION_NAME = env("SPELUNKER_S3_REGION_NAME", default="us-east-1")
|
||||
SPELUNKER_S3_USE_SSL = env.bool("SPELUNKER_S3_USE_SSL", default=False)
|
||||
SPELUNKER_S3_VERIFY = env.bool("SPELUNKER_S3_VERIFY", default=True)
|
||||
|
||||
# Kairos renders synced mail to text documents in its own bucket.
|
||||
KAIROS_S3_ENDPOINT_URL = env("KAIROS_S3_ENDPOINT_URL", default="")
|
||||
KAIROS_S3_ACCESS_KEY_ID = env("KAIROS_S3_ACCESS_KEY_ID", default="")
|
||||
KAIROS_S3_SECRET_ACCESS_KEY = env("KAIROS_S3_SECRET_ACCESS_KEY", default="")
|
||||
KAIROS_S3_BUCKET_NAME = env("KAIROS_S3_BUCKET_NAME", default="kairos")
|
||||
KAIROS_S3_REGION_NAME = env("KAIROS_S3_REGION_NAME", default="us-east-1")
|
||||
KAIROS_S3_USE_SSL = env.bool("KAIROS_S3_USE_SSL", default=False)
|
||||
KAIROS_S3_VERIFY = env.bool("KAIROS_S3_VERIFY", default=True)
|
||||
|
||||
# Registry keyed by the ingest `source` field. Unknown/blank sources fall
|
||||
# back to "daedalus" for backwards compatibility.
|
||||
SOURCE_S3_BUCKETS = {
|
||||
@@ -288,6 +298,15 @@ SOURCE_S3_BUCKETS = {
|
||||
"use_ssl": SPELUNKER_S3_USE_SSL,
|
||||
"verify": SPELUNKER_S3_VERIFY,
|
||||
},
|
||||
"kairos-mail": {
|
||||
"endpoint_url": KAIROS_S3_ENDPOINT_URL,
|
||||
"access_key_id": KAIROS_S3_ACCESS_KEY_ID,
|
||||
"secret_access_key": KAIROS_S3_SECRET_ACCESS_KEY,
|
||||
"bucket_name": KAIROS_S3_BUCKET_NAME,
|
||||
"region_name": KAIROS_S3_REGION_NAME,
|
||||
"use_ssl": KAIROS_S3_USE_SSL,
|
||||
"verify": KAIROS_S3_VERIFY,
|
||||
},
|
||||
}
|
||||
|
||||
# --- Celery / RabbitMQ ---
|
||||
|
||||
Reference in New Issue
Block a user