Compare commits
10 Commits
2df22941d2
...
236d9e2e74
| Author | SHA1 | Date | |
|---|---|---|---|
| 236d9e2e74 | |||
| 1cd556c3f6 | |||
| e2a6d45b77 | |||
| 97a14fb03a | |||
| 2a8a3d75b4 | |||
| 5527cf6bdb | |||
| f2af28d96d | |||
| c485a8560c | |||
| 33658fbc8d | |||
| 81426327bf |
120
.gitea/workflows/cve-scan-docker-build.yml
Normal file
120
.gitea/workflows/cve-scan-docker-build.yml
Normal file
@@ -0,0 +1,120 @@
|
||||
name: CVE Scan & Docker Build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
REGISTRY: git.helu.ca
|
||||
IMAGE_NAME: ${{ gitea.repository }}
|
||||
TRIVY_SEVERITY: MEDIUM,HIGH,CRITICAL
|
||||
TRIVY_NO_PROGRESS: "true"
|
||||
TRIVY_DISABLE_VEX_NOTICE: "true"
|
||||
|
||||
jobs:
|
||||
security-scan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Trivy
|
||||
run: |
|
||||
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin
|
||||
trivy --version
|
||||
|
||||
- name: Resolve full dependency set (incl. dev/test/lint/docs extras)
|
||||
run: |
|
||||
python3 -m venv /tmp/scanenv
|
||||
/tmp/scanenv/bin/pip install --quiet pip-tools
|
||||
/tmp/scanenv/bin/pip-compile pyproject.toml \
|
||||
--extra dev --extra test --extra lint --extra docs \
|
||||
-o requirements.txt --no-header --quiet --allow-unsafe
|
||||
echo "Resolved $(grep -cv '^\s*\(#\|$\)' requirements.txt) pinned packages."
|
||||
|
||||
- name: Scan Python dependencies for CVEs
|
||||
run: |
|
||||
trivy fs \
|
||||
--scanners vuln \
|
||||
--severity ${TRIVY_SEVERITY} \
|
||||
--format table \
|
||||
--exit-code 0 \
|
||||
requirements.txt
|
||||
|
||||
- name: Scan repository for secrets
|
||||
run: |
|
||||
trivy fs \
|
||||
--scanners secret \
|
||||
--format table \
|
||||
--exit-code 0 \
|
||||
.
|
||||
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
needs: security-scan
|
||||
if: always()
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Gitea Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ gitea.actor }}
|
||||
password: ${{ secrets.PACKAGE_TOKEN }}
|
||||
|
||||
- name: Extract metadata for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=sha,prefix=
|
||||
type=raw,value=latest,enable=${{ gitea.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Install Trivy
|
||||
run: |
|
||||
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin
|
||||
trivy --version
|
||||
|
||||
- name: Scan built Docker image (OS + Python + system libs)
|
||||
run: |
|
||||
IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1)
|
||||
echo "🔍 Scanning image: ${IMAGE_TAG}"
|
||||
trivy image \
|
||||
--scanners vuln \
|
||||
--severity ${TRIVY_SEVERITY} \
|
||||
--format table \
|
||||
--pkg-types os,library \
|
||||
--exit-code 0 \
|
||||
"${IMAGE_TAG}"
|
||||
|
||||
- name: Scan built Docker image for misconfigurations
|
||||
continue-on-error: true
|
||||
run: |
|
||||
IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1)
|
||||
trivy image \
|
||||
--scanners misconfig \
|
||||
--severity ${TRIVY_SEVERITY} \
|
||||
--format table \
|
||||
--exit-code 0 \
|
||||
"${IMAGE_TAG}"
|
||||
93
Dockerfile
Normal file
93
Dockerfile
Normal file
@@ -0,0 +1,93 @@
|
||||
# =============================================================================
|
||||
# Mnemosyne — production image
|
||||
# =============================================================================
|
||||
# Multi-stage:
|
||||
# builder installs Python deps and runs `collectstatic` once.
|
||||
# runtime copies only the artifacts the running process needs.
|
||||
#
|
||||
# The same image runs three different processes (Django web, MCP server,
|
||||
# Celery worker) — the compose file picks the command per service.
|
||||
# =============================================================================
|
||||
|
||||
# ── Stage 1: builder ────────────────────────────────────────────────────────
|
||||
FROM python:3.12-slim AS builder
|
||||
|
||||
# Build deps for psycopg, PyMuPDF, Pillow, cryptography, etc.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
libpq-dev \
|
||||
libffi-dev \
|
||||
libssl-dev \
|
||||
libjpeg-dev \
|
||||
zlib1g-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Install dependencies first (better layer caching).
|
||||
COPY pyproject.toml README.md ./
|
||||
COPY mnemosyne/ ./mnemosyne/
|
||||
|
||||
RUN pip install --upgrade pip \
|
||||
&& pip install .
|
||||
|
||||
# Bake static files into the image. The env vars below are build-time-only
|
||||
# stubs needed for settings.py to import without real infrastructure — they
|
||||
# never reach the runtime image because this is the builder stage.
|
||||
# Inlined into the RUN command (rather than ENV/ARG) so static analysis
|
||||
# tools (Trivy) don't flag them as baked-in secrets.
|
||||
ENV DJANGO_SETTINGS_MODULE=mnemosyne.settings \
|
||||
DEBUG=False \
|
||||
USE_LOCAL_STORAGE=True \
|
||||
APP_DB_NAME=collectstatic \
|
||||
APP_DB_USER=collectstatic
|
||||
|
||||
WORKDIR /build/mnemosyne
|
||||
RUN SECRET_KEY=collectstatic-stub \
|
||||
APP_DB_PASSWORD=collectstatic-stub \
|
||||
python manage.py collectstatic --noinput --clear
|
||||
|
||||
# ── Stage 2: runtime ────────────────────────────────────────────────────────
|
||||
FROM python:3.12-slim AS runtime
|
||||
|
||||
# Runtime libs for psycopg + PyMuPDF + Pillow + cryptography.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libpq5 \
|
||||
libjpeg62-turbo \
|
||||
zlib1g \
|
||||
libssl3 \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
DJANGO_SETTINGS_MODULE=mnemosyne.settings \
|
||||
PATH=/usr/local/bin:$PATH
|
||||
|
||||
# Copy installed packages from the builder.
|
||||
COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
|
||||
COPY --from=builder /usr/local/bin /usr/local/bin
|
||||
|
||||
# Application code + collected statics.
|
||||
WORKDIR /app
|
||||
COPY --from=builder /build/mnemosyne /app
|
||||
COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint.sh
|
||||
|
||||
# Non-root user for everything that runs in this image. uid:gid 1000:1000
|
||||
# matches the convention for a single-application container.
|
||||
RUN groupadd --gid 1000 mnemosyne \
|
||||
&& useradd --uid 1000 --gid mnemosyne --home /app --no-create-home --shell /sbin/nologin mnemosyne \
|
||||
&& mkdir -p /app/media /app/logs \
|
||||
&& chown -R mnemosyne:mnemosyne /app
|
||||
USER mnemosyne
|
||||
|
||||
# The compose file overrides this per service. Default = Django web.
|
||||
EXPOSE 8000 22091
|
||||
ENTRYPOINT ["entrypoint.sh"]
|
||||
CMD ["web"]
|
||||
159
README.md
159
README.md
@@ -37,11 +37,14 @@ This **content-type awareness** flows through every layer: chunking strategy, em
|
||||
| Library | Example Content | Multimodal? | Graph Relationships |
|
||||
|---------|----------------|-------------|-------------------|
|
||||
| **Fiction** | Novels, short stories | Cover art | Author → Book → Character → Theme |
|
||||
| **Nonfiction** | History, biography, science writing | Photos, charts | Author → Work → Topic → Person/Place |
|
||||
| **Technical** | Textbooks, manuals, docs | Diagrams, screenshots | Product → Manual → Section → Procedure |
|
||||
| **Music** | Lyrics, liner notes | Album artwork | Artist → Album → Track → Genre |
|
||||
| **Film** | Scripts, synopses | Stills, posters | Director → Film → Scene → Actor |
|
||||
| **Art** | Descriptions, catalogs | The artwork itself | Artist → Piece → Style → Movement |
|
||||
| **Journals** | Personal entries | Photos | Date → Entry → Topic → Person/Place |
|
||||
| **Journal** | Personal entries, plans, observations | Photos | Date → Entry → Topic → Person/Place |
|
||||
| **Business** | Proposals, marketing, strategy | Logos, charts | Client → Engagement → Deliverable |
|
||||
| **Finance** | Statements, tax, market commentary | Charts, statement scans | Account → Instrument → Period |
|
||||
|
||||
## Search Pipeline
|
||||
|
||||
@@ -55,32 +58,160 @@ Query → Vector Search (Neo4j) + Graph Traversal (Cypher) + Full-Text Search
|
||||
|
||||
Mnemosyne's RAG pipeline architecture is inspired by [Spelunker](https://git.helu.ca/r/spelunker), an enterprise RFP response platform. The proven patterns — hybrid search, two-stage RAG (responder + reviewer), citation-based retrieval, and async document processing — are carried forward and enhanced with multimodal capabilities and knowledge graph relationships.
|
||||
|
||||
## Running Celery Workers
|
||||
## Running Mnemosyne
|
||||
|
||||
Mnemosyne uses Celery with RabbitMQ for async document embedding. From the `mnemosyne/` directory:
|
||||
Mnemosyne runs as three cooperating processes: the Django web app (REST API + admin), the MCP server (LLM-facing tools), and one or more Celery workers (async embedding + ingest). All three read configuration from `mnemosyne/.env` (copy from `mnemosyne/.env example` and fill in secrets).
|
||||
|
||||
Hosts in the Ouranos lab:
|
||||
- **Postgres** — `portia.incus:5432` (Django ORM: users, IngestJob)
|
||||
- **Neo4j** — `ariel.incus:25554` (knowledge graph + vectors)
|
||||
- **RabbitMQ** — `oberon.incus:5672` (Celery broker)
|
||||
- **MinIO** — `nyx.helu.ca:8555` (S3-compatible; `mnemosyne-content` and `daedalus` buckets)
|
||||
- **Memcached** — `127.0.0.1:11211` (task progress)
|
||||
|
||||
### One-time setup
|
||||
|
||||
```bash
|
||||
# Development — single worker, all queues
|
||||
cd mnemosyne/
|
||||
python manage.py migrate # Apply Django ORM migrations
|
||||
python manage.py setup_neo4j_indexes # Create Neo4j vector + full-text indexes
|
||||
python manage.py load_library_types # Load LIBRARY_TYPE_DEFAULTS into Neo4j
|
||||
```
|
||||
|
||||
### Start the web app
|
||||
|
||||
The Django REST API serves `/library/api/*` (libraries, collections, items, search, workspaces, ingest) and Django admin. Use Gunicorn in production; `runserver` for dev.
|
||||
|
||||
```bash
|
||||
cd mnemosyne/
|
||||
|
||||
# Development
|
||||
python manage.py runserver 0.0.0.0:8000
|
||||
|
||||
# Production
|
||||
gunicorn --bind 0.0.0.0:8000 --workers 3 mnemosyne.wsgi:application
|
||||
```
|
||||
|
||||
### Start the MCP server
|
||||
|
||||
The MCP server exposes the LLM-facing tools (`search`, `get_chunk`, `list_libraries`, `list_collections`, `list_items`, `get_health`) over Streamable HTTP at `/mcp` and SSE at `/mcp/sse`. Run as a separate Uvicorn process, on its own port, so it can be reverse-proxied or scaled independently of the Django app.
|
||||
|
||||
```bash
|
||||
cd mnemosyne/
|
||||
|
||||
# Single command: ASGI server hosting the FastMCP app
|
||||
uvicorn mnemosyne.asgi:app --host 0.0.0.0 --port 22091 --workers 1
|
||||
```
|
||||
|
||||
The `mcp_server/asgi.py` mounts FastMCP at `/mcp` (Streamable HTTP) and `/mcp/sse` (SSE), with a `/mcp/health` JSON probe for HAProxy/Pallas.
|
||||
|
||||
### Start a Celery worker
|
||||
|
||||
A single worker that handles all queues (development) plus the focused command Daedalus depends on (the `embedding` queue, where the Daedalus ingest task lives).
|
||||
|
||||
```bash
|
||||
cd mnemosyne/
|
||||
|
||||
# Development — one worker, all queues
|
||||
celery -A mnemosyne worker -l info -Q celery,embedding,batch
|
||||
|
||||
# Or skip workers entirely with eager mode (.env):
|
||||
CELERY_TASK_ALWAYS_EAGER=True
|
||||
# Production — embedding queue (handles Daedalus ingest + embed_item)
|
||||
celery -A mnemosyne worker -l info -Q embedding -c 1 -n embedding@%h
|
||||
|
||||
# Production — batch queue (collection/library bulk operations)
|
||||
celery -A mnemosyne worker -l info -Q batch -c 2 -n batch@%h
|
||||
|
||||
# Production — default queue (LLM validation, misc)
|
||||
celery -A mnemosyne worker -l info -Q celery -c 2 -n default@%h
|
||||
```
|
||||
|
||||
**Production — separate workers:**
|
||||
```bash
|
||||
celery -A mnemosyne worker -l info -Q embedding -c 1 -n embedding@%h # GPU-bound embedding
|
||||
celery -A mnemosyne worker -l info -Q batch -c 2 -n batch@%h # Batch orchestration
|
||||
celery -A mnemosyne worker -l info -Q celery -c 2 -n default@%h # LLM API validation
|
||||
```
|
||||
Daedalus's `POST /library/api/ingest/` dispatches `library.tasks.ingest_from_daedalus` to the **embedding** queue. If you only run one worker, make sure it consumes `embedding` or that task will sit in the broker.
|
||||
|
||||
**Scheduler & Monitoring:**
|
||||
To bypass workers in dev/test, set `CELERY_TASK_ALWAYS_EAGER=True` in `.env`.
|
||||
|
||||
**Scheduler & monitoring (optional):**
|
||||
```bash
|
||||
celery -A mnemosyne beat -l info # Periodic task scheduler
|
||||
celery -A mnemosyne flower --port=5555 # Web monitoring UI
|
||||
```
|
||||
|
||||
See [Phase 2: Celery Workers & Scheduler](docs/PHASE_2_EMBEDDING_PIPELINE.md#celery-workers--scheduler) for full details on queues, reliability settings, and task progress tracking.
|
||||
See [Phase 2: Celery Workers & Scheduler](docs/PHASE_2_EMBEDDING_PIPELINE.md#celery-workers--scheduler) for queue tuning, reliability settings, and task progress tracking.
|
||||
|
||||
### Daedalus integration endpoints
|
||||
|
||||
These endpoints are used by the Daedalus FastAPI backend (HTTP Basic auth). All under `/library/api/`:
|
||||
|
||||
| Method | Route | Purpose |
|
||||
|--------|-------|---------|
|
||||
| POST | `/workspaces/` | Create a workspace (idempotent on `workspace_id`); body: `{workspace_id, name, library_type, description?}` |
|
||||
| GET | `/workspaces/{workspace_id}/` | Workspace status (item/chunk counts) |
|
||||
| DELETE | `/workspaces/{workspace_id}/` | Delete workspace + reachable content; preserves shared concepts |
|
||||
| POST | `/ingest/` | Queue a file for ingestion + embedding |
|
||||
| GET | `/jobs/{job_id}/` | Poll ingest job status |
|
||||
| POST | `/jobs/{job_id}/retry/` | Re-dispatch a failed job |
|
||||
| GET | `/jobs/?status=&library_uid=` | List recent jobs |
|
||||
|
||||
See [docs/mnemosyne_integration.md](docs/mnemosyne_integration.md) for the full Daedalus contract.
|
||||
|
||||
## Production Deployment
|
||||
|
||||
Production runs as four containers from a single image (built and pushed by [`.gitea/workflows/cve-scan-docker-build.yml`](.gitea/workflows/cve-scan-docker-build.yml) on every push to `main`):
|
||||
|
||||
| Service | Role | Port |
|
||||
|---------|------|------|
|
||||
| `web` | Django REST API + admin (gunicorn) | internal :8000 |
|
||||
| `mcp` | FastMCP server (uvicorn) | internal :22091 |
|
||||
| `worker` | Celery worker — embedding/ingest/batch | — |
|
||||
| `nginx` | Reverse proxy + static files | host :23090 |
|
||||
|
||||
Plus a one-shot `static-init` service that copies `/app/staticfiles` (baked into the image at build time via `collectstatic`) into the shared volume nginx reads from. It runs to completion on every `up`, so static-file changes propagate on each deploy without manual intervention.
|
||||
|
||||
External services (NOT spun up by compose): Postgres on Portia, Neo4j on Ariel, RabbitMQ on Oberon, S3/MinIO on Nyx, Memcached, embedder + reranker. All reached over the internal 10.10.0.0/24 network. URLs and credentials live in `mnemosyne/.env`.
|
||||
|
||||
### First-time bring-up
|
||||
|
||||
```bash
|
||||
# Pull the image (or build locally with `docker compose build`)
|
||||
docker compose pull
|
||||
|
||||
# DB migrations (one-shot)
|
||||
docker compose run --rm web migrate
|
||||
|
||||
# Neo4j indexes + library_type defaults (one-shot)
|
||||
docker compose run --rm web setup
|
||||
|
||||
# Bring the stack up
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
### Day-to-day
|
||||
|
||||
```bash
|
||||
docker compose ps # service status + health
|
||||
docker compose logs -f web # tail web logs
|
||||
docker compose logs -f worker # tail Celery worker logs
|
||||
docker compose restart mcp # restart just the MCP server
|
||||
|
||||
# After a new image is published:
|
||||
docker compose pull && docker compose up -d
|
||||
```
|
||||
|
||||
### Things to verify in `mnemosyne/.env` before bringing up
|
||||
|
||||
The development `.env` has a few values that need adjusting for production:
|
||||
|
||||
- `DEBUG=False`
|
||||
- `USE_LOCAL_STORAGE=False` (already set; just confirm)
|
||||
- `KVDB_LOCATION=<external-memcached-host>:11211` — `127.0.0.1` does not resolve from inside containers
|
||||
- `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` filled in
|
||||
- `DAEDALUS_S3_*` filled in for cross-bucket reads from the Daedalus bucket
|
||||
- `ALLOWED_HOSTS` includes the public hostname HAProxy routes to (e.g. `mnemosyne.ouranos.helu.ca`)
|
||||
- `LLM_API_SECRETS_ENCRYPTION_KEY` set to a real Fernet key
|
||||
|
||||
### Health probes
|
||||
|
||||
- `GET http://nginx-host:23090/healthz` → proxies to `/mcp/health`, returns `{"status":"ok"}` when the MCP server is up
|
||||
- `GET http://nginx-host:23090/metrics` → Prometheus scrape endpoint, internal-network-only
|
||||
|
||||
## Architecture Note: Retrieval, Not Synthesis
|
||||
|
||||
|
||||
111
docker-compose.yaml
Normal file
111
docker-compose.yaml
Normal file
@@ -0,0 +1,111 @@
|
||||
# =============================================================================
|
||||
# Mnemosyne — production deployment
|
||||
# =============================================================================
|
||||
# Four services, all from the same image:
|
||||
# web — Django REST API + admin (gunicorn, port 8000)
|
||||
# mcp — FastMCP server (uvicorn, port 22091)
|
||||
# worker — Celery worker (embedding/ingest/batch queues)
|
||||
# nginx — reverse proxy, public port 23090
|
||||
#
|
||||
# External services (NOT spun up here): Postgres on Portia, Neo4j on Ariel,
|
||||
# RabbitMQ on Oberon, S3/MinIO on Nyx, Memcached on its own host, embedder
|
||||
# and reranker on Nyx, smtp4dev on Oberon. All reached over the internal
|
||||
# 10.10.0.0/24 network.
|
||||
#
|
||||
# Run:
|
||||
# docker compose up -d
|
||||
# docker compose run --rm web migrate # one-shot DB migrate
|
||||
# docker compose run --rm web setup # Neo4j indexes + library types
|
||||
# =============================================================================
|
||||
|
||||
services:
|
||||
# ── Static-file seeder: copies /app/staticfiles into the shared volume on
|
||||
# every `up`. Runs once and exits. Without this, the named volume is only
|
||||
# seeded the first time it's empty, so static updates between deploys
|
||||
# would not propagate to nginx.
|
||||
static-init:
|
||||
image: git.helu.ca/r/mnemosyne:latest
|
||||
command: ["sh", "-c", "cp -a /app/staticfiles/. /shared-static/"]
|
||||
user: "0:0"
|
||||
volumes:
|
||||
- mnemosyne-static:/shared-static
|
||||
restart: "no"
|
||||
|
||||
# ── Web app: Django REST API + admin ───────────────────────────────────────
|
||||
web:
|
||||
image: git.helu.ca/r/mnemosyne:latest
|
||||
command: ["web"]
|
||||
env_file: mnemosyne/.env
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
static-init:
|
||||
condition: service_completed_successfully
|
||||
volumes:
|
||||
- mnemosyne-media:/app/media
|
||||
expose:
|
||||
- "8000"
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/admin/login/').read()"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
|
||||
# ── MCP server: FastMCP Streamable HTTP at /mcp/ ───────────────────────────
|
||||
mcp:
|
||||
image: git.helu.ca/r/mnemosyne:latest
|
||||
command: ["mcp"]
|
||||
env_file: mnemosyne/.env
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- mnemosyne-media:/app/media
|
||||
expose:
|
||||
- "22091"
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:22091/mcp/health').read()"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
|
||||
# ── Celery worker: embedding + ingest + batch queues ───────────────────────
|
||||
worker:
|
||||
image: git.helu.ca/r/mnemosyne:latest
|
||||
command: ["worker"]
|
||||
env_file: mnemosyne/.env
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- mnemosyne-media:/app/media
|
||||
healthcheck:
|
||||
test: ["CMD", "celery", "-A", "mnemosyne", "inspect", "ping", "-d", "celery@$$HOSTNAME"]
|
||||
interval: 60s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
|
||||
# ── nginx: reverse proxy, public port 23090 ────────────────────────────────
|
||||
nginx:
|
||||
image: nginx:alpine
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- web
|
||||
- mcp
|
||||
ports:
|
||||
- "23090:80"
|
||||
volumes:
|
||||
- ./nginx/mnemosyne.conf:/etc/nginx/conf.d/default.conf:ro
|
||||
- mnemosyne-static:/var/www/static:ro
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost/healthz"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
volumes:
|
||||
# Static files baked into the image at /app/staticfiles. The web service
|
||||
# mounts this volume, populating it on first start; nginx reads from it.
|
||||
mnemosyne-static:
|
||||
# Local FileSystemStorage fallback. Production uses USE_LOCAL_STORAGE=False
|
||||
# so this is mostly empty — kept for parity with dev and for any path
|
||||
# that writes to MEDIA_ROOT directly.
|
||||
mnemosyne-media:
|
||||
66
docker/entrypoint.sh
Normal file
66
docker/entrypoint.sh
Normal file
@@ -0,0 +1,66 @@
|
||||
#!/bin/sh
|
||||
# Mnemosyne container entrypoint.
|
||||
#
|
||||
# The same image runs all three processes — the compose service supplies
|
||||
# `web`, `mcp`, `worker`, or `migrate` as CMD.
|
||||
|
||||
set -e
|
||||
|
||||
case "$1" in
|
||||
web)
|
||||
# Django REST API + admin (gunicorn → wsgi).
|
||||
exec gunicorn \
|
||||
--bind 0.0.0.0:8000 \
|
||||
--workers "${GUNICORN_WORKERS:-3}" \
|
||||
--access-logfile - \
|
||||
--error-logfile - \
|
||||
mnemosyne.wsgi:application
|
||||
;;
|
||||
|
||||
mcp)
|
||||
# FastMCP over Streamable HTTP at /mcp/, mounted by mnemosyne.asgi.
|
||||
exec uvicorn \
|
||||
--host 0.0.0.0 \
|
||||
--port 22091 \
|
||||
--workers "${UVICORN_WORKERS:-1}" \
|
||||
mnemosyne.asgi:app
|
||||
;;
|
||||
|
||||
worker)
|
||||
# Celery worker covering embedding + ingest + batch + default queues.
|
||||
# In production you may want to split these onto separate worker
|
||||
# services for queue-level isolation; one process is fine to start.
|
||||
exec celery -A mnemosyne worker \
|
||||
--loglevel="${CELERY_LOG_LEVEL:-info}" \
|
||||
--queues="${CELERY_QUEUES:-celery,embedding,batch}" \
|
||||
--concurrency="${CELERY_CONCURRENCY:-2}"
|
||||
;;
|
||||
|
||||
beat)
|
||||
# Celery scheduled tasks (only needed if/when periodic jobs are wired).
|
||||
exec celery -A mnemosyne beat \
|
||||
--loglevel="${CELERY_LOG_LEVEL:-info}"
|
||||
;;
|
||||
|
||||
migrate)
|
||||
# One-shot DB migration runner — invoke before bringing services up
|
||||
# for the first time or after a deploy.
|
||||
exec python manage.py migrate --noinput
|
||||
;;
|
||||
|
||||
setup)
|
||||
# One-shot init — Neo4j indexes + library_type seed data.
|
||||
python manage.py setup_neo4j_indexes
|
||||
python manage.py load_library_types
|
||||
;;
|
||||
|
||||
shell)
|
||||
# Drop into the management shell for ad-hoc work.
|
||||
exec python manage.py shell
|
||||
;;
|
||||
|
||||
*)
|
||||
# Fall through: run whatever was passed (e.g. `manage.py <cmd>`).
|
||||
exec "$@"
|
||||
;;
|
||||
esac
|
||||
@@ -40,6 +40,23 @@ AWS_S3_REGION_NAME=us-east-1
|
||||
# Set to True to use local FileSystemStorage instead of S3 (dev/test)
|
||||
USE_LOCAL_STORAGE=True
|
||||
|
||||
# --- Daedalus S3 (cross-bucket reads for ingest) ---
|
||||
# Mnemosyne ingests files from the Daedalus S3 bucket. These vars
|
||||
# configure read access; the file is copied into AWS_STORAGE_BUCKET_NAME
|
||||
# (Mnemosyne's own bucket) by the ingest Celery task.
|
||||
DAEDALUS_S3_ENDPOINT_URL=
|
||||
DAEDALUS_S3_ACCESS_KEY_ID=
|
||||
DAEDALUS_S3_SECRET_ACCESS_KEY=
|
||||
DAEDALUS_S3_BUCKET_NAME=daedalus
|
||||
DAEDALUS_S3_REGION_NAME=us-east-1
|
||||
DAEDALUS_S3_USE_SSL=False
|
||||
DAEDALUS_S3_VERIFY=True
|
||||
|
||||
# --- MCP Server ---
|
||||
# Set to False for internal-only deployments where the MCP transport is
|
||||
# already on a trusted network (10.10.0.0/24).
|
||||
MCP_REQUIRE_AUTH=True
|
||||
|
||||
# --- Email (smtp4dev on Oberon) ---
|
||||
EMAIL_HOST=oberon.incus
|
||||
EMAIL_PORT=22025
|
||||
|
||||
@@ -7,12 +7,23 @@ Serialize Neo4j neomodel nodes into JSON for the REST API.
|
||||
from rest_framework import serializers
|
||||
|
||||
|
||||
LIBRARY_TYPE_CHOICES = [
|
||||
"fiction",
|
||||
"nonfiction",
|
||||
"technical",
|
||||
"music",
|
||||
"film",
|
||||
"art",
|
||||
"journal",
|
||||
"business",
|
||||
"finance",
|
||||
]
|
||||
|
||||
|
||||
class LibrarySerializer(serializers.Serializer):
|
||||
uid = serializers.CharField(read_only=True)
|
||||
name = serializers.CharField(max_length=200)
|
||||
library_type = serializers.ChoiceField(
|
||||
choices=["fiction", "nonfiction", "technical", "music", "film", "art", "journal"]
|
||||
)
|
||||
library_type = serializers.ChoiceField(choices=LIBRARY_TYPE_CHOICES)
|
||||
description = serializers.CharField(required=False, allow_blank=True, default="")
|
||||
chunking_config = serializers.JSONField(required=False, default=dict)
|
||||
embedding_instruction = serializers.CharField(
|
||||
@@ -24,6 +35,7 @@ class LibrarySerializer(serializers.Serializer):
|
||||
llm_context_prompt = serializers.CharField(
|
||||
required=False, allow_blank=True, default=""
|
||||
)
|
||||
workspace_id = serializers.CharField(read_only=True)
|
||||
created_at = serializers.DateTimeField(read_only=True)
|
||||
|
||||
|
||||
@@ -90,12 +102,11 @@ class SearchRequestSerializer(serializers.Serializer):
|
||||
query = serializers.CharField(max_length=2000)
|
||||
library_uid = serializers.CharField(required=False, allow_blank=True)
|
||||
library_type = serializers.ChoiceField(
|
||||
choices=[
|
||||
"fiction", "nonfiction", "technical", "music", "film", "art", "journal",
|
||||
],
|
||||
choices=LIBRARY_TYPE_CHOICES,
|
||||
required=False,
|
||||
)
|
||||
collection_uid = serializers.CharField(required=False, allow_blank=True)
|
||||
workspace_id = serializers.CharField(required=False, allow_blank=True)
|
||||
search_types = serializers.ListField(
|
||||
child=serializers.ChoiceField(choices=["vector", "fulltext", "graph"]),
|
||||
required=False,
|
||||
@@ -139,3 +150,73 @@ class SearchResponseSerializer(serializers.Serializer):
|
||||
reranker_used = serializers.BooleanField()
|
||||
reranker_model = serializers.CharField(allow_null=True)
|
||||
search_types_used = serializers.ListField(child=serializers.CharField())
|
||||
|
||||
|
||||
# --- Workspace lifecycle (Daedalus integration) ---
|
||||
|
||||
|
||||
class WorkspaceCreateSerializer(serializers.Serializer):
|
||||
"""Inbound payload for POST /api/v1/workspaces/."""
|
||||
|
||||
workspace_id = serializers.CharField(max_length=64)
|
||||
name = serializers.CharField(max_length=200)
|
||||
library_type = serializers.ChoiceField(choices=LIBRARY_TYPE_CHOICES)
|
||||
description = serializers.CharField(required=False, allow_blank=True, default="")
|
||||
|
||||
|
||||
class WorkspaceStatusSerializer(serializers.Serializer):
|
||||
"""Outbound payload for workspace lifecycle endpoints."""
|
||||
|
||||
workspace_id = serializers.CharField()
|
||||
library_uid = serializers.CharField()
|
||||
name = serializers.CharField()
|
||||
library_type = serializers.CharField()
|
||||
description = serializers.CharField(allow_blank=True)
|
||||
item_count = serializers.IntegerField()
|
||||
chunk_count = serializers.IntegerField()
|
||||
created_at = serializers.DateTimeField()
|
||||
|
||||
|
||||
# --- Ingest (Daedalus integration) ---
|
||||
|
||||
|
||||
class IngestRequestSerializer(serializers.Serializer):
|
||||
"""Inbound payload for POST /api/v1/library/ingest/."""
|
||||
|
||||
s3_key = serializers.CharField(max_length=500)
|
||||
title = serializers.CharField(max_length=500)
|
||||
library_uid = serializers.CharField(required=False, allow_blank=True)
|
||||
workspace_id = serializers.CharField(required=False, allow_blank=True)
|
||||
collection_uid = serializers.CharField(required=False, allow_blank=True)
|
||||
file_type = serializers.CharField(required=False, allow_blank=True, default="")
|
||||
file_size = serializers.IntegerField(required=False, default=0)
|
||||
content_hash = serializers.CharField(max_length=64)
|
||||
source = serializers.CharField(required=False, allow_blank=True, default="")
|
||||
source_ref = serializers.CharField(required=False, allow_blank=True, default="")
|
||||
|
||||
def validate(self, data):
|
||||
if not data.get("library_uid") and not data.get("workspace_id"):
|
||||
raise serializers.ValidationError(
|
||||
"Either library_uid or workspace_id is required."
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
class IngestJobSerializer(serializers.Serializer):
|
||||
"""Outbound payload for ingest job status."""
|
||||
|
||||
job_id = serializers.CharField(source="id")
|
||||
item_uid = serializers.CharField(allow_blank=True)
|
||||
library_uid = serializers.CharField()
|
||||
status = serializers.CharField()
|
||||
progress = serializers.CharField()
|
||||
error = serializers.CharField(allow_null=True)
|
||||
chunks_created = serializers.IntegerField()
|
||||
concepts_extracted = serializers.IntegerField()
|
||||
embedding_model = serializers.CharField(allow_blank=True)
|
||||
content_hash = serializers.CharField(allow_blank=True)
|
||||
source = serializers.CharField(allow_blank=True)
|
||||
source_ref = serializers.CharField(allow_blank=True)
|
||||
created_at = serializers.DateTimeField()
|
||||
started_at = serializers.DateTimeField(allow_null=True)
|
||||
completed_at = serializers.DateTimeField(allow_null=True)
|
||||
|
||||
@@ -4,7 +4,7 @@ URL patterns for the library DRF API.
|
||||
|
||||
from django.urls import path
|
||||
|
||||
from . import views
|
||||
from . import views, workspaces
|
||||
|
||||
app_name = "library-api"
|
||||
|
||||
@@ -28,4 +28,16 @@ urlpatterns = [
|
||||
# Concepts (Phase 3)
|
||||
path("concepts/", views.concept_list, name="concept-list"),
|
||||
path("concepts/<str:uid>/graph/", views.concept_graph, name="concept-graph"),
|
||||
# Workspaces (Daedalus integration)
|
||||
path("workspaces/", workspaces.workspace_create, name="workspace-create"),
|
||||
path(
|
||||
"workspaces/<str:workspace_id>/",
|
||||
workspaces.workspace_detail_or_delete,
|
||||
name="workspace-detail",
|
||||
),
|
||||
# Ingest (Daedalus integration)
|
||||
path("ingest/", views.ingest_create, name="ingest-create"),
|
||||
path("jobs/", views.ingest_job_list, name="ingest-job-list"),
|
||||
path("jobs/<str:job_id>/", views.ingest_job_detail, name="ingest-job-detail"),
|
||||
path("jobs/<str:job_id>/retry/", views.ingest_job_retry, name="ingest-job-retry"),
|
||||
]
|
||||
|
||||
@@ -21,6 +21,8 @@ from library.content_types import get_library_type_config
|
||||
from .serializers import (
|
||||
CollectionSerializer,
|
||||
ConceptSerializer,
|
||||
IngestJobSerializer,
|
||||
IngestRequestSerializer,
|
||||
ItemSerializer,
|
||||
LibrarySerializer,
|
||||
SearchRequestSerializer,
|
||||
@@ -456,6 +458,7 @@ def search(request):
|
||||
library_uid=data.get("library_uid") or None,
|
||||
library_type=data.get("library_type") or None,
|
||||
collection_uid=data.get("collection_uid") or None,
|
||||
workspace_id=data.get("workspace_id") or None,
|
||||
search_types=data.get("search_types", ["vector", "fulltext", "graph"]),
|
||||
limit=data.get("limit", getattr(django_settings, "SEARCH_DEFAULT_LIMIT", 20)),
|
||||
vector_top_k=getattr(django_settings, "SEARCH_VECTOR_TOP_K", 50),
|
||||
@@ -487,6 +490,7 @@ def search_vector(request):
|
||||
library_uid=data.get("library_uid") or None,
|
||||
library_type=data.get("library_type") or None,
|
||||
collection_uid=data.get("collection_uid") or None,
|
||||
workspace_id=data.get("workspace_id") or None,
|
||||
search_types=["vector"],
|
||||
limit=data.get("limit", 20),
|
||||
vector_top_k=getattr(django_settings, "SEARCH_VECTOR_TOP_K", 50),
|
||||
@@ -517,6 +521,7 @@ def search_fulltext(request):
|
||||
library_uid=data.get("library_uid") or None,
|
||||
library_type=data.get("library_type") or None,
|
||||
collection_uid=data.get("collection_uid") or None,
|
||||
workspace_id=data.get("workspace_id") or None,
|
||||
search_types=["fulltext"],
|
||||
limit=data.get("limit", 20),
|
||||
fulltext_top_k=getattr(django_settings, "SEARCH_FULLTEXT_TOP_K", 30),
|
||||
@@ -635,3 +640,196 @@ def concept_graph(request, uid):
|
||||
{"detail": f"Failed: {exc}"},
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ingest API (Daedalus integration)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@api_view(["POST"])
|
||||
@permission_classes([IsAuthenticated])
|
||||
def ingest_create(request):
|
||||
"""
|
||||
Accept a file (already in S3) for ingestion + embedding.
|
||||
|
||||
Daedalus calls this after committing a file to its own S3 bucket.
|
||||
We resolve the target Library (by workspace_id or library_uid),
|
||||
enforce idempotency on (library, source_ref, content_hash), create
|
||||
an IngestJob row, and dispatch a Celery task.
|
||||
|
||||
Idempotency:
|
||||
- Same source_ref + same content_hash → return existing completed job
|
||||
(no new task dispatched).
|
||||
- Same source_ref + different content_hash → dispatch a new task that
|
||||
will supersede the prior Item.
|
||||
- New source_ref → fresh ingest.
|
||||
"""
|
||||
import uuid
|
||||
|
||||
from library.models import IngestJob, Library
|
||||
from library.tasks import ingest_from_daedalus
|
||||
|
||||
serializer = IngestRequestSerializer(data=request.data)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
data = serializer.validated_data
|
||||
|
||||
# --- Resolve target Library ---
|
||||
workspace_id = data.get("workspace_id") or ""
|
||||
library_uid = data.get("library_uid") or ""
|
||||
if workspace_id:
|
||||
try:
|
||||
lib = Library.nodes.get(workspace_id=workspace_id)
|
||||
except Library.DoesNotExist:
|
||||
return Response(
|
||||
{"detail": f"Workspace '{workspace_id}' not registered."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
else:
|
||||
try:
|
||||
lib = Library.nodes.get(uid=library_uid)
|
||||
except Library.DoesNotExist:
|
||||
return Response(
|
||||
{"detail": f"Library '{library_uid}' not found."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
# --- Idempotency check on (library, source_ref, content_hash) ---
|
||||
source_ref = data.get("source_ref") or ""
|
||||
content_hash = data["content_hash"]
|
||||
|
||||
if source_ref:
|
||||
existing = (
|
||||
IngestJob.objects
|
||||
.filter(
|
||||
library_uid=lib.uid,
|
||||
source_ref=source_ref,
|
||||
content_hash=content_hash,
|
||||
status__in=["pending", "processing", "completed"],
|
||||
)
|
||||
.order_by("-created_at")
|
||||
.first()
|
||||
)
|
||||
if existing is not None:
|
||||
logger.info(
|
||||
"Ingest idempotent hit job_id=%s source_ref=%s status=%s",
|
||||
existing.id, source_ref, existing.status,
|
||||
)
|
||||
return Response(
|
||||
IngestJobSerializer(existing).data,
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
# --- Create job + dispatch ---
|
||||
job = IngestJob.objects.create(
|
||||
id=f"job_{uuid.uuid4().hex[:24]}",
|
||||
library_uid=lib.uid,
|
||||
s3_key=data["s3_key"],
|
||||
title=data["title"],
|
||||
file_type=data.get("file_type", ""),
|
||||
file_size=data.get("file_size", 0),
|
||||
content_hash=content_hash,
|
||||
source=data.get("source", ""),
|
||||
source_ref=source_ref,
|
||||
collection_uid=data.get("collection_uid", ""),
|
||||
status="pending",
|
||||
progress="queued",
|
||||
)
|
||||
|
||||
try:
|
||||
async_result = ingest_from_daedalus.delay(job.id)
|
||||
job.celery_task_id = async_result.id
|
||||
job.save(update_fields=["celery_task_id"])
|
||||
except Exception as exc:
|
||||
logger.error("Failed to dispatch ingest task job_id=%s: %s", job.id, exc)
|
||||
job.status = "failed"
|
||||
job.error = f"dispatch failed: {exc}"
|
||||
job.save(update_fields=["status", "error"])
|
||||
return Response(
|
||||
IngestJobSerializer(job).data,
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Ingest dispatched job_id=%s library_uid=%s source_ref=%s task_id=%s",
|
||||
job.id, lib.uid, source_ref, job.celery_task_id,
|
||||
)
|
||||
|
||||
return Response(
|
||||
IngestJobSerializer(job).data,
|
||||
status=status.HTTP_202_ACCEPTED,
|
||||
)
|
||||
|
||||
|
||||
@api_view(["GET"])
|
||||
@permission_classes([IsAuthenticated])
|
||||
def ingest_job_detail(request, job_id):
|
||||
"""Get the current status of an IngestJob."""
|
||||
from library.models import IngestJob
|
||||
|
||||
try:
|
||||
job = IngestJob.objects.get(pk=job_id)
|
||||
except IngestJob.DoesNotExist:
|
||||
return Response(
|
||||
{"detail": "Job not found."}, status=status.HTTP_404_NOT_FOUND
|
||||
)
|
||||
|
||||
return Response(IngestJobSerializer(job).data)
|
||||
|
||||
|
||||
@api_view(["POST"])
|
||||
@permission_classes([IsAuthenticated])
|
||||
def ingest_job_retry(request, job_id):
|
||||
"""Re-dispatch a failed IngestJob."""
|
||||
from library.models import IngestJob
|
||||
from library.tasks import ingest_from_daedalus
|
||||
|
||||
try:
|
||||
job = IngestJob.objects.get(pk=job_id)
|
||||
except IngestJob.DoesNotExist:
|
||||
return Response(
|
||||
{"detail": "Job not found."}, status=status.HTTP_404_NOT_FOUND
|
||||
)
|
||||
|
||||
if job.status not in ("failed", "completed"):
|
||||
return Response(
|
||||
{"detail": f"Job is currently {job.status}; cannot retry."},
|
||||
status=status.HTTP_409_CONFLICT,
|
||||
)
|
||||
|
||||
job.status = "pending"
|
||||
job.progress = "queued"
|
||||
job.error = ""
|
||||
job.save(update_fields=["status", "progress", "error"])
|
||||
|
||||
async_result = ingest_from_daedalus.delay(job.id)
|
||||
job.celery_task_id = async_result.id
|
||||
job.save(update_fields=["celery_task_id"])
|
||||
|
||||
logger.info("Ingest retry dispatched job_id=%s task_id=%s", job.id, async_result.id)
|
||||
return Response(IngestJobSerializer(job).data, status=status.HTTP_202_ACCEPTED)
|
||||
|
||||
|
||||
@api_view(["GET"])
|
||||
@permission_classes([IsAuthenticated])
|
||||
def ingest_job_list(request):
|
||||
"""List recent IngestJob rows, optionally filtered by status / library_uid."""
|
||||
from library.models import IngestJob
|
||||
|
||||
qs = IngestJob.objects.all()
|
||||
status_filter = request.query_params.get("status")
|
||||
library_uid = request.query_params.get("library_uid")
|
||||
limit = min(int(request.query_params.get("limit", 50)), 200)
|
||||
|
||||
if status_filter:
|
||||
qs = qs.filter(status=status_filter)
|
||||
if library_uid:
|
||||
qs = qs.filter(library_uid=library_uid)
|
||||
|
||||
jobs = list(qs.order_by("-created_at")[:limit])
|
||||
return Response(
|
||||
{
|
||||
"jobs": IngestJobSerializer(jobs, many=True).data,
|
||||
"count": len(jobs),
|
||||
}
|
||||
)
|
||||
|
||||
217
mnemosyne/library/api/workspaces.py
Normal file
217
mnemosyne/library/api/workspaces.py
Normal file
@@ -0,0 +1,217 @@
|
||||
"""
|
||||
Workspace lifecycle endpoints for the Daedalus integration.
|
||||
|
||||
A "workspace" in Mnemosyne is a Library scoped to a Daedalus workspace UUID.
|
||||
It uses the same Library node as a global library; the difference is that
|
||||
`workspace_id` is set, and search must filter on it.
|
||||
|
||||
These endpoints are called by the Daedalus backend (HTTP Basic auth as
|
||||
the `daedalus-service` user). Daedalus owns the workspace_id; Mnemosyne
|
||||
just persists what Daedalus tells it.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from neomodel import db
|
||||
from rest_framework import status
|
||||
from rest_framework.decorators import api_view, permission_classes
|
||||
from rest_framework.permissions import IsAuthenticated
|
||||
from rest_framework.response import Response
|
||||
|
||||
from library.content_types import get_library_type_config
|
||||
|
||||
from .serializers import WorkspaceCreateSerializer, WorkspaceStatusSerializer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _serialize_workspace(lib):
|
||||
"""Build a WorkspaceStatus payload from a Library node + chunk/item counts."""
|
||||
counts, _ = db.cypher_query(
|
||||
"MATCH (l:Library {workspace_id: $wsid}) "
|
||||
"OPTIONAL MATCH (l)-[:CONTAINS]->(:Collection)-[:CONTAINS]->(i:Item) "
|
||||
"OPTIONAL MATCH (i)-[:HAS_CHUNK]->(c:Chunk) "
|
||||
"RETURN count(DISTINCT i) AS item_count, count(DISTINCT c) AS chunk_count",
|
||||
{"wsid": lib.workspace_id},
|
||||
)
|
||||
item_count = counts[0][0] if counts else 0
|
||||
chunk_count = counts[0][1] if counts else 0
|
||||
|
||||
return {
|
||||
"workspace_id": lib.workspace_id,
|
||||
"library_uid": lib.uid,
|
||||
"name": lib.name,
|
||||
"library_type": lib.library_type,
|
||||
"description": lib.description or "",
|
||||
"item_count": item_count,
|
||||
"chunk_count": chunk_count,
|
||||
"created_at": lib.created_at,
|
||||
}
|
||||
|
||||
|
||||
@api_view(["POST"])
|
||||
@permission_classes([IsAuthenticated])
|
||||
def workspace_create(request):
|
||||
"""
|
||||
Create a workspace Library, idempotently.
|
||||
|
||||
A POST with a `workspace_id` already in use returns the existing
|
||||
workspace (200) — not an error. The library_type is frozen at first
|
||||
create; subsequent calls are not allowed to change it.
|
||||
"""
|
||||
from library.models import Library
|
||||
|
||||
serializer = WorkspaceCreateSerializer(data=request.data)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
data = serializer.validated_data
|
||||
|
||||
# Idempotent path: workspace already exists.
|
||||
try:
|
||||
existing = Library.nodes.get(workspace_id=data["workspace_id"])
|
||||
except Library.DoesNotExist:
|
||||
existing = None
|
||||
|
||||
if existing is not None:
|
||||
if existing.library_type != data["library_type"]:
|
||||
return Response(
|
||||
{
|
||||
"detail": (
|
||||
"library_type is immutable for an existing workspace "
|
||||
f"(have '{existing.library_type}', "
|
||||
f"got '{data['library_type']}')."
|
||||
)
|
||||
},
|
||||
status=status.HTTP_409_CONFLICT,
|
||||
)
|
||||
logger.info(
|
||||
"Workspace already exists workspace_id=%s library_uid=%s",
|
||||
data["workspace_id"], existing.uid,
|
||||
)
|
||||
return Response(
|
||||
WorkspaceStatusSerializer(_serialize_workspace(existing)).data,
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
defaults = get_library_type_config(data["library_type"])
|
||||
lib = Library(
|
||||
name=data["name"],
|
||||
library_type=data["library_type"],
|
||||
description=data.get("description", ""),
|
||||
workspace_id=data["workspace_id"],
|
||||
chunking_config=defaults["chunking_config"],
|
||||
embedding_instruction=defaults["embedding_instruction"],
|
||||
reranker_instruction=defaults["reranker_instruction"],
|
||||
llm_context_prompt=defaults["llm_context_prompt"],
|
||||
)
|
||||
lib.save()
|
||||
logger.info(
|
||||
"Workspace created workspace_id=%s library_uid=%s library_type=%s",
|
||||
data["workspace_id"], lib.uid, lib.library_type,
|
||||
)
|
||||
|
||||
return Response(
|
||||
WorkspaceStatusSerializer(_serialize_workspace(lib)).data,
|
||||
status=status.HTTP_201_CREATED,
|
||||
)
|
||||
|
||||
|
||||
@api_view(["GET", "DELETE"])
|
||||
@permission_classes([IsAuthenticated])
|
||||
def workspace_detail_or_delete(request, workspace_id):
|
||||
"""
|
||||
GET: return workspace status (item/chunk counts, metadata).
|
||||
|
||||
DELETE: delete the workspace Library and everything reachable AND unique
|
||||
to it. Concept-safe: orphan-only Concept GC happens at the end.
|
||||
Concepts referenced by other libraries (workspace or global) are preserved.
|
||||
"""
|
||||
from library.models import Library
|
||||
|
||||
if request.method == "GET":
|
||||
try:
|
||||
lib = Library.nodes.get(workspace_id=workspace_id)
|
||||
except Library.DoesNotExist:
|
||||
return Response(
|
||||
{"detail": "Workspace not found."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
return Response(WorkspaceStatusSerializer(_serialize_workspace(lib)).data)
|
||||
|
||||
# DELETE — idempotent: a missing workspace returns 204.
|
||||
try:
|
||||
lib = Library.nodes.get(workspace_id=workspace_id)
|
||||
except Library.DoesNotExist:
|
||||
return Response(status=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
library_uid = lib.uid
|
||||
library_name = lib.name
|
||||
|
||||
# Step 1-4: delete chunks, items, collections, then the library itself.
|
||||
# We collect Item s3_keys first so the caller can clean up S3
|
||||
# asynchronously (a future enhancement — for now, the keys are logged).
|
||||
s3_rows, _ = db.cypher_query(
|
||||
"MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(:Collection)"
|
||||
"-[:CONTAINS]->(i:Item) RETURN i.uid, i.s3_key",
|
||||
{"wsid": workspace_id},
|
||||
)
|
||||
item_s3_keys = [(r[0], r[1]) for r in s3_rows if r[1]]
|
||||
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(:Collection)
|
||||
-[:CONTAINS]->(i:Item)-[:HAS_CHUNK]->(c:Chunk)
|
||||
DETACH DELETE c
|
||||
""",
|
||||
{"wsid": workspace_id},
|
||||
)
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(:Collection)
|
||||
-[:CONTAINS]->(i:Item)-[:HAS_IMAGE]->(img:Image)
|
||||
OPTIONAL MATCH (img)-[:HAS_EMBEDDING]->(emb:ImageEmbedding)
|
||||
DETACH DELETE img, emb
|
||||
""",
|
||||
{"wsid": workspace_id},
|
||||
)
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(:Collection)
|
||||
-[:CONTAINS]->(i:Item)
|
||||
DETACH DELETE i
|
||||
""",
|
||||
{"wsid": workspace_id},
|
||||
)
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(col:Collection)
|
||||
DETACH DELETE col
|
||||
""",
|
||||
{"wsid": workspace_id},
|
||||
)
|
||||
db.cypher_query(
|
||||
"MATCH (l:Library {workspace_id: $wsid}) DETACH DELETE l",
|
||||
{"wsid": workspace_id},
|
||||
)
|
||||
|
||||
# Step 5: orphan Concept garbage collection.
|
||||
orphan_result, _ = db.cypher_query(
|
||||
"""
|
||||
MATCH (con:Concept)
|
||||
WHERE NOT (con)<-[:REFERENCES]-() AND NOT (con)<-[:MENTIONS]-()
|
||||
AND NOT (con)<-[:DEPICTS]-()
|
||||
WITH con
|
||||
DETACH DELETE con
|
||||
RETURN count(con) AS deleted
|
||||
"""
|
||||
)
|
||||
orphans_deleted = orphan_result[0][0] if orphan_result else 0
|
||||
|
||||
logger.info(
|
||||
"Workspace deleted workspace_id=%s library_uid=%s name=%s "
|
||||
"items=%d orphans_deleted=%d",
|
||||
workspace_id, library_uid, library_name,
|
||||
len(item_s3_keys), orphans_deleted,
|
||||
)
|
||||
|
||||
return Response(status=status.HTTP_204_NO_CONTENT)
|
||||
@@ -210,6 +210,69 @@ LIBRARY_TYPE_DEFAULTS = {
|
||||
"4) Context clues about when and where this was taken or created."
|
||||
),
|
||||
},
|
||||
"business": {
|
||||
"chunking_config": {
|
||||
"strategy": "section_aware",
|
||||
"chunk_size": 640,
|
||||
"chunk_overlap": 96,
|
||||
"respect_boundaries": ["section", "subsection", "list", "table"],
|
||||
},
|
||||
"embedding_instruction": (
|
||||
"Represent this passage from a business document for retrieval. "
|
||||
"Focus on value propositions, positioning, pricing, scope of work, "
|
||||
"client outcomes, and commercial commitments."
|
||||
),
|
||||
"reranker_instruction": (
|
||||
"Re-rank passages from business documents based on commercial relevance. "
|
||||
"Prioritize value framing, deliverables, client outcomes, and specific "
|
||||
"pricing or scope language."
|
||||
),
|
||||
"llm_context_prompt": (
|
||||
"The following excerpts are from business documents (proposals, marketing, "
|
||||
"sales, strategy). Interpret in commercial context. Distinguish positioning "
|
||||
"claims from committed deliverables. Preserve numbers, scope language, and "
|
||||
"client names exactly as written."
|
||||
),
|
||||
"vision_prompt": (
|
||||
"Analyze this image from a business document. Identify:\n"
|
||||
"1) Image type (logo, chart, diagram, screenshot, photograph, table).\n"
|
||||
"2) What it depicts — brand marks, data, organizational structure, products.\n"
|
||||
"3) Any visible text — company names, figures, captions, headings.\n"
|
||||
"4) The commercial purpose — positioning, pricing, capability demonstration."
|
||||
),
|
||||
},
|
||||
"finance": {
|
||||
"chunking_config": {
|
||||
"strategy": "section_aware",
|
||||
"chunk_size": 512,
|
||||
"chunk_overlap": 64,
|
||||
"respect_boundaries": ["section", "table", "row", "paragraph"],
|
||||
},
|
||||
"embedding_instruction": (
|
||||
"Represent this passage from a financial document for retrieval. "
|
||||
"Focus on accounts, instruments, dates, amounts, balances, and "
|
||||
"analytical commentary."
|
||||
),
|
||||
"reranker_instruction": (
|
||||
"Re-rank passages from financial documents based on relevance to the query. "
|
||||
"Prioritize the matching account, instrument, time period, and figures."
|
||||
),
|
||||
"llm_context_prompt": (
|
||||
"The following excerpts are from financial documents (statements, tax "
|
||||
"documents, market commentary, planning). Distinguish factual figures "
|
||||
"(statements, transactions, balances) from opinion (forecasts, commentary). "
|
||||
"Quote numbers, dates, and account identifiers exactly as they appear. "
|
||||
"Do not infer, round, or fabricate financial figures. If a figure is not "
|
||||
"present in the excerpts, say so explicitly."
|
||||
),
|
||||
"vision_prompt": (
|
||||
"Analyze this image from a financial document. Identify:\n"
|
||||
"1) Image type (chart, table, statement scan, dashboard screenshot, receipt).\n"
|
||||
"2) What it depicts — account, instrument, time period, data series.\n"
|
||||
"3) Any visible text — figures, dates, account identifiers, labels.\n"
|
||||
"4) Whether the data is factual (statement) or analytical (forecast/commentary)."
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -219,7 +282,7 @@ def get_library_type_config(library_type):
|
||||
|
||||
Args:
|
||||
library_type: One of 'fiction', 'nonfiction', 'technical', 'music',
|
||||
'film', 'art', 'journal'
|
||||
'film', 'art', 'journal', 'business', 'finance'
|
||||
|
||||
Returns:
|
||||
dict with keys: chunking_config, embedding_instruction,
|
||||
|
||||
45
mnemosyne/library/migrations/0001_initial.py
Normal file
45
mnemosyne/library/migrations/0001_initial.py
Normal file
@@ -0,0 +1,45 @@
|
||||
# Generated by Django 5.2.13 on 2026-04-28 12:36
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='IngestJob',
|
||||
fields=[
|
||||
('id', models.CharField(max_length=64, primary_key=True, serialize=False)),
|
||||
('item_uid', models.CharField(blank=True, db_index=True, max_length=64)),
|
||||
('library_uid', models.CharField(db_index=True, max_length=64)),
|
||||
('celery_task_id', models.CharField(blank=True, max_length=255)),
|
||||
('status', models.CharField(choices=[('pending', 'Pending'), ('processing', 'Processing'), ('completed', 'Completed'), ('failed', 'Failed')], db_index=True, default='pending', max_length=20)),
|
||||
('progress', models.CharField(default='queued', max_length=50)),
|
||||
('error', models.TextField(blank=True, null=True)),
|
||||
('retry_count', models.PositiveIntegerField(default=0)),
|
||||
('chunks_created', models.PositiveIntegerField(default=0)),
|
||||
('concepts_extracted', models.PositiveIntegerField(default=0)),
|
||||
('embedding_model', models.CharField(blank=True, max_length=100)),
|
||||
('content_hash', models.CharField(blank=True, db_index=True, max_length=64)),
|
||||
('source', models.CharField(default='', max_length=50)),
|
||||
('source_ref', models.CharField(blank=True, db_index=True, max_length=200)),
|
||||
('s3_key', models.CharField(max_length=500)),
|
||||
('title', models.CharField(blank=True, max_length=500)),
|
||||
('file_type', models.CharField(blank=True, max_length=50)),
|
||||
('file_size', models.PositiveBigIntegerField(default=0)),
|
||||
('collection_uid', models.CharField(blank=True, max_length=64)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('started_at', models.DateTimeField(blank=True, null=True)),
|
||||
('completed_at', models.DateTimeField(blank=True, null=True)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-created_at'],
|
||||
'indexes': [models.Index(fields=['status', '-created_at'], name='library_ing_status_9c95b2_idx'), models.Index(fields=['source', 'source_ref'], name='library_ing_source_a48684_idx')],
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -1,11 +1,16 @@
|
||||
"""
|
||||
Neo4j graph models for the Mnemosyne content library.
|
||||
Models for the Mnemosyne content library.
|
||||
|
||||
All content data (libraries, collections, items, chunks, concepts, images)
|
||||
lives in Neo4j as a knowledge graph. These models use neomodel's StructuredNode
|
||||
OGM — they do NOT participate in Django's ORM or migrations.
|
||||
Most content (libraries, collections, items, chunks, concepts, images)
|
||||
lives in Neo4j as a knowledge graph via neomodel StructuredNode. These do
|
||||
NOT participate in Django's ORM or migrations.
|
||||
|
||||
The IngestJob model at the bottom of this file is the exception: it tracks
|
||||
the lifecycle of asynchronous ingestion requests (file → embedding pipeline)
|
||||
in PostgreSQL via Django's ORM.
|
||||
"""
|
||||
|
||||
from django.db import models
|
||||
from neomodel import (
|
||||
ArrayProperty,
|
||||
DateTimeProperty,
|
||||
@@ -50,8 +55,14 @@ class Library(StructuredNode):
|
||||
"""
|
||||
Top-level container representing a content library.
|
||||
|
||||
Each library has a type (fiction, technical, music, film, art, journal)
|
||||
that drives chunking strategy, embedding instructions, and LLM prompts.
|
||||
Each library has a type (fiction, nonfiction, technical, music, film,
|
||||
art, journal, business, finance) that drives chunking strategy,
|
||||
embedding instructions, and LLM prompts.
|
||||
|
||||
A library may be either *global* (workspace_id is null — searchable
|
||||
across the whole instance) or *workspace-scoped* (workspace_id set —
|
||||
visible only to agents inside that Daedalus workspace). Scoping is
|
||||
enforced structurally by every search query.
|
||||
"""
|
||||
|
||||
uid = UniqueIdProperty()
|
||||
@@ -66,10 +77,16 @@ class Library(StructuredNode):
|
||||
"film": "Film",
|
||||
"art": "Art",
|
||||
"journal": "Journal",
|
||||
"business": "Business",
|
||||
"finance": "Finance",
|
||||
},
|
||||
)
|
||||
description = StringProperty(default="")
|
||||
|
||||
# Daedalus workspace UUID this library is scoped to. Null for global
|
||||
# libraries. Unique-indexed so a workspace cannot have two libraries.
|
||||
workspace_id = StringProperty(unique_index=True, required=False)
|
||||
|
||||
# Content-type configuration
|
||||
chunking_config = JSONProperty(default={})
|
||||
embedding_instruction = StringProperty(default="")
|
||||
@@ -270,3 +287,78 @@ class ImageEmbedding(StructuredNode):
|
||||
|
||||
def __str__(self):
|
||||
return f"ImageEmbedding ({self.uid})"
|
||||
|
||||
|
||||
# --- Django ORM models (PostgreSQL) ---
|
||||
|
||||
|
||||
class IngestJob(models.Model):
|
||||
"""
|
||||
Tracks the lifecycle of an asynchronous ingestion + embedding job.
|
||||
|
||||
Created when an external client (e.g. Daedalus) posts a file via the
|
||||
REST ingest API. The Celery worker reads and updates this row as the
|
||||
job moves through fetch / chunk / embed / graph stages.
|
||||
|
||||
Idempotency: a (library, source_ref, content_hash) triple uniquely
|
||||
identifies a piece of content. A second POST with the same triple
|
||||
returns the existing job; a POST with the same source_ref but a new
|
||||
content_hash supersedes the prior Item.
|
||||
"""
|
||||
|
||||
STATUS_CHOICES = [
|
||||
("pending", "Pending"),
|
||||
("processing", "Processing"),
|
||||
("completed", "Completed"),
|
||||
("failed", "Failed"),
|
||||
]
|
||||
|
||||
id = models.CharField(max_length=64, primary_key=True)
|
||||
item_uid = models.CharField(max_length=64, db_index=True, blank=True)
|
||||
library_uid = models.CharField(max_length=64, db_index=True)
|
||||
celery_task_id = models.CharField(max_length=255, blank=True)
|
||||
|
||||
status = models.CharField(
|
||||
max_length=20,
|
||||
choices=STATUS_CHOICES,
|
||||
default="pending",
|
||||
db_index=True,
|
||||
)
|
||||
progress = models.CharField(max_length=50, default="queued")
|
||||
error = models.TextField(blank=True, null=True)
|
||||
retry_count = models.PositiveIntegerField(default=0)
|
||||
|
||||
chunks_created = models.PositiveIntegerField(default=0)
|
||||
concepts_extracted = models.PositiveIntegerField(default=0)
|
||||
embedding_model = models.CharField(max_length=100, blank=True)
|
||||
|
||||
# The file's content hash (sha256). Used for idempotency: a second
|
||||
# ingest with the same source_ref + same hash is a no-op; a second
|
||||
# ingest with the same source_ref + different hash supersedes.
|
||||
content_hash = models.CharField(max_length=64, db_index=True, blank=True)
|
||||
|
||||
# Where the file came from. For Daedalus: source="daedalus",
|
||||
# source_ref="<workspace_id>/<file_id>".
|
||||
source = models.CharField(max_length=50, default="")
|
||||
source_ref = models.CharField(max_length=200, blank=True, db_index=True)
|
||||
s3_key = models.CharField(max_length=500)
|
||||
|
||||
# Optional metadata carried forward to the Item node.
|
||||
title = models.CharField(max_length=500, blank=True)
|
||||
file_type = models.CharField(max_length=50, blank=True)
|
||||
file_size = models.PositiveBigIntegerField(default=0)
|
||||
collection_uid = models.CharField(max_length=64, blank=True)
|
||||
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
started_at = models.DateTimeField(null=True, blank=True)
|
||||
completed_at = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["-created_at"]
|
||||
indexes = [
|
||||
models.Index(fields=["status", "-created_at"]),
|
||||
models.Index(fields=["source", "source_ref"]),
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"IngestJob {self.id} [{self.status}]"
|
||||
|
||||
70
mnemosyne/library/services/daedalus_s3.py
Normal file
70
mnemosyne/library/services/daedalus_s3.py
Normal file
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
Cross-bucket S3 helper for ingesting files from the Daedalus S3 bucket
|
||||
into Mnemosyne's own bucket.
|
||||
|
||||
Daedalus uploads files to its own bucket (configured per Daedalus deployment)
|
||||
and posts an ingest request to Mnemosyne with the s3_key. This module fetches
|
||||
that file using read-only Daedalus credentials and writes it to Mnemosyne's
|
||||
bucket via the standard `default_storage` backend so the rest of the pipeline
|
||||
(parsing, chunking, embedding) works unchanged.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import boto3
|
||||
from django.conf import settings
|
||||
from django.core.files.base import ContentFile
|
||||
from django.core.files.storage import default_storage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _daedalus_s3_client():
|
||||
"""Build a boto3 S3 client pointed at the Daedalus bucket."""
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.DAEDALUS_S3_ENDPOINT_URL or None,
|
||||
aws_access_key_id=settings.DAEDALUS_S3_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=settings.DAEDALUS_S3_SECRET_ACCESS_KEY,
|
||||
region_name=settings.DAEDALUS_S3_REGION_NAME,
|
||||
use_ssl=settings.DAEDALUS_S3_USE_SSL,
|
||||
verify=settings.DAEDALUS_S3_VERIFY,
|
||||
)
|
||||
|
||||
|
||||
def fetch_from_daedalus(daedalus_s3_key: str) -> bytes:
|
||||
"""
|
||||
Read a file from the Daedalus S3 bucket.
|
||||
|
||||
:param daedalus_s3_key: Object key in the Daedalus bucket.
|
||||
:returns: File bytes.
|
||||
:raises: botocore exceptions on failure (caller decides retry).
|
||||
"""
|
||||
client = _daedalus_s3_client()
|
||||
bucket = settings.DAEDALUS_S3_BUCKET_NAME
|
||||
logger.debug(
|
||||
"Fetching from Daedalus S3 bucket=%s key=%s", bucket, daedalus_s3_key
|
||||
)
|
||||
response = client.get_object(Bucket=bucket, Key=daedalus_s3_key)
|
||||
data = response["Body"].read()
|
||||
logger.info(
|
||||
"Fetched from Daedalus S3 bucket=%s key=%s size=%d",
|
||||
bucket, daedalus_s3_key, len(data),
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def copy_into_mnemosyne(data: bytes, mnemosyne_s3_key: str) -> str:
|
||||
"""
|
||||
Write bytes into Mnemosyne's S3 bucket via the default_storage backend.
|
||||
|
||||
:param data: File bytes (already in memory).
|
||||
:param mnemosyne_s3_key: Target object key in Mnemosyne's bucket.
|
||||
:returns: The actual key written (may differ from requested if
|
||||
`file_overwrite=False` and the key existed).
|
||||
"""
|
||||
saved_key = default_storage.save(mnemosyne_s3_key, ContentFile(data))
|
||||
logger.info(
|
||||
"Wrote to Mnemosyne S3 key=%s size=%d", saved_key, len(data),
|
||||
)
|
||||
return saved_key
|
||||
@@ -26,15 +26,32 @@ from .fusion import ImageSearchResult, SearchCandidate, reciprocal_rank_fusion
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Workspace scoping clause appended to every search Cypher query.
|
||||
#
|
||||
# A request with workspace_id set returns ONLY that workspace's content.
|
||||
# A request with workspace_id null returns ONLY global content (libraries
|
||||
# with no workspace_id). There is no third mode.
|
||||
_WORKSPACE_SCOPE_CLAUSE = (
|
||||
" AND ($workspace_id IS NULL AND lib.workspace_id IS NULL OR "
|
||||
"lib.workspace_id = $workspace_id)"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchRequest:
|
||||
"""Parameters for a search query."""
|
||||
"""Parameters for a search query.
|
||||
|
||||
Scope is single-mode: a request is either workspace-scoped (workspace_id
|
||||
set) or global (workspace_id is None). There is no parameter combination
|
||||
that returns both workspace and global content in one call.
|
||||
"""
|
||||
|
||||
query: str
|
||||
query_image: Optional[bytes] = None
|
||||
library_uid: Optional[str] = None
|
||||
library_type: Optional[str] = None
|
||||
collection_uid: Optional[str] = None
|
||||
workspace_id: Optional[str] = None
|
||||
search_types: list[str] = field(
|
||||
default_factory=lambda: ["vector", "fulltext", "graph"]
|
||||
)
|
||||
@@ -45,6 +62,18 @@ class SearchRequest:
|
||||
rerank: bool = True
|
||||
include_images: bool = True
|
||||
|
||||
def __post_init__(self):
|
||||
# Normalize empty strings to None so "" doesn't slip through as
|
||||
# truthy at the Cypher boundary.
|
||||
if self.workspace_id == "":
|
||||
self.workspace_id = None
|
||||
if self.library_uid == "":
|
||||
self.library_uid = None
|
||||
if self.library_type == "":
|
||||
self.library_type = None
|
||||
if self.collection_uid == "":
|
||||
self.collection_uid = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResponse:
|
||||
@@ -243,7 +272,8 @@ class SearchService:
|
||||
top_k = request.vector_top_k
|
||||
|
||||
# Build Cypher with optional filtering
|
||||
cypher = """
|
||||
cypher = (
|
||||
"""
|
||||
CALL db.index.vector.queryNodes('chunk_embedding_index', $top_k, $query_vector)
|
||||
YIELD node AS chunk, score
|
||||
MATCH (item:Item)-[:HAS_CHUNK]->(chunk)
|
||||
@@ -251,13 +281,17 @@ class SearchService:
|
||||
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
||||
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
||||
AND ($collection_uid IS NULL OR col.uid = $collection_uid)
|
||||
"""
|
||||
+ _WORKSPACE_SCOPE_CLAUSE
|
||||
+ """
|
||||
RETURN chunk.uid AS chunk_uid, chunk.text_preview AS text_preview,
|
||||
chunk.chunk_s3_key AS chunk_s3_key, chunk.chunk_index AS chunk_index,
|
||||
item.uid AS item_uid, item.title AS item_title,
|
||||
lib.library_type AS library_type, score
|
||||
ORDER BY score DESC
|
||||
LIMIT $top_k
|
||||
"""
|
||||
"""
|
||||
)
|
||||
|
||||
params = {
|
||||
"top_k": top_k,
|
||||
@@ -265,6 +299,7 @@ class SearchService:
|
||||
"library_uid": request.library_uid,
|
||||
"library_type": request.library_type,
|
||||
"collection_uid": request.collection_uid,
|
||||
"workspace_id": request.workspace_id,
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -348,7 +383,8 @@ class SearchService:
|
||||
candidates: dict[str, SearchCandidate],
|
||||
):
|
||||
"""Search chunk_text_fulltext index and add to candidates dict."""
|
||||
cypher = """
|
||||
cypher = (
|
||||
"""
|
||||
CALL db.index.fulltext.queryNodes('chunk_text_fulltext', $query)
|
||||
YIELD node AS chunk, score
|
||||
MATCH (item:Item)-[:HAS_CHUNK]->(chunk)
|
||||
@@ -356,13 +392,17 @@ class SearchService:
|
||||
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
||||
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
||||
AND ($collection_uid IS NULL OR col.uid = $collection_uid)
|
||||
"""
|
||||
+ _WORKSPACE_SCOPE_CLAUSE
|
||||
+ """
|
||||
RETURN chunk.uid AS chunk_uid, chunk.text_preview AS text_preview,
|
||||
chunk.chunk_s3_key AS chunk_s3_key, chunk.chunk_index AS chunk_index,
|
||||
item.uid AS item_uid, item.title AS item_title,
|
||||
lib.library_type AS library_type, score
|
||||
ORDER BY score DESC
|
||||
LIMIT $top_k
|
||||
"""
|
||||
"""
|
||||
)
|
||||
|
||||
params = {
|
||||
"query": request.query,
|
||||
@@ -370,6 +410,7 @@ class SearchService:
|
||||
"library_uid": request.library_uid,
|
||||
"library_type": request.library_type,
|
||||
"collection_uid": request.collection_uid,
|
||||
"workspace_id": request.workspace_id,
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -402,7 +443,8 @@ class SearchService:
|
||||
candidates: dict[str, SearchCandidate],
|
||||
):
|
||||
"""Search concept_name_fulltext and traverse to chunks."""
|
||||
cypher = """
|
||||
cypher = (
|
||||
"""
|
||||
CALL db.index.fulltext.queryNodes('concept_name_fulltext', $query)
|
||||
YIELD node AS concept, score AS concept_score
|
||||
MATCH (chunk:Chunk)-[:MENTIONS]->(concept)
|
||||
@@ -410,6 +452,9 @@ class SearchService:
|
||||
MATCH (lib:Library)-[:CONTAINS]->(:Collection)-[:CONTAINS]->(item)
|
||||
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
||||
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
||||
"""
|
||||
+ _WORKSPACE_SCOPE_CLAUSE
|
||||
+ """
|
||||
RETURN chunk.uid AS chunk_uid, chunk.text_preview AS text_preview,
|
||||
chunk.chunk_s3_key AS chunk_s3_key, chunk.chunk_index AS chunk_index,
|
||||
item.uid AS item_uid, item.title AS item_title,
|
||||
@@ -417,13 +462,15 @@ class SearchService:
|
||||
concept_score * 0.8 AS score
|
||||
ORDER BY score DESC
|
||||
LIMIT $top_k
|
||||
"""
|
||||
"""
|
||||
)
|
||||
|
||||
params = {
|
||||
"query": request.query,
|
||||
"top_k": top_k,
|
||||
"library_uid": request.library_uid,
|
||||
"library_type": request.library_type,
|
||||
"workspace_id": request.workspace_id,
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -465,7 +512,8 @@ class SearchService:
|
||||
"""
|
||||
start = time.time()
|
||||
|
||||
cypher = """
|
||||
cypher = (
|
||||
"""
|
||||
CALL db.index.fulltext.queryNodes('concept_name_fulltext', $query)
|
||||
YIELD node AS concept, score AS concept_score
|
||||
WITH concept, concept_score
|
||||
@@ -476,6 +524,9 @@ class SearchService:
|
||||
MATCH (lib:Library)-[:CONTAINS]->(:Collection)-[:CONTAINS]->(item)
|
||||
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
||||
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
||||
"""
|
||||
+ _WORKSPACE_SCOPE_CLAUSE
|
||||
+ """
|
||||
WITH chunk, item, lib,
|
||||
max(concept_score) AS score,
|
||||
collect(DISTINCT concept.name)[..5] AS concept_names
|
||||
@@ -486,13 +537,15 @@ class SearchService:
|
||||
score, concept_names
|
||||
ORDER BY score DESC
|
||||
LIMIT $limit
|
||||
"""
|
||||
"""
|
||||
)
|
||||
|
||||
params = {
|
||||
"query": request.query,
|
||||
"limit": request.fulltext_top_k,
|
||||
"library_uid": request.library_uid,
|
||||
"library_type": request.library_type,
|
||||
"workspace_id": request.workspace_id,
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -550,7 +603,8 @@ class SearchService:
|
||||
"""
|
||||
start = time.time()
|
||||
|
||||
cypher = """
|
||||
cypher = (
|
||||
"""
|
||||
CALL db.index.vector.queryNodes('image_embedding_index', $top_k, $query_vector)
|
||||
YIELD node AS emb_node, score
|
||||
MATCH (img:Image)-[:HAS_EMBEDDING]->(emb_node)
|
||||
@@ -558,19 +612,24 @@ class SearchService:
|
||||
MATCH (lib:Library)-[:CONTAINS]->(:Collection)-[:CONTAINS]->(item)
|
||||
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
||||
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
||||
"""
|
||||
+ _WORKSPACE_SCOPE_CLAUSE
|
||||
+ """
|
||||
RETURN img.uid AS image_uid, img.image_type AS image_type,
|
||||
img.description AS description, img.s3_key AS s3_key,
|
||||
item.uid AS item_uid, item.title AS item_title,
|
||||
score
|
||||
ORDER BY score DESC
|
||||
LIMIT 10
|
||||
"""
|
||||
"""
|
||||
)
|
||||
|
||||
params = {
|
||||
"top_k": 10,
|
||||
"query_vector": query_vector,
|
||||
"library_uid": request.library_uid,
|
||||
"library_type": request.library_type,
|
||||
"workspace_id": request.workspace_id,
|
||||
}
|
||||
|
||||
try:
|
||||
|
||||
@@ -10,6 +10,7 @@ import logging
|
||||
|
||||
from celery import shared_task
|
||||
from django.core.cache import cache
|
||||
from neomodel import db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -280,3 +281,212 @@ def _resolve_user(user_id: int = None):
|
||||
return User.objects.get(pk=user_id)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ingest task (Daedalus integration)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@shared_task(
|
||||
name="library.tasks.ingest_from_daedalus",
|
||||
bind=True,
|
||||
queue="embedding",
|
||||
max_retries=3,
|
||||
default_retry_delay=60,
|
||||
acks_late=True,
|
||||
)
|
||||
def ingest_from_daedalus(self, job_id: str):
|
||||
"""
|
||||
Process a single IngestJob: fetch from Daedalus S3 → create Item →
|
||||
run embedding pipeline → mark complete.
|
||||
|
||||
Idempotent on (library_uid, source_ref, content_hash) — handled in the
|
||||
REST view that creates the IngestJob, so by the time this task runs the
|
||||
job either represents new content or a content_hash-changed re-ingest.
|
||||
|
||||
For a content_hash-changed re-ingest, the prior Item with the same
|
||||
source_ref is deleted before the new one is processed (ensures no
|
||||
stale chunks linger).
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from library.models import IngestJob, Item, Library
|
||||
from library.services.daedalus_s3 import (
|
||||
copy_into_mnemosyne,
|
||||
fetch_from_daedalus,
|
||||
)
|
||||
from library.services.pipeline import EmbeddingPipeline
|
||||
|
||||
logger.info(
|
||||
"Task ingest_from_daedalus starting job_id=%s task_id=%s",
|
||||
job_id, self.request.id,
|
||||
)
|
||||
|
||||
try:
|
||||
job = IngestJob.objects.get(pk=job_id)
|
||||
except IngestJob.DoesNotExist:
|
||||
logger.error("IngestJob not found job_id=%s", job_id)
|
||||
return {"success": False, "error": "job_not_found"}
|
||||
|
||||
job.status = "processing"
|
||||
job.progress = "fetching"
|
||||
job.started_at = datetime.now(timezone.utc)
|
||||
job.celery_task_id = self.request.id
|
||||
job.save(update_fields=["status", "progress", "started_at", "celery_task_id"])
|
||||
|
||||
try:
|
||||
# --- 1. Resolve target Library ---
|
||||
try:
|
||||
lib = Library.nodes.get(uid=job.library_uid)
|
||||
except Library.DoesNotExist:
|
||||
raise RuntimeError(f"Library not found: {job.library_uid}")
|
||||
|
||||
# --- 2. Supersede prior Item with same source_ref but different hash ---
|
||||
prior_item_uid = None
|
||||
if job.source_ref:
|
||||
rows, _ = db.cypher_query(
|
||||
"""
|
||||
MATCH (l:Library {uid: $library_uid})-[:CONTAINS]->(:Collection)
|
||||
-[:CONTAINS]->(i:Item)
|
||||
WHERE i.metadata IS NOT NULL
|
||||
AND i.metadata CONTAINS $source_ref_marker
|
||||
RETURN i.uid LIMIT 1
|
||||
""",
|
||||
{
|
||||
"library_uid": lib.uid,
|
||||
"source_ref_marker": f'"source_ref": "{job.source_ref}"',
|
||||
},
|
||||
)
|
||||
if rows:
|
||||
prior_item_uid = rows[0][0]
|
||||
logger.info(
|
||||
"Superseding prior Item job_id=%s prior_item_uid=%s",
|
||||
job_id, prior_item_uid,
|
||||
)
|
||||
_delete_item_and_chunks(prior_item_uid)
|
||||
|
||||
# --- 3. Fetch from Daedalus, copy into Mnemosyne bucket ---
|
||||
job.progress = "copying"
|
||||
job.save(update_fields=["progress"])
|
||||
|
||||
data = fetch_from_daedalus(job.s3_key)
|
||||
|
||||
# --- 4. Create Item node ---
|
||||
ext = (job.file_type or "bin").lstrip(".").lower() or "bin"
|
||||
item = Item(
|
||||
title=job.title,
|
||||
file_type=ext,
|
||||
file_size=len(data),
|
||||
content_hash=job.content_hash,
|
||||
embedding_status="pending",
|
||||
metadata={
|
||||
"source": job.source,
|
||||
"source_ref": job.source_ref,
|
||||
},
|
||||
)
|
||||
item.save()
|
||||
|
||||
mnemosyne_s3_key = f"items/{item.uid}/original.{ext}"
|
||||
copy_into_mnemosyne(data, mnemosyne_s3_key)
|
||||
item.s3_key = mnemosyne_s3_key
|
||||
item.save()
|
||||
|
||||
# --- 5. Connect to library/collection ---
|
||||
col = _resolve_or_create_default_collection(lib, job.collection_uid)
|
||||
col.items.connect(item)
|
||||
|
||||
job.item_uid = item.uid
|
||||
job.save(update_fields=["item_uid"])
|
||||
|
||||
# --- 6. Run the embedding pipeline ---
|
||||
job.progress = "embedding"
|
||||
job.save(update_fields=["progress"])
|
||||
|
||||
def progress_cb(percent, message):
|
||||
_update_progress(self, percent, message)
|
||||
|
||||
pipeline = EmbeddingPipeline(user=None)
|
||||
result = pipeline.process_item(item.uid, progress_callback=progress_cb)
|
||||
|
||||
# --- 7. Mark complete ---
|
||||
job.status = "completed"
|
||||
job.progress = "done"
|
||||
job.chunks_created = result.get("chunks_created", 0)
|
||||
job.concepts_extracted = result.get("concepts_extracted", 0)
|
||||
job.embedding_model = result.get("embedding_model", "")
|
||||
job.completed_at = datetime.now(timezone.utc)
|
||||
job.save()
|
||||
|
||||
logger.info(
|
||||
"Task ingest_from_daedalus completed job_id=%s item_uid=%s "
|
||||
"chunks=%d concepts=%d",
|
||||
job_id, item.uid, job.chunks_created, job.concepts_extracted,
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"job_id": job_id,
|
||||
"item_uid": item.uid,
|
||||
**result,
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Task ingest_from_daedalus failed job_id=%s: %s",
|
||||
job_id, exc, exc_info=True,
|
||||
)
|
||||
if self.request.retries < self.max_retries:
|
||||
job.retry_count = self.request.retries + 1
|
||||
job.save(update_fields=["retry_count"])
|
||||
raise self.retry(exc=exc)
|
||||
|
||||
job.status = "failed"
|
||||
job.error = str(exc)
|
||||
job.completed_at = datetime.now(timezone.utc)
|
||||
job.save(update_fields=["status", "error", "completed_at"])
|
||||
return {"success": False, "job_id": job_id, "error": str(exc)}
|
||||
|
||||
|
||||
def _delete_item_and_chunks(item_uid: str):
|
||||
"""Delete an Item, its chunks, and its images. Concept GC is workspace-delete only."""
|
||||
db.cypher_query(
|
||||
"""
|
||||
MATCH (i:Item {uid: $uid})
|
||||
OPTIONAL MATCH (i)-[:HAS_CHUNK]->(c:Chunk)
|
||||
OPTIONAL MATCH (i)-[:HAS_IMAGE]->(img:Image)
|
||||
OPTIONAL MATCH (img)-[:HAS_EMBEDDING]->(emb:ImageEmbedding)
|
||||
DETACH DELETE c, img, emb, i
|
||||
""",
|
||||
{"uid": item_uid},
|
||||
)
|
||||
|
||||
|
||||
def _resolve_or_create_default_collection(lib, collection_uid: str = ""):
|
||||
"""
|
||||
Find or create the default Collection for a Library.
|
||||
|
||||
Daedalus integration creates one Collection per Library, named "default".
|
||||
Explicit collection_uid is honored if provided.
|
||||
"""
|
||||
from library.models import Collection
|
||||
|
||||
if collection_uid:
|
||||
try:
|
||||
return Collection.nodes.get(uid=collection_uid)
|
||||
except Collection.DoesNotExist:
|
||||
pass
|
||||
|
||||
# Look for an existing "default" collection in this library
|
||||
rows, _ = db.cypher_query(
|
||||
"MATCH (l:Library {uid: $library_uid})-[:CONTAINS]->(c:Collection {name: 'default'}) "
|
||||
"RETURN c.uid LIMIT 1",
|
||||
{"library_uid": lib.uid},
|
||||
)
|
||||
if rows:
|
||||
return Collection.nodes.get(uid=rows[0][0])
|
||||
|
||||
col = Collection(name="default", description="Default collection")
|
||||
col.save()
|
||||
lib.collections.connect(col)
|
||||
col.library.connect(lib)
|
||||
return col
|
||||
|
||||
@@ -13,7 +13,17 @@ from library.content_types import LIBRARY_TYPE_DEFAULTS, get_library_type_config
|
||||
class LibraryTypeDefaultsTests(TestCase):
|
||||
"""Tests for the LIBRARY_TYPE_DEFAULTS registry."""
|
||||
|
||||
EXPECTED_TYPES = {"fiction", "nonfiction", "technical", "music", "film", "art", "journal"}
|
||||
EXPECTED_TYPES = {
|
||||
"fiction",
|
||||
"nonfiction",
|
||||
"technical",
|
||||
"music",
|
||||
"film",
|
||||
"art",
|
||||
"journal",
|
||||
"business",
|
||||
"finance",
|
||||
}
|
||||
|
||||
def test_all_expected_types_present(self):
|
||||
for lib_type in self.EXPECTED_TYPES:
|
||||
@@ -105,6 +115,16 @@ class VisionPromptTests(TestCase):
|
||||
prompt = config["vision_prompt"].lower()
|
||||
self.assertIn("historical", prompt)
|
||||
|
||||
def test_business_vision_prompt_mentions_logo_or_chart(self):
|
||||
config = get_library_type_config("business")
|
||||
prompt = config["vision_prompt"].lower()
|
||||
self.assertTrue("logo" in prompt or "chart" in prompt)
|
||||
|
||||
def test_finance_llm_context_forbids_fabrication(self):
|
||||
config = get_library_type_config("finance")
|
||||
prompt = config["llm_context_prompt"].lower()
|
||||
self.assertIn("fabricate", prompt)
|
||||
|
||||
|
||||
class GetLibraryTypeConfigTests(TestCase):
|
||||
"""Tests for the get_library_type_config helper."""
|
||||
|
||||
71
mnemosyne/library/tests/test_search_scoping.py
Normal file
71
mnemosyne/library/tests/test_search_scoping.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""
|
||||
Tests for workspace scoping in SearchRequest and the Cypher scope clause.
|
||||
|
||||
These exercise the dataclass-level normalization and the construction
|
||||
of Cypher parameter dicts. The actual Cypher execution against Neo4j
|
||||
is validated by the manual end-to-end test plan.
|
||||
"""
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
from library.services.search import _WORKSPACE_SCOPE_CLAUSE, SearchRequest
|
||||
|
||||
|
||||
class SearchRequestScopingTests(TestCase):
|
||||
"""SearchRequest workspace_id behavior."""
|
||||
|
||||
def test_default_workspace_id_is_none(self):
|
||||
req = SearchRequest(query="hello")
|
||||
self.assertIsNone(req.workspace_id)
|
||||
|
||||
def test_explicit_workspace_id_preserved(self):
|
||||
req = SearchRequest(query="hello", workspace_id="ws_abc")
|
||||
self.assertEqual(req.workspace_id, "ws_abc")
|
||||
|
||||
def test_empty_string_workspace_id_normalized_to_none(self):
|
||||
"""Empty strings must NOT slip through as a truthy filter at the Cypher boundary."""
|
||||
req = SearchRequest(query="hello", workspace_id="")
|
||||
self.assertIsNone(req.workspace_id)
|
||||
|
||||
def test_empty_string_library_uid_normalized_to_none(self):
|
||||
req = SearchRequest(query="hello", library_uid="")
|
||||
self.assertIsNone(req.library_uid)
|
||||
|
||||
def test_empty_string_library_type_normalized_to_none(self):
|
||||
req = SearchRequest(query="hello", library_type="")
|
||||
self.assertIsNone(req.library_type)
|
||||
|
||||
def test_empty_string_collection_uid_normalized_to_none(self):
|
||||
req = SearchRequest(query="hello", collection_uid="")
|
||||
self.assertIsNone(req.collection_uid)
|
||||
|
||||
|
||||
class WorkspaceScopeClauseTests(TestCase):
|
||||
"""Sanity checks on the Cypher snippet itself.
|
||||
|
||||
The clause must produce two distinct, non-overlapping result sets:
|
||||
1. workspace_id IS NULL → only global libraries (lib.workspace_id IS NULL)
|
||||
2. workspace_id = X → only libraries with workspace_id = X
|
||||
|
||||
A "leaks both" bug would be a Cypher OR that fails to bracket properly.
|
||||
Verifying the literal string here is a cheap regression guard against
|
||||
refactors that accidentally change the operator precedence.
|
||||
"""
|
||||
|
||||
def test_clause_references_lib_workspace_id(self):
|
||||
self.assertIn("lib.workspace_id", _WORKSPACE_SCOPE_CLAUSE)
|
||||
|
||||
def test_clause_references_workspace_id_param(self):
|
||||
self.assertIn("$workspace_id", _WORKSPACE_SCOPE_CLAUSE)
|
||||
|
||||
def test_clause_handles_both_modes(self):
|
||||
"""Both 'IS NULL' and '=' branches must be present."""
|
||||
self.assertIn("IS NULL", _WORKSPACE_SCOPE_CLAUSE)
|
||||
self.assertIn("=", _WORKSPACE_SCOPE_CLAUSE)
|
||||
|
||||
def test_clause_starts_with_AND_so_it_appends_safely(self):
|
||||
"""The clause is appended to existing WHERE filters."""
|
||||
self.assertTrue(
|
||||
_WORKSPACE_SCOPE_CLAUSE.lstrip().startswith("AND"),
|
||||
f"Clause must start with AND: {_WORKSPACE_SCOPE_CLAUSE!r}",
|
||||
)
|
||||
210
mnemosyne/library/tests/test_workspaces_api.py
Normal file
210
mnemosyne/library/tests/test_workspaces_api.py
Normal file
@@ -0,0 +1,210 @@
|
||||
"""
|
||||
Tests for workspace and ingest REST endpoints.
|
||||
|
||||
These exercise serializer validation, idempotency rules, and Django ORM
|
||||
behavior for IngestJob. The Cypher-touching paths (Library node CRUD,
|
||||
search scoping) require Neo4j and are validated by the manual end-to-end
|
||||
test plan, not these unit tests.
|
||||
"""
|
||||
|
||||
from django.test import TestCase
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from library.api.serializers import (
|
||||
IngestRequestSerializer,
|
||||
WorkspaceCreateSerializer,
|
||||
)
|
||||
from library.models import IngestJob
|
||||
|
||||
|
||||
class WorkspaceCreateSerializerTests(TestCase):
|
||||
"""Validation rules for the create-workspace payload."""
|
||||
|
||||
def test_minimal_payload_validates(self):
|
||||
s = WorkspaceCreateSerializer(
|
||||
data={
|
||||
"workspace_id": "ws_abc",
|
||||
"name": "My Workspace",
|
||||
"library_type": "technical",
|
||||
}
|
||||
)
|
||||
self.assertTrue(s.is_valid(), s.errors)
|
||||
|
||||
def test_business_type_accepted(self):
|
||||
s = WorkspaceCreateSerializer(
|
||||
data={
|
||||
"workspace_id": "ws_abc",
|
||||
"name": "Sales",
|
||||
"library_type": "business",
|
||||
}
|
||||
)
|
||||
self.assertTrue(s.is_valid(), s.errors)
|
||||
|
||||
def test_finance_type_accepted(self):
|
||||
s = WorkspaceCreateSerializer(
|
||||
data={
|
||||
"workspace_id": "ws_abc",
|
||||
"name": "Money",
|
||||
"library_type": "finance",
|
||||
}
|
||||
)
|
||||
self.assertTrue(s.is_valid(), s.errors)
|
||||
|
||||
def test_unknown_type_rejected(self):
|
||||
s = WorkspaceCreateSerializer(
|
||||
data={
|
||||
"workspace_id": "ws_abc",
|
||||
"name": "x",
|
||||
"library_type": "miscellaneous",
|
||||
}
|
||||
)
|
||||
self.assertFalse(s.is_valid())
|
||||
self.assertIn("library_type", s.errors)
|
||||
|
||||
def test_workspace_id_required(self):
|
||||
s = WorkspaceCreateSerializer(
|
||||
data={"name": "x", "library_type": "technical"}
|
||||
)
|
||||
self.assertFalse(s.is_valid())
|
||||
self.assertIn("workspace_id", s.errors)
|
||||
|
||||
|
||||
class IngestRequestSerializerTests(TestCase):
|
||||
"""Validation rules for the ingest payload."""
|
||||
|
||||
BASE_PAYLOAD = {
|
||||
"s3_key": "workspaces/ws/files/f/x.pdf",
|
||||
"title": "Q4 Report",
|
||||
"content_hash": "a" * 64,
|
||||
}
|
||||
|
||||
def test_workspace_id_only_validates(self):
|
||||
s = IngestRequestSerializer(
|
||||
data={**self.BASE_PAYLOAD, "workspace_id": "ws_abc"}
|
||||
)
|
||||
self.assertTrue(s.is_valid(), s.errors)
|
||||
|
||||
def test_library_uid_only_validates(self):
|
||||
s = IngestRequestSerializer(
|
||||
data={**self.BASE_PAYLOAD, "library_uid": "lib_xyz"}
|
||||
)
|
||||
self.assertTrue(s.is_valid(), s.errors)
|
||||
|
||||
def test_neither_workspace_nor_library_rejected(self):
|
||||
s = IngestRequestSerializer(data=self.BASE_PAYLOAD)
|
||||
self.assertFalse(s.is_valid())
|
||||
# ValidationError on the whole payload, not a specific field
|
||||
self.assertIn("non_field_errors", s.errors)
|
||||
|
||||
def test_content_hash_required(self):
|
||||
s = IngestRequestSerializer(
|
||||
data={
|
||||
"s3_key": "x",
|
||||
"title": "y",
|
||||
"workspace_id": "ws_abc",
|
||||
}
|
||||
)
|
||||
self.assertFalse(s.is_valid())
|
||||
self.assertIn("content_hash", s.errors)
|
||||
|
||||
|
||||
class IngestJobModelTests(TestCase):
|
||||
"""IngestJob persists and queries correctly."""
|
||||
|
||||
def test_create_with_minimal_fields(self):
|
||||
job = IngestJob.objects.create(
|
||||
id="job_test1",
|
||||
library_uid="lib_xyz",
|
||||
s3_key="x.pdf",
|
||||
)
|
||||
self.assertEqual(job.status, "pending")
|
||||
self.assertEqual(job.progress, "queued")
|
||||
self.assertEqual(job.retry_count, 0)
|
||||
self.assertEqual(job.chunks_created, 0)
|
||||
|
||||
def test_idempotency_query_pattern(self):
|
||||
"""The idempotency query in ingest_create uses (library, source_ref, hash)."""
|
||||
IngestJob.objects.create(
|
||||
id="job_a",
|
||||
library_uid="lib_xyz",
|
||||
source_ref="ws_a/file_1",
|
||||
content_hash="h1",
|
||||
s3_key="a.pdf",
|
||||
status="completed",
|
||||
)
|
||||
IngestJob.objects.create(
|
||||
id="job_b",
|
||||
library_uid="lib_xyz",
|
||||
source_ref="ws_a/file_1",
|
||||
content_hash="h2", # different hash — supersedes
|
||||
s3_key="a.pdf",
|
||||
status="completed",
|
||||
)
|
||||
|
||||
# Same library + source_ref + h1 → finds job_a
|
||||
match = IngestJob.objects.filter(
|
||||
library_uid="lib_xyz",
|
||||
source_ref="ws_a/file_1",
|
||||
content_hash="h1",
|
||||
).first()
|
||||
self.assertIsNotNone(match)
|
||||
self.assertEqual(match.id, "job_a")
|
||||
|
||||
# h2 → finds job_b
|
||||
match = IngestJob.objects.filter(
|
||||
library_uid="lib_xyz",
|
||||
source_ref="ws_a/file_1",
|
||||
content_hash="h2",
|
||||
).first()
|
||||
self.assertEqual(match.id, "job_b")
|
||||
|
||||
# Different source_ref → no match
|
||||
match = IngestJob.objects.filter(
|
||||
library_uid="lib_xyz",
|
||||
source_ref="ws_a/file_2",
|
||||
content_hash="h1",
|
||||
).first()
|
||||
self.assertIsNone(match)
|
||||
|
||||
|
||||
class IngestEndpointAuthTests(TestCase):
|
||||
"""Auth boundary on the ingest endpoint (matches the existing convention)."""
|
||||
|
||||
def setUp(self):
|
||||
self.client = APIClient()
|
||||
|
||||
def test_ingest_requires_auth(self):
|
||||
response = self.client.post(
|
||||
"/library/api/ingest/",
|
||||
{
|
||||
"s3_key": "x",
|
||||
"title": "y",
|
||||
"workspace_id": "ws_abc",
|
||||
"content_hash": "a" * 64,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
self.assertIn(response.status_code, [401, 403])
|
||||
|
||||
|
||||
class WorkspaceEndpointAuthTests(TestCase):
|
||||
"""Auth on workspace endpoints."""
|
||||
|
||||
def setUp(self):
|
||||
self.client = APIClient()
|
||||
|
||||
def test_workspace_create_requires_auth(self):
|
||||
response = self.client.post(
|
||||
"/library/api/workspaces/",
|
||||
{"workspace_id": "ws_a", "name": "x", "library_type": "technical"},
|
||||
format="json",
|
||||
)
|
||||
self.assertIn(response.status_code, [401, 403])
|
||||
|
||||
def test_workspace_get_requires_auth(self):
|
||||
response = self.client.get("/library/api/workspaces/ws_a/")
|
||||
self.assertIn(response.status_code, [401, 403])
|
||||
|
||||
def test_workspace_delete_requires_auth(self):
|
||||
response = self.client.delete("/library/api/workspaces/ws_a/")
|
||||
self.assertIn(response.status_code, [401, 403])
|
||||
@@ -16,14 +16,23 @@ class MCPTokenAdmin(admin.ModelAdmin):
|
||||
]
|
||||
list_filter = ["is_active"]
|
||||
search_fields = ["name", "user__email", "user__username"]
|
||||
readonly_fields = ["token", "last_used_at", "created_at", "updated_at"]
|
||||
readonly_fields = ["token_hash", "last_used_at", "created_at", "updated_at"]
|
||||
fieldsets = (
|
||||
(None, {"fields": ("user", "name", "is_active")}),
|
||||
("Restrictions", {"fields": ("allowed_tools", "expires_at")}),
|
||||
("Token (shown once at creation)", {"fields": ("token",)}),
|
||||
(
|
||||
"Token (hashed at rest — plaintext is shown only once at creation)",
|
||||
{"fields": ("token_hash",)},
|
||||
),
|
||||
("Audit", {"fields": ("last_used_at", "created_at", "updated_at")}),
|
||||
)
|
||||
|
||||
@admin.display(description="Token")
|
||||
def masked_token(self, obj):
|
||||
return obj.get_masked_token()
|
||||
|
||||
def has_add_permission(self, request):
|
||||
# Tokens must be created via the dashboard or management command
|
||||
# so the plaintext can be surfaced to the user. Adding via admin
|
||||
# would persist a hash with no plaintext ever shown.
|
||||
return False
|
||||
|
||||
@@ -12,7 +12,7 @@ from fastmcp.server.dependencies import get_http_request
|
||||
from fastmcp.server.middleware import Middleware, MiddlewareContext
|
||||
|
||||
from .metrics import mcp_auth_failures_total
|
||||
from .models import MCPToken
|
||||
from .models import MCPToken, hash_token
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -25,9 +25,17 @@ class MCPAuthError(Exception):
|
||||
|
||||
|
||||
def resolve_mcp_user(token_string: str):
|
||||
"""Resolve a bearer token to (user, MCPToken). Raises MCPAuthError on any failure."""
|
||||
"""Resolve a bearer token to (user, MCPToken). Raises MCPAuthError on any failure.
|
||||
|
||||
Hashes the incoming bearer and looks up by the hash — plaintext is never
|
||||
stored or compared directly.
|
||||
"""
|
||||
try:
|
||||
token = MCPToken.objects.select_related("user").get(token=token_string)
|
||||
token = (
|
||||
MCPToken.objects
|
||||
.select_related("user")
|
||||
.get(token_hash=hash_token(token_string))
|
||||
)
|
||||
except MCPToken.DoesNotExist:
|
||||
raise MCPAuthError("Invalid MCP token.")
|
||||
|
||||
|
||||
85
mnemosyne/mcp_server/forms.py
Normal file
85
mnemosyne/mcp_server/forms.py
Normal file
@@ -0,0 +1,85 @@
|
||||
"""Forms for the MCP token self-service dashboard."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
|
||||
from django import forms
|
||||
|
||||
from .models import MCPToken
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _registered_tool_names() -> list[str]:
|
||||
"""Pull the list of registered MCP tool names from the FastMCP instance.
|
||||
|
||||
Cached at module level — the tool registry is fixed at server startup.
|
||||
Importing here (rather than at module import) avoids circulars when the
|
||||
server module imports the dashboard pieces transitively.
|
||||
"""
|
||||
from .server import mcp
|
||||
|
||||
tools = asyncio.run(mcp.get_tools())
|
||||
return sorted(tools.keys())
|
||||
|
||||
|
||||
def _tool_choices() -> list[tuple[str, str]]:
|
||||
return [(name, name) for name in _registered_tool_names()]
|
||||
|
||||
|
||||
class MCPTokenCreateForm(forms.Form):
|
||||
"""Generate a new bearer token. The token value itself is server-generated."""
|
||||
|
||||
name = forms.CharField(
|
||||
max_length=100,
|
||||
widget=forms.TextInput(attrs={
|
||||
"class": "input input-bordered w-full",
|
||||
"placeholder": "e.g. Claude Desktop, CI script",
|
||||
}),
|
||||
)
|
||||
expires_at = forms.DateTimeField(
|
||||
required=False,
|
||||
widget=forms.DateTimeInput(attrs={
|
||||
"class": "input input-bordered w-full",
|
||||
"type": "datetime-local",
|
||||
}),
|
||||
help_text="Leave blank for no expiry.",
|
||||
)
|
||||
allowed_tools = forms.MultipleChoiceField(
|
||||
required=False,
|
||||
widget=forms.CheckboxSelectMultiple(attrs={"class": "checkbox checkbox-sm"}),
|
||||
help_text="Leave all unchecked to permit every tool.",
|
||||
)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields["allowed_tools"].choices = _tool_choices()
|
||||
|
||||
|
||||
class MCPTokenEditForm(forms.ModelForm):
|
||||
"""Edit token metadata. The hashed token itself cannot be edited."""
|
||||
|
||||
allowed_tools = forms.MultipleChoiceField(
|
||||
required=False,
|
||||
widget=forms.CheckboxSelectMultiple(attrs={"class": "checkbox checkbox-sm"}),
|
||||
help_text="Leave all unchecked to permit every tool.",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = MCPToken
|
||||
fields = ["name", "is_active", "expires_at", "allowed_tools"]
|
||||
widgets = {
|
||||
"name": forms.TextInput(attrs={"class": "input input-bordered w-full"}),
|
||||
"is_active": forms.CheckboxInput(attrs={"class": "toggle toggle-primary"}),
|
||||
"expires_at": forms.DateTimeInput(attrs={
|
||||
"class": "input input-bordered w-full",
|
||||
"type": "datetime-local",
|
||||
}),
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields["allowed_tools"].choices = _tool_choices()
|
||||
if self.instance and self.instance.pk:
|
||||
self.fields["allowed_tools"].initial = self.instance.allowed_tools or []
|
||||
@@ -57,7 +57,7 @@ class Command(BaseCommand):
|
||||
raise CommandError("--expires-days must be at least 1.")
|
||||
expires_at = timezone.now() + timedelta(days=options["expires_days"])
|
||||
|
||||
token = MCPToken.objects.create(
|
||||
token, plaintext = MCPToken.objects.create_token(
|
||||
user=user,
|
||||
name=options["name"],
|
||||
allowed_tools=allowed_tools,
|
||||
@@ -73,5 +73,5 @@ class Command(BaseCommand):
|
||||
self.stdout.write(" Tools: (all)")
|
||||
if expires_at:
|
||||
self.stdout.write(f" Expires: {expires_at.isoformat()}")
|
||||
self.stdout.write(self.style.WARNING(" Token (shown once):"))
|
||||
self.stdout.write(f" {token.token}")
|
||||
self.stdout.write(self.style.WARNING(" Token (shown once — store it now):"))
|
||||
self.stdout.write(f" {plaintext}")
|
||||
|
||||
43
mnemosyne/mcp_server/migrations/0002_hash_token.py
Normal file
43
mnemosyne/mcp_server/migrations/0002_hash_token.py
Normal file
@@ -0,0 +1,43 @@
|
||||
"""Hash MCPToken values at rest.
|
||||
|
||||
Renames ``token`` → ``token_hash`` and rewrites any pre-existing plaintext
|
||||
values into SHA-256 hex digests in-place. Forward-only: hashing is one-way,
|
||||
so no reverse migration is provided.
|
||||
|
||||
Existing tokens issued before this migration keep working only because
|
||||
``resolve_mcp_user`` hashes the incoming bearer before lookup; the original
|
||||
plaintext the client holds still hashes to what we just wrote.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
def hash_existing_tokens(apps, schema_editor):
|
||||
MCPToken = apps.get_model("mcp_server", "MCPToken")
|
||||
for token in MCPToken.objects.all():
|
||||
plaintext = token.token_hash # post-rename, still holds original plaintext
|
||||
token.token_hash = hashlib.sha256(plaintext.encode("utf-8")).hexdigest()
|
||||
token.save(update_fields=["token_hash"])
|
||||
|
||||
|
||||
def noop_reverse(apps, schema_editor):
|
||||
# Cannot reverse a hash. Leaving as no-op so the schema can be rolled
|
||||
# back, but operators must understand any hashed rows are unrecoverable.
|
||||
pass
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("mcp_server", "0001_initial"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RenameField(
|
||||
model_name="mcptoken",
|
||||
old_name="token",
|
||||
new_name="token_hash",
|
||||
),
|
||||
migrations.RunPython(hash_existing_tokens, noop_reverse),
|
||||
]
|
||||
@@ -1,3 +1,4 @@
|
||||
import hashlib
|
||||
import secrets
|
||||
|
||||
from django.conf import settings
|
||||
@@ -5,15 +6,44 @@ from django.db import models
|
||||
from django.utils import timezone
|
||||
|
||||
|
||||
def hash_token(plaintext: str) -> str:
|
||||
"""SHA-256 hex digest of an MCP bearer token. 64 chars."""
|
||||
return hashlib.sha256(plaintext.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class MCPTokenManager(models.Manager):
|
||||
def create_token(self, *, user, name, allowed_tools=None, expires_at=None):
|
||||
"""Generate a new bearer token, store its hash, and return (instance, plaintext).
|
||||
|
||||
The plaintext is returned exactly once and is never persisted. Callers
|
||||
must surface it to the human and rely on the user to copy it; after
|
||||
this method returns, the plaintext is unrecoverable from the database.
|
||||
"""
|
||||
plaintext = secrets.token_urlsafe(48)
|
||||
instance = self.create(
|
||||
user=user,
|
||||
name=name,
|
||||
token_hash=hash_token(plaintext),
|
||||
allowed_tools=list(allowed_tools or []),
|
||||
expires_at=expires_at,
|
||||
)
|
||||
return instance, plaintext
|
||||
|
||||
|
||||
class MCPToken(models.Model):
|
||||
"""Bearer token for authenticating MCP tool calls. See docs/Pattern_Django-MCP_V1-00.md."""
|
||||
"""Bearer token for authenticating MCP tool calls.
|
||||
|
||||
Tokens are hashed at rest (SHA-256, 64-char hex). Plaintext exists only in
|
||||
memory at creation time, on the wire to the client, and in the user's own
|
||||
storage. A leaked database backup discloses no usable credentials.
|
||||
"""
|
||||
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="mcp_tokens",
|
||||
)
|
||||
token = models.CharField(max_length=64, unique=True, db_index=True)
|
||||
token_hash = models.CharField(max_length=64, unique=True, db_index=True)
|
||||
name = models.CharField(max_length=100)
|
||||
is_active = models.BooleanField(default=True)
|
||||
expires_at = models.DateTimeField(null=True, blank=True)
|
||||
@@ -22,17 +52,14 @@ class MCPToken(models.Model):
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
objects = MCPTokenManager()
|
||||
|
||||
class Meta:
|
||||
ordering = ["-created_at"]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.name} ({self.user})"
|
||||
|
||||
def save(self, **kwargs):
|
||||
if not self.token:
|
||||
self.token = secrets.token_urlsafe(48)
|
||||
super().save(**kwargs)
|
||||
|
||||
@property
|
||||
def is_valid(self) -> bool:
|
||||
if not self.is_active:
|
||||
@@ -51,6 +78,9 @@ class MCPToken(models.Model):
|
||||
self.save(update_fields=["last_used_at"])
|
||||
|
||||
def get_masked_token(self) -> str:
|
||||
if len(self.token) > 8:
|
||||
return f"{'*' * (len(self.token) - 8)}{self.token[-8:]}"
|
||||
return "*" * len(self.token)
|
||||
"""Token-id-style display for admin and dashboard.
|
||||
|
||||
Plaintext is unrecoverable, so we display the first 8 chars of the
|
||||
hash prefixed with `mcp_…`. Stable per token, never reveals plaintext.
|
||||
"""
|
||||
return f"mcp_…{self.token_hash[:8]}"
|
||||
|
||||
@@ -5,7 +5,11 @@ from __future__ import annotations
|
||||
from fastmcp import FastMCP
|
||||
|
||||
from .auth import MCPAuthMiddleware
|
||||
from .tools import register_discovery_tools, register_search_tools
|
||||
from .tools import (
|
||||
register_discovery_tools,
|
||||
register_health_tools,
|
||||
register_search_tools,
|
||||
)
|
||||
|
||||
INSTRUCTIONS = """\
|
||||
Mnemosyne is a content-type-aware, multimodal knowledge base. It indexes
|
||||
@@ -23,6 +27,8 @@ shapes how content is chunked, embedded, and re-ranked:
|
||||
- film — Scripts, synopses, stills.
|
||||
- art — Catalogs, descriptions, artwork itself.
|
||||
- journal — Personal entries; temporal/reflective.
|
||||
- business — Proposals, marketing, sales, strategy. Commercial context.
|
||||
- finance — Statements, tax, market commentary. Quote figures exactly.
|
||||
|
||||
Tools:
|
||||
- search Hybrid retrieval. Filter by library_uid, library_type,
|
||||
@@ -32,6 +38,7 @@ Tools:
|
||||
- list_libraries Discover libraries (and their library_type).
|
||||
- list_collections Discover collections, optionally per library.
|
||||
- list_items Discover indexed items (documents).
|
||||
- get_health Health check (used by Pallas/Daedalus pollers).
|
||||
|
||||
Workflow: list_libraries → search(query, library_type=...) → get_chunk(chunk_uid)
|
||||
when the preview isn't enough. The calling LLM is responsible for synthesis
|
||||
@@ -47,6 +54,7 @@ def build_server() -> FastMCP:
|
||||
mcp.add_middleware(MCPAuthMiddleware())
|
||||
register_search_tools(mcp)
|
||||
register_discovery_tools(mcp)
|
||||
register_health_tools(mcp)
|
||||
return mcp
|
||||
|
||||
|
||||
|
||||
63
mnemosyne/mcp_server/templates/mcp_server/tokens/create.html
Normal file
63
mnemosyne/mcp_server/templates/mcp_server/tokens/create.html
Normal file
@@ -0,0 +1,63 @@
|
||||
{% extends "themis/base.html" %}
|
||||
|
||||
{% block title %}Generate MCP Token — {{ themis_app_name }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="max-w-2xl mx-auto">
|
||||
<h1 class="text-2xl font-bold mb-6">Generate MCP Token</h1>
|
||||
|
||||
<div class="alert alert-info mb-6">
|
||||
<span>The token will be displayed once after creation. Save it before leaving the page — it cannot be recovered.</span>
|
||||
</div>
|
||||
|
||||
<form method="post" action="{% url 'mcp_server:mcp-token-create' %}">
|
||||
{% csrf_token %}
|
||||
|
||||
<div class="card bg-base-200 mb-6">
|
||||
<div class="card-body">
|
||||
<div class="form-control">
|
||||
<label class="label" for="id_name">
|
||||
<span class="label-text">Name</span>
|
||||
</label>
|
||||
{{ form.name }}
|
||||
<label class="label">
|
||||
<span class="label-text-alt opacity-60">A friendly label so you can identify this token later (e.g. “Claude Desktop”).</span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="form-control mt-4">
|
||||
<label class="label" for="id_expires_at">
|
||||
<span class="label-text">Expires at (optional)</span>
|
||||
</label>
|
||||
{{ form.expires_at }}
|
||||
<label class="label">
|
||||
<span class="label-text-alt opacity-60">{{ form.expires_at.help_text }}</span>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card bg-base-200 mb-6">
|
||||
<div class="card-body">
|
||||
<h2 class="card-title text-lg">Allowed Tools</h2>
|
||||
<p class="text-sm opacity-60 mb-2">{{ form.allowed_tools.help_text }}</p>
|
||||
<div class="space-y-1">
|
||||
{% for choice in form.allowed_tools %}
|
||||
<label class="label cursor-pointer justify-start gap-3">
|
||||
{{ choice.tag }}
|
||||
<span class="label-text font-mono">{{ choice.choice_label }}</span>
|
||||
</label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% if form.allowed_tools.errors %}
|
||||
<div class="text-error text-sm mt-2">{{ form.allowed_tools.errors }}</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex justify-between">
|
||||
<a href="{% url 'mcp_server:mcp-token-list' %}" class="btn btn-ghost">Cancel</a>
|
||||
<button type="submit" class="btn btn-primary">Generate Token</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,50 @@
|
||||
{% extends "themis/base.html" %}
|
||||
|
||||
{% block title %}MCP Token Created — {{ themis_app_name }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="max-w-2xl mx-auto">
|
||||
<h1 class="text-2xl font-bold mb-2">MCP Token Created</h1>
|
||||
<p class="opacity-60 mb-6">{{ token.name }}</p>
|
||||
|
||||
<div class="alert alert-warning mb-6">
|
||||
<span><strong>Save this token now.</strong> Once you leave this page it cannot be retrieved — only the hash is stored. If lost, you will need to generate a new one.</span>
|
||||
</div>
|
||||
|
||||
<div class="card bg-base-200 mb-6">
|
||||
<div class="card-body">
|
||||
<h2 class="card-title text-lg mb-2">Token</h2>
|
||||
<div class="font-mono break-all bg-base-300 p-4 rounded select-all" id="mcp-plaintext">{{ plaintext }}</div>
|
||||
<div class="card-actions justify-end mt-3">
|
||||
<button type="button" id="mcp-copy-btn" class="btn btn-sm btn-primary"
|
||||
onclick="(()=>{const t=document.getElementById('mcp-plaintext').textContent;navigator.clipboard.writeText(t).then(()=>{const b=document.getElementById('mcp-copy-btn');b.textContent='Copied!';setTimeout(()=>b.textContent='Copy to clipboard',2000);});})()">
|
||||
Copy to clipboard
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card bg-base-200 mb-6">
|
||||
<div class="card-body">
|
||||
<h2 class="card-title text-lg">Use it</h2>
|
||||
<p class="text-sm opacity-80 mb-3">
|
||||
Add the token to your MCP client config as a Bearer credential. Example for Claude Desktop:
|
||||
</p>
|
||||
<pre class="bg-base-300 p-4 rounded text-xs overflow-x-auto"><code>{
|
||||
"mcpServers": {
|
||||
"mnemosyne": {
|
||||
"url": "http://localhost:8001/mcp/",
|
||||
"headers": {
|
||||
"Authorization": "Bearer <paste token here>"
|
||||
}
|
||||
}
|
||||
}
|
||||
}</code></pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex justify-end">
|
||||
<a href="{% url 'mcp_server:mcp-token-list' %}" class="btn btn-primary">I’ve saved it — go to token list</a>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
81
mnemosyne/mcp_server/templates/mcp_server/tokens/detail.html
Normal file
81
mnemosyne/mcp_server/templates/mcp_server/tokens/detail.html
Normal file
@@ -0,0 +1,81 @@
|
||||
{% extends "themis/base.html" %}
|
||||
{% load humanize %}
|
||||
|
||||
{% block title %}{{ token.name }} — {{ themis_app_name }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="max-w-2xl mx-auto">
|
||||
<div class="flex items-center justify-between mb-6">
|
||||
<h1 class="text-2xl font-bold">{{ token.name }}</h1>
|
||||
<div class="flex gap-2">
|
||||
<a href="{% url 'mcp_server:mcp-token-edit' pk=token.pk %}" class="btn btn-ghost btn-sm">Edit</a>
|
||||
{% if token.is_active %}
|
||||
<form method="post" action="{% url 'mcp_server:mcp-token-revoke' pk=token.pk %}"
|
||||
onsubmit="return confirm('Revoke this token? It will no longer authenticate MCP requests.');">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="btn btn-warning btn-sm btn-outline">Revoke</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
<form method="post" action="{% url 'mcp_server:mcp-token-delete' pk=token.pk %}"
|
||||
onsubmit="return confirm('Delete this token permanently? This removes the audit trail.');">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="btn btn-error btn-sm btn-outline">Delete</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card bg-base-200 mb-6">
|
||||
<div class="card-body">
|
||||
<div class="grid grid-cols-2 gap-y-3">
|
||||
<div class="opacity-60">Token ID</div>
|
||||
<div class="font-mono">{{ token.get_masked_token }}</div>
|
||||
|
||||
<div class="opacity-60">Status</div>
|
||||
<div>
|
||||
{% if token.is_valid %}
|
||||
<span class="badge badge-success badge-sm">Active</span>
|
||||
{% elif not token.is_active %}
|
||||
<span class="badge badge-error badge-sm">Revoked</span>
|
||||
{% else %}
|
||||
<span class="badge badge-error badge-sm">Expired</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% if token.expires_at %}
|
||||
<div class="opacity-60">Expires</div>
|
||||
<div>{{ token.expires_at }}</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="opacity-60">Last Used</div>
|
||||
<div>
|
||||
{% if token.last_used_at %}
|
||||
{{ token.last_used_at }} <span class="opacity-60">({{ token.last_used_at|naturaltime }})</span>
|
||||
{% else %}
|
||||
<span class="opacity-60">Never</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="opacity-60">Created</div>
|
||||
<div>{{ token.created_at }}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card bg-base-200 mb-6">
|
||||
<div class="card-body">
|
||||
<h2 class="card-title text-lg">Allowed Tools</h2>
|
||||
{% if token.allowed_tools %}
|
||||
<div class="flex flex-wrap gap-2">
|
||||
{% for tool in token.allowed_tools %}
|
||||
<span class="badge badge-outline font-mono">{{ tool }}</span>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% else %}
|
||||
<p class="opacity-60">All tools permitted.</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<a href="{% url 'mcp_server:mcp-token-list' %}" class="btn btn-ghost btn-sm">← Back to Tokens</a>
|
||||
</div>
|
||||
{% endblock %}
|
||||
60
mnemosyne/mcp_server/templates/mcp_server/tokens/edit.html
Normal file
60
mnemosyne/mcp_server/templates/mcp_server/tokens/edit.html
Normal file
@@ -0,0 +1,60 @@
|
||||
{% extends "themis/base.html" %}
|
||||
|
||||
{% block title %}Edit {{ token.name }} — {{ themis_app_name }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="max-w-2xl mx-auto">
|
||||
<h1 class="text-2xl font-bold mb-6">Edit Token: {{ token.name }}</h1>
|
||||
|
||||
<div class="alert alert-info mb-6">
|
||||
<span>You can edit metadata below. The token value itself cannot be changed — generate a new token if needed.</span>
|
||||
</div>
|
||||
|
||||
<form method="post" action="{% url 'mcp_server:mcp-token-edit' pk=token.pk %}">
|
||||
{% csrf_token %}
|
||||
|
||||
<div class="card bg-base-200 mb-6">
|
||||
<div class="card-body">
|
||||
<div class="form-control">
|
||||
<label class="label" for="id_name">
|
||||
<span class="label-text">Name</span>
|
||||
</label>
|
||||
{{ form.name }}
|
||||
</div>
|
||||
<div class="form-control mt-4">
|
||||
<label class="label cursor-pointer justify-start gap-3">
|
||||
{{ form.is_active }}
|
||||
<span class="label-text">Active</span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="form-control mt-4">
|
||||
<label class="label" for="id_expires_at">
|
||||
<span class="label-text">Expires at (optional)</span>
|
||||
</label>
|
||||
{{ form.expires_at }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card bg-base-200 mb-6">
|
||||
<div class="card-body">
|
||||
<h2 class="card-title text-lg">Allowed Tools</h2>
|
||||
<p class="text-sm opacity-60 mb-2">{{ form.allowed_tools.help_text }}</p>
|
||||
<div class="space-y-1">
|
||||
{% for choice in form.allowed_tools %}
|
||||
<label class="label cursor-pointer justify-start gap-3">
|
||||
{{ choice.tag }}
|
||||
<span class="label-text font-mono">{{ choice.choice_label }}</span>
|
||||
</label>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex justify-between">
|
||||
<a href="{% url 'mcp_server:mcp-token-detail' pk=token.pk %}" class="btn btn-ghost">Cancel</a>
|
||||
<button type="submit" class="btn btn-primary">Save Changes</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
{% endblock %}
|
||||
61
mnemosyne/mcp_server/templates/mcp_server/tokens/list.html
Normal file
61
mnemosyne/mcp_server/templates/mcp_server/tokens/list.html
Normal file
@@ -0,0 +1,61 @@
|
||||
{% extends "themis/base.html" %}
|
||||
{% load humanize %}
|
||||
|
||||
{% block title %}MCP Tokens — {{ themis_app_name }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="max-w-3xl mx-auto">
|
||||
<div class="flex items-center justify-between mb-6">
|
||||
<h1 class="text-2xl font-bold">MCP Tokens</h1>
|
||||
<a href="{% url 'mcp_server:mcp-token-create' %}" class="btn btn-primary btn-sm">Generate Token</a>
|
||||
</div>
|
||||
|
||||
<p class="text-sm opacity-60 mb-4">
|
||||
Bearer tokens used by MCP clients (Claude Desktop, Cursor, etc.) to call Mnemosyne. Stored hashed at rest — the plaintext is shown only once at creation.
|
||||
</p>
|
||||
|
||||
{% if tokens %}
|
||||
<div class="space-y-3">
|
||||
{% for token in tokens %}
|
||||
<div class="card bg-base-200">
|
||||
<div class="card-body p-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<div>
|
||||
<a href="{% url 'mcp_server:mcp-token-detail' pk=token.pk %}"
|
||||
class="font-semibold link link-hover">
|
||||
{{ token.name }}
|
||||
</a>
|
||||
<div class="flex items-center gap-2 mt-1 flex-wrap">
|
||||
<span class="text-sm opacity-60 font-mono">{{ token.get_masked_token }}</span>
|
||||
{% if not token.is_valid %}
|
||||
<span class="badge badge-sm badge-error">
|
||||
{% if not token.is_active %}Revoked{% else %}Expired{% endif %}
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="badge badge-sm badge-success">Active</span>
|
||||
{% endif %}
|
||||
<span class="text-xs opacity-60">
|
||||
Last used:
|
||||
{% if token.last_used_at %}{{ token.last_used_at|naturaltime }}{% else %}never{% endif %}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex gap-1">
|
||||
<a href="{% url 'mcp_server:mcp-token-edit' pk=token.pk %}"
|
||||
class="btn btn-ghost btn-xs">Edit</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="card bg-base-200">
|
||||
<div class="card-body items-center text-center py-12">
|
||||
<p class="opacity-60 mb-4">No MCP tokens yet.</p>
|
||||
<a href="{% url 'mcp_server:mcp-token-create' %}" class="btn btn-primary btn-sm">Generate Your First Token</a>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -17,16 +17,18 @@ class ResolveMCPUserTest(TestCase):
|
||||
self.user = User.objects.create_user(
|
||||
username="bob", email="bob@example.com", password="pw"
|
||||
)
|
||||
self.token = MCPToken.objects.create(user=self.user, name="t")
|
||||
self.token, self.plaintext = MCPToken.objects.create_token(
|
||||
user=self.user, name="t"
|
||||
)
|
||||
|
||||
def test_resolves_valid_token(self):
|
||||
user, token = resolve_mcp_user(self.token.token)
|
||||
user, token = resolve_mcp_user(self.plaintext)
|
||||
self.assertEqual(user.pk, self.user.pk)
|
||||
self.assertEqual(token.pk, self.token.pk)
|
||||
|
||||
def test_records_usage(self):
|
||||
self.assertIsNone(self.token.last_used_at)
|
||||
resolve_mcp_user(self.token.token)
|
||||
resolve_mcp_user(self.plaintext)
|
||||
self.token.refresh_from_db()
|
||||
self.assertIsNotNone(self.token.last_used_at)
|
||||
|
||||
@@ -38,16 +40,31 @@ class ResolveMCPUserTest(TestCase):
|
||||
self.token.is_active = False
|
||||
self.token.save()
|
||||
with self.assertRaises(MCPAuthError):
|
||||
resolve_mcp_user(self.token.token)
|
||||
resolve_mcp_user(self.plaintext)
|
||||
|
||||
def test_expired_token_raises(self):
|
||||
self.token.expires_at = timezone.now() - timedelta(hours=1)
|
||||
self.token.save()
|
||||
with self.assertRaises(MCPAuthError):
|
||||
resolve_mcp_user(self.token.token)
|
||||
resolve_mcp_user(self.plaintext)
|
||||
|
||||
def test_disabled_user_raises(self):
|
||||
self.user.is_active = False
|
||||
self.user.save()
|
||||
with self.assertRaises(MCPAuthError):
|
||||
resolve_mcp_user(self.token.token)
|
||||
resolve_mcp_user(self.plaintext)
|
||||
|
||||
def test_plaintext_not_in_db(self):
|
||||
# Defense in depth: scan every column for the plaintext value.
|
||||
from django.db import connection
|
||||
|
||||
plaintext = self.plaintext
|
||||
with connection.cursor() as cur:
|
||||
cur.execute("SELECT * FROM mcp_server_mcptoken")
|
||||
rows = cur.fetchall()
|
||||
for row in rows:
|
||||
for value in row:
|
||||
self.assertNotEqual(
|
||||
value, plaintext,
|
||||
f"Plaintext token leaked into the database: {value!r}",
|
||||
)
|
||||
|
||||
58
mnemosyne/mcp_server/tests/test_forms.py
Normal file
58
mnemosyne/mcp_server/tests/test_forms.py
Normal file
@@ -0,0 +1,58 @@
|
||||
"""Form tests for the MCP token dashboard."""
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.test import TestCase
|
||||
|
||||
from mcp_server.forms import MCPTokenCreateForm, MCPTokenEditForm
|
||||
from mcp_server.models import MCPToken
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
class CreateFormTest(TestCase):
|
||||
def test_required_fields(self):
|
||||
form = MCPTokenCreateForm(data={})
|
||||
self.assertFalse(form.is_valid())
|
||||
self.assertIn("name", form.errors)
|
||||
|
||||
def test_name_only_is_valid(self):
|
||||
form = MCPTokenCreateForm(data={"name": "Test"})
|
||||
self.assertTrue(form.is_valid(), form.errors)
|
||||
|
||||
def test_tool_choices_match_registered_tools(self):
|
||||
form = MCPTokenCreateForm()
|
||||
choices = {value for value, _ in form.fields["allowed_tools"].choices}
|
||||
# These five must always be present per the FastMCP server.
|
||||
for expected in {"search", "get_chunk", "list_libraries", "list_collections", "list_items"}:
|
||||
self.assertIn(expected, choices)
|
||||
|
||||
|
||||
class EditFormTest(TestCase):
|
||||
def setUp(self):
|
||||
self.user = User.objects.create_user(username="alice", password="pw")
|
||||
self.token, _ = MCPToken.objects.create_token(
|
||||
user=self.user, name="t", allowed_tools=["search"]
|
||||
)
|
||||
|
||||
def test_initial_allowed_tools_populated(self):
|
||||
form = MCPTokenEditForm(instance=self.token)
|
||||
self.assertEqual(form.fields["allowed_tools"].initial, ["search"])
|
||||
|
||||
def test_save_updates_metadata(self):
|
||||
form = MCPTokenEditForm(
|
||||
data={
|
||||
"name": "Renamed",
|
||||
"is_active": False,
|
||||
"expires_at": "",
|
||||
"allowed_tools": ["search", "get_chunk"],
|
||||
},
|
||||
instance=self.token,
|
||||
)
|
||||
self.assertTrue(form.is_valid(), form.errors)
|
||||
instance = form.save(commit=False)
|
||||
instance.allowed_tools = form.cleaned_data["allowed_tools"]
|
||||
instance.save()
|
||||
self.token.refresh_from_db()
|
||||
self.assertEqual(self.token.name, "Renamed")
|
||||
self.assertFalse(self.token.is_active)
|
||||
self.assertEqual(self.token.allowed_tools, ["search", "get_chunk"])
|
||||
@@ -7,7 +7,14 @@ from django.test import TestCase
|
||||
from mcp_server.server import mcp
|
||||
|
||||
|
||||
EXPECTED_TOOLS = {"search", "get_chunk", "list_libraries", "list_collections", "list_items"}
|
||||
EXPECTED_TOOLS = {
|
||||
"search",
|
||||
"get_chunk",
|
||||
"list_libraries",
|
||||
"list_collections",
|
||||
"list_items",
|
||||
"get_health",
|
||||
}
|
||||
|
||||
|
||||
class ServerRegistrationTest(TestCase):
|
||||
|
||||
@@ -6,7 +6,7 @@ from django.contrib.auth import get_user_model
|
||||
from django.test import TestCase
|
||||
from django.utils import timezone
|
||||
|
||||
from mcp_server.models import MCPToken
|
||||
from mcp_server.models import MCPToken, hash_token
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
@@ -17,21 +17,33 @@ class MCPTokenModelTest(TestCase):
|
||||
username="alice", email="alice@example.com", password="pw"
|
||||
)
|
||||
|
||||
def test_token_auto_generated(self):
|
||||
token = MCPToken.objects.create(user=self.user, name="t")
|
||||
self.assertTrue(token.token)
|
||||
self.assertGreater(len(token.token), 20)
|
||||
def test_create_token_returns_plaintext_and_stores_hash(self):
|
||||
token, plaintext = MCPToken.objects.create_token(user=self.user, name="t")
|
||||
self.assertTrue(plaintext)
|
||||
self.assertGreater(len(plaintext), 20)
|
||||
# Database stores hash, not plaintext
|
||||
self.assertEqual(len(token.token_hash), 64)
|
||||
self.assertNotEqual(token.token_hash, plaintext)
|
||||
self.assertEqual(token.token_hash, hash_token(plaintext))
|
||||
|
||||
def test_token_hash_never_equals_plaintext(self):
|
||||
# Regression guard: if anyone ever wires plaintext back into token_hash,
|
||||
# this fails.
|
||||
token, plaintext = MCPToken.objects.create_token(user=self.user, name="t")
|
||||
self.assertNotIn(plaintext, token.token_hash)
|
||||
|
||||
def test_active_token_is_valid(self):
|
||||
token = MCPToken.objects.create(user=self.user, name="t")
|
||||
token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||
self.assertTrue(token.is_valid)
|
||||
|
||||
def test_inactive_token_not_valid(self):
|
||||
token = MCPToken.objects.create(user=self.user, name="t", is_active=False)
|
||||
token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||
token.is_active = False
|
||||
token.save()
|
||||
self.assertFalse(token.is_valid)
|
||||
|
||||
def test_expired_token_not_valid(self):
|
||||
token = MCPToken.objects.create(
|
||||
token, _ = MCPToken.objects.create_token(
|
||||
user=self.user,
|
||||
name="t",
|
||||
expires_at=timezone.now() - timedelta(hours=1),
|
||||
@@ -39,25 +51,27 @@ class MCPTokenModelTest(TestCase):
|
||||
self.assertFalse(token.is_valid)
|
||||
|
||||
def test_unrestricted_permits_all(self):
|
||||
token = MCPToken.objects.create(user=self.user, name="t")
|
||||
token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||
self.assertTrue(token.can_use_tool("anything"))
|
||||
|
||||
def test_tool_whitelist(self):
|
||||
token = MCPToken.objects.create(
|
||||
token, _ = MCPToken.objects.create_token(
|
||||
user=self.user, name="t", allowed_tools=["search"]
|
||||
)
|
||||
self.assertTrue(token.can_use_tool("search"))
|
||||
self.assertFalse(token.can_use_tool("get_chunk"))
|
||||
|
||||
def test_record_usage(self):
|
||||
token = MCPToken.objects.create(user=self.user, name="t")
|
||||
token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||
self.assertIsNone(token.last_used_at)
|
||||
token.record_usage()
|
||||
token.refresh_from_db()
|
||||
self.assertIsNotNone(token.last_used_at)
|
||||
|
||||
def test_masked_token(self):
|
||||
token = MCPToken.objects.create(user=self.user, name="t")
|
||||
def test_masked_token_is_hash_prefix(self):
|
||||
token, plaintext = MCPToken.objects.create_token(user=self.user, name="t")
|
||||
masked = token.get_masked_token()
|
||||
self.assertTrue(masked.endswith(token.token[-8:]))
|
||||
self.assertIn("*", masked)
|
||||
self.assertTrue(masked.startswith("mcp_…"))
|
||||
self.assertIn(token.token_hash[:8], masked)
|
||||
# Plaintext must never leak through the masked display
|
||||
self.assertNotIn(plaintext, masked)
|
||||
|
||||
177
mnemosyne/mcp_server/tests/test_views.py
Normal file
177
mnemosyne/mcp_server/tests/test_views.py
Normal file
@@ -0,0 +1,177 @@
|
||||
"""View tests for the MCP token self-service dashboard."""
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.test import TestCase
|
||||
from django.urls import reverse
|
||||
|
||||
from mcp_server.models import MCPToken
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
class TokenListViewTest(TestCase):
|
||||
def setUp(self):
|
||||
self.user = User.objects.create_user(
|
||||
username="alice", email="alice@example.com", password="pw"
|
||||
)
|
||||
self.url = reverse("mcp_server:mcp-token-list")
|
||||
|
||||
def test_login_required(self):
|
||||
resp = self.client.get(self.url)
|
||||
self.assertEqual(resp.status_code, 302)
|
||||
self.assertIn("/login/", resp.url)
|
||||
|
||||
def test_list_shows_only_own_tokens(self):
|
||||
other = User.objects.create_user(username="bob", password="pw")
|
||||
MCPToken.objects.create_token(user=self.user, name="mine")
|
||||
MCPToken.objects.create_token(user=other, name="theirs")
|
||||
self.client.force_login(self.user)
|
||||
resp = self.client.get(self.url)
|
||||
self.assertContains(resp, "mine")
|
||||
self.assertNotContains(resp, "theirs")
|
||||
|
||||
def test_empty_state(self):
|
||||
self.client.force_login(self.user)
|
||||
resp = self.client.get(self.url)
|
||||
self.assertContains(resp, "No MCP tokens yet.")
|
||||
|
||||
|
||||
class TokenCreateViewTest(TestCase):
|
||||
def setUp(self):
|
||||
self.user = User.objects.create_user(username="alice", password="pw")
|
||||
self.client.force_login(self.user)
|
||||
self.url = reverse("mcp_server:mcp-token-create")
|
||||
|
||||
def test_get_renders_form(self):
|
||||
resp = self.client.get(self.url)
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
self.assertContains(resp, "Generate MCP Token")
|
||||
|
||||
def test_post_creates_token_and_shows_plaintext_once(self):
|
||||
resp = self.client.post(self.url, {"name": "Claude Desktop"})
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
self.assertContains(resp, "Save this token now")
|
||||
# Pull the created row, verify the response contained a plaintext that
|
||||
# is NOT what we stored.
|
||||
token = MCPToken.objects.get(user=self.user, name="Claude Desktop")
|
||||
self.assertNotContains(resp, token.token_hash) # hash is not what we display
|
||||
# And the detail page never renders the plaintext.
|
||||
body = resp.content.decode()
|
||||
# Find the plaintext from the response: the only long alphanumeric
|
||||
# block inside the #mcp-plaintext div.
|
||||
import re
|
||||
|
||||
m = re.search(r'id="mcp-plaintext">([A-Za-z0-9_\-]+)<', body)
|
||||
self.assertIsNotNone(m, "plaintext block not found in response")
|
||||
plaintext = m.group(1)
|
||||
# Sanity: round-tripping the plaintext through hash_token reproduces
|
||||
# what's stored.
|
||||
from mcp_server.models import hash_token
|
||||
|
||||
self.assertEqual(hash_token(plaintext), token.token_hash)
|
||||
# Detail page must NOT contain the plaintext.
|
||||
detail_resp = self.client.get(
|
||||
reverse("mcp_server:mcp-token-detail", args=[token.pk])
|
||||
)
|
||||
self.assertNotContains(detail_resp, plaintext)
|
||||
|
||||
def test_post_invalid_renders_form_again(self):
|
||||
resp = self.client.post(self.url, {"name": ""})
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
self.assertContains(resp, "Generate MCP Token")
|
||||
self.assertEqual(MCPToken.objects.count(), 0)
|
||||
|
||||
|
||||
class TokenDetailViewTest(TestCase):
|
||||
def setUp(self):
|
||||
self.user = User.objects.create_user(username="alice", password="pw")
|
||||
self.client.force_login(self.user)
|
||||
self.token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||
|
||||
def test_renders_token(self):
|
||||
resp = self.client.get(
|
||||
reverse("mcp_server:mcp-token-detail", args=[self.token.pk])
|
||||
)
|
||||
self.assertContains(resp, self.token.name)
|
||||
self.assertContains(resp, self.token.get_masked_token())
|
||||
|
||||
def test_cannot_view_other_users_token(self):
|
||||
other = User.objects.create_user(username="bob", password="pw")
|
||||
other_token, _ = MCPToken.objects.create_token(user=other, name="theirs")
|
||||
resp = self.client.get(
|
||||
reverse("mcp_server:mcp-token-detail", args=[other_token.pk])
|
||||
)
|
||||
self.assertEqual(resp.status_code, 404)
|
||||
|
||||
|
||||
class TokenEditViewTest(TestCase):
|
||||
def setUp(self):
|
||||
self.user = User.objects.create_user(username="alice", password="pw")
|
||||
self.client.force_login(self.user)
|
||||
self.token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||
|
||||
def test_post_updates_metadata(self):
|
||||
resp = self.client.post(
|
||||
reverse("mcp_server:mcp-token-edit", args=[self.token.pk]),
|
||||
{
|
||||
"name": "Renamed",
|
||||
"is_active": "on",
|
||||
"expires_at": "",
|
||||
"allowed_tools": ["search"],
|
||||
},
|
||||
)
|
||||
self.assertEqual(resp.status_code, 302)
|
||||
self.token.refresh_from_db()
|
||||
self.assertEqual(self.token.name, "Renamed")
|
||||
self.assertEqual(self.token.allowed_tools, ["search"])
|
||||
|
||||
def test_cannot_edit_other_users_token(self):
|
||||
other = User.objects.create_user(username="bob", password="pw")
|
||||
other_token, _ = MCPToken.objects.create_token(user=other, name="theirs")
|
||||
resp = self.client.post(
|
||||
reverse("mcp_server:mcp-token-edit", args=[other_token.pk]),
|
||||
{"name": "hacked"},
|
||||
)
|
||||
self.assertEqual(resp.status_code, 404)
|
||||
|
||||
|
||||
class TokenRevokeViewTest(TestCase):
|
||||
def setUp(self):
|
||||
self.user = User.objects.create_user(username="alice", password="pw")
|
||||
self.client.force_login(self.user)
|
||||
self.token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||
|
||||
def test_revoke_sets_inactive_keeps_row(self):
|
||||
url = reverse("mcp_server:mcp-token-revoke", args=[self.token.pk])
|
||||
resp = self.client.post(url)
|
||||
self.assertEqual(resp.status_code, 302)
|
||||
self.token.refresh_from_db()
|
||||
self.assertFalse(self.token.is_active)
|
||||
# Row still exists for audit trail.
|
||||
self.assertTrue(MCPToken.objects.filter(pk=self.token.pk).exists())
|
||||
|
||||
def test_get_not_allowed(self):
|
||||
url = reverse("mcp_server:mcp-token-revoke", args=[self.token.pk])
|
||||
resp = self.client.get(url)
|
||||
self.assertEqual(resp.status_code, 405)
|
||||
|
||||
|
||||
class TokenDeleteViewTest(TestCase):
|
||||
def setUp(self):
|
||||
self.user = User.objects.create_user(username="alice", password="pw")
|
||||
self.client.force_login(self.user)
|
||||
self.token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||
|
||||
def test_delete_removes_row(self):
|
||||
url = reverse("mcp_server:mcp-token-delete", args=[self.token.pk])
|
||||
resp = self.client.post(url)
|
||||
self.assertEqual(resp.status_code, 302)
|
||||
self.assertFalse(MCPToken.objects.filter(pk=self.token.pk).exists())
|
||||
|
||||
def test_cannot_delete_other_users_token(self):
|
||||
other = User.objects.create_user(username="bob", password="pw")
|
||||
other_token, _ = MCPToken.objects.create_token(user=other, name="theirs")
|
||||
url = reverse("mcp_server:mcp-token-delete", args=[other_token.pk])
|
||||
resp = self.client.post(url)
|
||||
self.assertEqual(resp.status_code, 404)
|
||||
self.assertTrue(MCPToken.objects.filter(pk=other_token.pk).exists())
|
||||
@@ -1,4 +1,9 @@
|
||||
from .discovery import register_discovery_tools
|
||||
from .health import register_health_tools
|
||||
from .search import register_search_tools
|
||||
|
||||
__all__ = ["register_search_tools", "register_discovery_tools"]
|
||||
__all__ = [
|
||||
"register_search_tools",
|
||||
"register_discovery_tools",
|
||||
"register_health_tools",
|
||||
]
|
||||
|
||||
@@ -20,16 +20,21 @@ def _clamp(limit: int) -> int:
|
||||
|
||||
def register_discovery_tools(mcp):
|
||||
@mcp.tool
|
||||
async def list_libraries(limit: int = DEFAULT_LIMIT, offset: int = 0) -> dict[str, Any]:
|
||||
async def list_libraries(
|
||||
limit: int = DEFAULT_LIMIT,
|
||||
offset: int = 0,
|
||||
# System-injected; deliberately absent from the docstring.
|
||||
workspace_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""List Mnemosyne libraries. Each library has a content-aware library_type
|
||||
(fiction, nonfiction, technical, music, film, art, journal) that drives
|
||||
chunking, embedding, and re-ranking. Returns uid, name, library_type,
|
||||
description for each library — use the uid or library_type to scope a
|
||||
subsequent search.
|
||||
(fiction, nonfiction, technical, music, film, art, journal, business,
|
||||
finance) that drives chunking, embedding, and re-ranking. Returns uid,
|
||||
name, library_type, description for each library — use the uid or
|
||||
library_type to scope a subsequent search.
|
||||
"""
|
||||
with record_tool_call("list_libraries"):
|
||||
return await sync_to_async(_query_libraries, thread_sensitive=True)(
|
||||
_clamp(limit), max(offset, 0)
|
||||
_clamp(limit), max(offset, 0), workspace_id
|
||||
)
|
||||
|
||||
@mcp.tool
|
||||
@@ -37,6 +42,8 @@ def register_discovery_tools(mcp):
|
||||
library_uid: str | None = None,
|
||||
limit: int = DEFAULT_LIMIT,
|
||||
offset: int = 0,
|
||||
# System-injected; deliberately absent from the docstring.
|
||||
workspace_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""List collections, optionally filtered by parent library_uid.
|
||||
Collections group related items inside a library (e.g. a series of novels,
|
||||
@@ -45,7 +52,7 @@ def register_discovery_tools(mcp):
|
||||
"""
|
||||
with record_tool_call("list_collections"):
|
||||
return await sync_to_async(_query_collections, thread_sensitive=True)(
|
||||
library_uid, _clamp(limit), max(offset, 0)
|
||||
library_uid, _clamp(limit), max(offset, 0), workspace_id
|
||||
)
|
||||
|
||||
@mcp.tool
|
||||
@@ -54,6 +61,8 @@ def register_discovery_tools(mcp):
|
||||
library_uid: str | None = None,
|
||||
limit: int = DEFAULT_LIMIT,
|
||||
offset: int = 0,
|
||||
# System-injected; deliberately absent from the docstring.
|
||||
workspace_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""List items (the indexed documents/files), optionally filtered by
|
||||
collection_uid or library_uid. Returns uid, title, item_type, file_type,
|
||||
@@ -63,17 +72,27 @@ def register_discovery_tools(mcp):
|
||||
"""
|
||||
with record_tool_call("list_items"):
|
||||
return await sync_to_async(_query_items, thread_sensitive=True)(
|
||||
collection_uid, library_uid, _clamp(limit), max(offset, 0)
|
||||
collection_uid, library_uid, _clamp(limit), max(offset, 0), workspace_id
|
||||
)
|
||||
|
||||
|
||||
def _query_libraries(limit: int, offset: int) -> dict[str, Any]:
|
||||
_WORKSPACE_SCOPE = (
|
||||
"($workspace_id IS NULL AND l.workspace_id IS NULL OR "
|
||||
"l.workspace_id = $workspace_id)"
|
||||
)
|
||||
|
||||
|
||||
def _query_libraries(
|
||||
limit: int, offset: int, workspace_id: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
from neomodel import db
|
||||
|
||||
rows, _ = db.cypher_query(
|
||||
"MATCH (l:Library) RETURN l.uid, l.name, l.library_type, l.description "
|
||||
"MATCH (l:Library) "
|
||||
f"WHERE {_WORKSPACE_SCOPE} "
|
||||
"RETURN l.uid, l.name, l.library_type, l.description "
|
||||
"ORDER BY l.name SKIP $offset LIMIT $limit",
|
||||
{"offset": offset, "limit": limit},
|
||||
{"offset": offset, "limit": limit, "workspace_id": workspace_id},
|
||||
)
|
||||
return {
|
||||
"libraries": [
|
||||
@@ -91,24 +110,33 @@ def _query_libraries(limit: int, offset: int) -> dict[str, Any]:
|
||||
|
||||
|
||||
def _query_collections(
|
||||
library_uid: str | None, limit: int, offset: int
|
||||
library_uid: str | None, limit: int, offset: int,
|
||||
workspace_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
from neomodel import db
|
||||
|
||||
if library_uid:
|
||||
cypher = (
|
||||
"MATCH (l:Library {uid: $library_uid})-[:CONTAINS]->(c:Collection) "
|
||||
f"WHERE {_WORKSPACE_SCOPE} "
|
||||
"RETURN c.uid, c.name, c.description, l.uid, l.name "
|
||||
"ORDER BY c.name SKIP $offset LIMIT $limit"
|
||||
)
|
||||
params = {"library_uid": library_uid, "offset": offset, "limit": limit}
|
||||
params = {
|
||||
"library_uid": library_uid, "offset": offset, "limit": limit,
|
||||
"workspace_id": workspace_id,
|
||||
}
|
||||
else:
|
||||
cypher = (
|
||||
"MATCH (l:Library)-[:CONTAINS]->(c:Collection) "
|
||||
f"WHERE {_WORKSPACE_SCOPE} "
|
||||
"RETURN c.uid, c.name, c.description, l.uid, l.name "
|
||||
"ORDER BY l.name, c.name SKIP $offset LIMIT $limit"
|
||||
)
|
||||
params = {"offset": offset, "limit": limit}
|
||||
params = {
|
||||
"offset": offset, "limit": limit,
|
||||
"workspace_id": workspace_id,
|
||||
}
|
||||
|
||||
rows, _ = db.cypher_query(cypher, params)
|
||||
return {
|
||||
@@ -132,11 +160,15 @@ def _query_items(
|
||||
library_uid: str | None,
|
||||
limit: int,
|
||||
offset: int,
|
||||
workspace_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
from neomodel import db
|
||||
|
||||
where = []
|
||||
params: dict[str, Any] = {"offset": offset, "limit": limit}
|
||||
where = [_WORKSPACE_SCOPE]
|
||||
params: dict[str, Any] = {
|
||||
"offset": offset, "limit": limit,
|
||||
"workspace_id": workspace_id,
|
||||
}
|
||||
if collection_uid:
|
||||
where.append("c.uid = $collection_uid")
|
||||
params["collection_uid"] = collection_uid
|
||||
@@ -144,7 +176,7 @@ def _query_items(
|
||||
where.append("l.uid = $library_uid")
|
||||
params["library_uid"] = library_uid
|
||||
|
||||
where_clause = ("WHERE " + " AND ".join(where)) if where else ""
|
||||
where_clause = "WHERE " + " AND ".join(where)
|
||||
cypher = (
|
||||
"MATCH (l:Library)-[:CONTAINS]->(c:Collection)-[:CONTAINS]->(i:Item) "
|
||||
f"{where_clause} "
|
||||
|
||||
123
mnemosyne/mcp_server/tools/health.py
Normal file
123
mnemosyne/mcp_server/tools/health.py
Normal file
@@ -0,0 +1,123 @@
|
||||
"""Health-check MCP tool — used by Pallas/Daedalus health pollers.
|
||||
|
||||
Per the Pallas health spec, returns one of:
|
||||
- ok — all dependencies reachable
|
||||
- degraded — non-critical dependency unhealthy (chat allowed)
|
||||
- error — critical dependency unhealthy (chat blocked)
|
||||
|
||||
The tool is intercepted by the FastMCP server and never invokes an LLM —
|
||||
it executes synchronously against Neo4j, S3, and the embedding model
|
||||
endpoint, and returns within the poller's timeout.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from asgiref.sync import sync_to_async
|
||||
|
||||
from ..metrics import record_tool_call
|
||||
|
||||
|
||||
def register_health_tools(mcp):
|
||||
@mcp.tool
|
||||
async def get_health() -> dict[str, Any]:
|
||||
"""Health check for Mnemosyne.
|
||||
|
||||
Returns a status object compatible with the Pallas health spec:
|
||||
{status: "ok"|"degraded"|"error", checks: {neo4j, s3, embedding}}.
|
||||
"""
|
||||
with record_tool_call("get_health"):
|
||||
return await sync_to_async(_run_health_check, thread_sensitive=True)()
|
||||
|
||||
|
||||
def _run_health_check() -> dict[str, Any]:
|
||||
"""Synchronous health check across Neo4j, S3, and embedding model."""
|
||||
checks: dict[str, dict[str, Any]] = {}
|
||||
|
||||
checks["neo4j"] = _check_neo4j()
|
||||
checks["s3"] = _check_s3()
|
||||
checks["embedding"] = _check_embedding_model()
|
||||
|
||||
# Aggregate status: error if any critical check failed; degraded if a
|
||||
# non-critical check failed; ok otherwise.
|
||||
if checks["neo4j"]["status"] == "error" or checks["s3"]["status"] == "error":
|
||||
status = "error"
|
||||
elif any(c["status"] != "ok" for c in checks.values()):
|
||||
status = "degraded"
|
||||
else:
|
||||
status = "ok"
|
||||
|
||||
return {
|
||||
"status": status,
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
|
||||
def _check_neo4j() -> dict[str, Any]:
|
||||
start = time.time()
|
||||
try:
|
||||
from neomodel import db
|
||||
|
||||
db.cypher_query("RETURN 1")
|
||||
return {
|
||||
"status": "ok",
|
||||
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||
}
|
||||
|
||||
|
||||
def _check_s3() -> dict[str, Any]:
|
||||
start = time.time()
|
||||
try:
|
||||
from django.core.files.storage import default_storage
|
||||
|
||||
# `exists` on a path that won't exist is the cheapest round-trip
|
||||
# we have. It returns False rather than raising on most backends.
|
||||
default_storage.exists("__healthcheck__")
|
||||
return {
|
||||
"status": "ok",
|
||||
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||
}
|
||||
|
||||
|
||||
def _check_embedding_model() -> dict[str, Any]:
|
||||
"""Soft check: confirm a system embedding model is configured.
|
||||
|
||||
We don't hit the model endpoint here — that would burn GPU time on
|
||||
every poll. The poller-level check is "is a model registered."
|
||||
"""
|
||||
start = time.time()
|
||||
try:
|
||||
from llm_manager.models import LLMModel
|
||||
|
||||
model = LLMModel.get_system_embedding_model()
|
||||
if model is None:
|
||||
return {
|
||||
"status": "degraded",
|
||||
"error": "no system embedding model configured",
|
||||
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||
}
|
||||
return {
|
||||
"status": "ok",
|
||||
"model": model.name,
|
||||
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"status": "degraded",
|
||||
"error": str(exc),
|
||||
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||
}
|
||||
@@ -27,13 +27,19 @@ def register_search_tools(mcp):
|
||||
rerank: bool = True,
|
||||
include_images: bool = True,
|
||||
search_types: list[str] | None = None,
|
||||
# workspace_id is system-injected by Daedalus's chat path. It is
|
||||
# intentionally absent from the docstring so the calling LLM is
|
||||
# never told it exists. Whatever value the LLM produces here is
|
||||
# overwritten by Daedalus before the call reaches Mnemosyne.
|
||||
workspace_id: str | None = None,
|
||||
ctx: Context | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Hybrid retrieval over Mnemosyne: vector + full-text + concept-graph
|
||||
candidates fused by RRF and optionally re-ranked by Synesis.
|
||||
|
||||
Filters: library_uid (exact library), library_type (one of fiction,
|
||||
nonfiction, technical, music, film, art, journal), or collection_uid.
|
||||
nonfiction, technical, music, film, art, journal, business, finance),
|
||||
or collection_uid.
|
||||
Set rerank=False to skip re-ranking. search_types defaults to all three.
|
||||
|
||||
Returns ranked candidates with chunk_uid (use get_chunk for full text),
|
||||
@@ -49,6 +55,7 @@ def register_search_tools(mcp):
|
||||
library_uid=library_uid,
|
||||
library_type=library_type,
|
||||
collection_uid=collection_uid,
|
||||
workspace_id=workspace_id,
|
||||
limit=limit,
|
||||
rerank=rerank,
|
||||
include_images=include_images,
|
||||
@@ -56,7 +63,12 @@ def register_search_tools(mcp):
|
||||
)
|
||||
|
||||
@mcp.tool
|
||||
async def get_chunk(chunk_uid: str, ctx: Context | None = None) -> dict[str, Any]:
|
||||
async def get_chunk(
|
||||
chunk_uid: str,
|
||||
# System-injected; deliberately absent from the docstring.
|
||||
workspace_id: str | None = None,
|
||||
ctx: Context | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Fetch the full text of a chunk by its uid (typically obtained from `search`).
|
||||
|
||||
Returns the chunk text plus parent item context: chunk_uid, chunk_index,
|
||||
@@ -64,11 +76,13 @@ def register_search_tools(mcp):
|
||||
text_preview from `search` isn't enough.
|
||||
"""
|
||||
with record_tool_call("get_chunk"):
|
||||
return await sync_to_async(_load_chunk, thread_sensitive=True)(chunk_uid)
|
||||
return await sync_to_async(_load_chunk, thread_sensitive=True)(
|
||||
chunk_uid, workspace_id
|
||||
)
|
||||
|
||||
|
||||
def _run_search(*, user, query, library_uid, library_type, collection_uid, limit,
|
||||
rerank, include_images, search_types) -> dict[str, Any]:
|
||||
def _run_search(*, user, query, library_uid, library_type, collection_uid,
|
||||
workspace_id, limit, rerank, include_images, search_types) -> dict[str, Any]:
|
||||
from library.services.search import SearchRequest, SearchService
|
||||
|
||||
req = SearchRequest(
|
||||
@@ -76,6 +90,7 @@ def _run_search(*, user, query, library_uid, library_type, collection_uid, limit
|
||||
library_uid=library_uid,
|
||||
library_type=library_type,
|
||||
collection_uid=collection_uid,
|
||||
workspace_id=workspace_id,
|
||||
search_types=search_types,
|
||||
limit=limit,
|
||||
vector_top_k=getattr(settings, "SEARCH_VECTOR_TOP_K", 50),
|
||||
@@ -97,15 +112,17 @@ def _run_search(*, user, query, library_uid, library_type, collection_uid, limit
|
||||
}
|
||||
|
||||
|
||||
def _load_chunk(chunk_uid: str) -> dict[str, Any]:
|
||||
def _load_chunk(chunk_uid: str, workspace_id: str | None = None) -> dict[str, Any]:
|
||||
from neomodel import db
|
||||
|
||||
rows, _ = db.cypher_query(
|
||||
"MATCH (l:Library)-[:CONTAINS]->(:Collection)-[:CONTAINS]->"
|
||||
"(i:Item)-[:HAS_CHUNK]->(c:Chunk {uid: $uid}) "
|
||||
"WHERE ($workspace_id IS NULL AND l.workspace_id IS NULL OR "
|
||||
" l.workspace_id = $workspace_id) "
|
||||
"RETURN c.uid, c.chunk_index, c.chunk_s3_key, "
|
||||
"i.uid, i.title, l.library_type LIMIT 1",
|
||||
{"uid": chunk_uid},
|
||||
{"uid": chunk_uid, "workspace_id": workspace_id},
|
||||
)
|
||||
if not rows:
|
||||
raise ValueError(f"Chunk not found: {chunk_uid}")
|
||||
|
||||
16
mnemosyne/mcp_server/urls.py
Normal file
16
mnemosyne/mcp_server/urls.py
Normal file
@@ -0,0 +1,16 @@
|
||||
"""URL routes for the MCP token self-service dashboard."""
|
||||
|
||||
from django.urls import path
|
||||
|
||||
from . import views
|
||||
|
||||
app_name = "mcp_server"
|
||||
|
||||
urlpatterns = [
|
||||
path("profile/mcp-tokens/", views.mcp_token_list, name="mcp-token-list"),
|
||||
path("profile/mcp-tokens/add/", views.mcp_token_create, name="mcp-token-create"),
|
||||
path("profile/mcp-tokens/<int:pk>/", views.mcp_token_detail, name="mcp-token-detail"),
|
||||
path("profile/mcp-tokens/<int:pk>/edit/", views.mcp_token_edit, name="mcp-token-edit"),
|
||||
path("profile/mcp-tokens/<int:pk>/revoke/", views.mcp_token_revoke, name="mcp-token-revoke"),
|
||||
path("profile/mcp-tokens/<int:pk>/delete/", views.mcp_token_delete, name="mcp-token-delete"),
|
||||
]
|
||||
95
mnemosyne/mcp_server/views.py
Normal file
95
mnemosyne/mcp_server/views.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""Self-service dashboard for MCP bearer tokens.
|
||||
|
||||
Mirrors the Themis API-keys flow visually but stores hashed tokens. Plaintext
|
||||
is shown to the user exactly once (on the create-success page) and never
|
||||
persisted.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from django.contrib import messages
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.http import HttpRequest, HttpResponse
|
||||
from django.shortcuts import get_object_or_404, redirect, render
|
||||
from django.views.decorators.http import require_GET, require_http_methods, require_POST
|
||||
|
||||
from .forms import MCPTokenCreateForm, MCPTokenEditForm
|
||||
from .models import MCPToken
|
||||
|
||||
|
||||
@login_required
|
||||
@require_GET
|
||||
def mcp_token_list(request: HttpRequest) -> HttpResponse:
|
||||
tokens = MCPToken.objects.filter(user=request.user).order_by("-created_at")
|
||||
return render(request, "mcp_server/tokens/list.html", {"tokens": tokens})
|
||||
|
||||
|
||||
@login_required
|
||||
@require_http_methods(["GET", "POST"])
|
||||
def mcp_token_create(request: HttpRequest) -> HttpResponse:
|
||||
if request.method == "POST":
|
||||
form = MCPTokenCreateForm(request.POST)
|
||||
if form.is_valid():
|
||||
token, plaintext = MCPToken.objects.create_token(
|
||||
user=request.user,
|
||||
name=form.cleaned_data["name"],
|
||||
allowed_tools=form.cleaned_data.get("allowed_tools") or [],
|
||||
expires_at=form.cleaned_data.get("expires_at") or None,
|
||||
)
|
||||
return render(
|
||||
request,
|
||||
"mcp_server/tokens/created.html",
|
||||
{"token": token, "plaintext": plaintext},
|
||||
)
|
||||
else:
|
||||
form = MCPTokenCreateForm()
|
||||
|
||||
return render(request, "mcp_server/tokens/create.html", {"form": form})
|
||||
|
||||
|
||||
@login_required
|
||||
@require_GET
|
||||
def mcp_token_detail(request: HttpRequest, pk: int) -> HttpResponse:
|
||||
token = get_object_or_404(MCPToken, pk=pk, user=request.user)
|
||||
return render(request, "mcp_server/tokens/detail.html", {"token": token})
|
||||
|
||||
|
||||
@login_required
|
||||
@require_http_methods(["GET", "POST"])
|
||||
def mcp_token_edit(request: HttpRequest, pk: int) -> HttpResponse:
|
||||
token = get_object_or_404(MCPToken, pk=pk, user=request.user)
|
||||
|
||||
if request.method == "POST":
|
||||
form = MCPTokenEditForm(request.POST, instance=token)
|
||||
if form.is_valid():
|
||||
instance = form.save(commit=False)
|
||||
instance.allowed_tools = form.cleaned_data.get("allowed_tools") or []
|
||||
instance.save()
|
||||
messages.success(request, "MCP token updated.")
|
||||
return redirect("mcp_server:mcp-token-detail", pk=token.pk)
|
||||
else:
|
||||
form = MCPTokenEditForm(instance=token)
|
||||
|
||||
return render(
|
||||
request, "mcp_server/tokens/edit.html", {"form": form, "token": token}
|
||||
)
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def mcp_token_revoke(request: HttpRequest, pk: int) -> HttpResponse:
|
||||
token = get_object_or_404(MCPToken, pk=pk, user=request.user)
|
||||
token.is_active = False
|
||||
token.save(update_fields=["is_active", "updated_at"])
|
||||
messages.success(request, f"Revoked “{token.name}”. The token can no longer be used.")
|
||||
return redirect("mcp_server:mcp-token-detail", pk=token.pk)
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def mcp_token_delete(request: HttpRequest, pk: int) -> HttpResponse:
|
||||
token = get_object_or_404(MCPToken, pk=pk, user=request.user)
|
||||
name = token.name
|
||||
token.delete()
|
||||
messages.success(request, f"Deleted “{name}”.")
|
||||
return redirect("mcp_server:mcp-token-list")
|
||||
@@ -20,7 +20,7 @@ django.setup()
|
||||
|
||||
from django.core.asgi import get_asgi_application # noqa: E402
|
||||
from starlette.applications import Starlette # noqa: E402
|
||||
from starlette.responses import JSONResponse # noqa: E402
|
||||
from starlette.responses import JSONResponse, RedirectResponse # noqa: E402
|
||||
from starlette.routing import Mount, Route # noqa: E402
|
||||
|
||||
from mcp_server.server import mcp # noqa: E402
|
||||
@@ -35,6 +35,13 @@ async def health(request):
|
||||
return JSONResponse({"status": "ok"})
|
||||
|
||||
|
||||
async def mcp_redirect(request):
|
||||
"""Bare /mcp → /mcp/ — MCP clients that omit the trailing slash hit
|
||||
Starlette's Mount which only matches /mcp/* paths, falling through to
|
||||
Django and returning 404. A 307 preserves the POST body."""
|
||||
return RedirectResponse(url="/mcp/", status_code=307)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
async with mcp_http_app.lifespan(app), mcp_sse_app.lifespan(app):
|
||||
@@ -44,6 +51,7 @@ async def lifespan(app):
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Route("/mcp/health", health),
|
||||
Route("/mcp", mcp_redirect, methods=["GET", "POST"]),
|
||||
Mount("/mcp/sse", app=mcp_sse_app),
|
||||
Mount("/mcp", app=mcp_http_app),
|
||||
Mount("/", app=application),
|
||||
|
||||
@@ -181,6 +181,18 @@ else:
|
||||
},
|
||||
}
|
||||
|
||||
# --- Daedalus S3 (cross-bucket reads for ingest) ---
|
||||
# Mnemosyne ingests files written to Daedalus's S3 bucket. These vars
|
||||
# configure read access; the file is copied into AWS_STORAGE_BUCKET_NAME
|
||||
# (Mnemosyne's own bucket) by the Celery ingest task before processing.
|
||||
DAEDALUS_S3_ENDPOINT_URL = env("DAEDALUS_S3_ENDPOINT_URL", default="")
|
||||
DAEDALUS_S3_ACCESS_KEY_ID = env("DAEDALUS_S3_ACCESS_KEY_ID", default="")
|
||||
DAEDALUS_S3_SECRET_ACCESS_KEY = env("DAEDALUS_S3_SECRET_ACCESS_KEY", default="")
|
||||
DAEDALUS_S3_BUCKET_NAME = env("DAEDALUS_S3_BUCKET_NAME", default="daedalus")
|
||||
DAEDALUS_S3_REGION_NAME = env("DAEDALUS_S3_REGION_NAME", default="us-east-1")
|
||||
DAEDALUS_S3_USE_SSL = env.bool("DAEDALUS_S3_USE_SSL", default=False)
|
||||
DAEDALUS_S3_VERIFY = env.bool("DAEDALUS_S3_VERIFY", default=True)
|
||||
|
||||
# --- Celery / RabbitMQ ---
|
||||
CELERY_BROKER_URL = env(
|
||||
"CELERY_BROKER_URL",
|
||||
@@ -196,6 +208,7 @@ CELERY_TASK_ACKS_LATE = True
|
||||
CELERY_WORKER_PREFETCH_MULTIPLIER = 1
|
||||
CELERY_TASK_ROUTES = {
|
||||
"library.tasks.embed_*": {"queue": "embedding"},
|
||||
"library.tasks.ingest_*": {"queue": "embedding"},
|
||||
"library.tasks.batch_*": {"queue": "batch"},
|
||||
}
|
||||
|
||||
|
||||
@@ -25,4 +25,6 @@ urlpatterns = [
|
||||
path("library/", include("library.urls")),
|
||||
# LLM Manager
|
||||
path("llm/", include("llm_manager.urls")),
|
||||
# MCP server (token dashboard at /profile/mcp-tokens/)
|
||||
path("", include("mcp_server.urls")),
|
||||
]
|
||||
|
||||
@@ -52,15 +52,20 @@ class PostgreSQLTestRunner(DiscoverRunner):
|
||||
from django.db import connections
|
||||
|
||||
db_cfg = self.pg_manager.get_django_database_config()
|
||||
# Preserve Django's defaulted TEST sub-dict (CHARSET/MIRROR/MIGRATE…).
|
||||
existing_test = connections["default"].settings_dict.get("TEST", {})
|
||||
merged_test = {**existing_test, **db_cfg.get("TEST", {})}
|
||||
db_cfg["TEST"] = merged_test
|
||||
settings.DATABASES["default"] = db_cfg
|
||||
# Preserve Django's defaulted top-level keys (ATOMIC_REQUESTS,
|
||||
# AUTOCOMMIT, OPTIONS, …) and the TEST sub-dict (CHARSET, MIRROR,
|
||||
# MIGRATE, …) — these are populated lazily by Django and absent from
|
||||
# the user's raw settings.DATABASES, so a naive overwrite breaks
|
||||
# request handling that consults them.
|
||||
existing = connections["default"].settings_dict
|
||||
existing_test = existing.get("TEST", {})
|
||||
merged = {**existing, **db_cfg}
|
||||
merged["TEST"] = {**existing_test, **db_cfg.get("TEST", {})}
|
||||
settings.DATABASES["default"] = merged
|
||||
# The default connection was instantiated at Django bootstrap; its
|
||||
# settings_dict is independent of settings.DATABASES. Sync it
|
||||
# settings_dict is independent of settings.DATABASES. Sync it
|
||||
# manually so test code talks to the container, not the dev DB.
|
||||
connections["default"].settings_dict.update(db_cfg)
|
||||
connections["default"].settings_dict.update(merged)
|
||||
logger.info("PostgreSQL test DB ready on port %s", self.pg_manager.assigned_port)
|
||||
|
||||
# ── Neo4j ──────────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# Generated by Django 5.2.13 on 2026-04-27 11:30
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('themis', '0003_alter_userprofile_current_timezone_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='userapikey',
|
||||
name='key_type',
|
||||
field=models.CharField(choices=[('api', 'API Key'), ('dav', 'DAV Credentials'), ('token', 'Access Token'), ('secret', 'Secret Key'), ('other', 'Other')], default='api', help_text='Type of credential', max_length=30),
|
||||
),
|
||||
]
|
||||
@@ -228,7 +228,6 @@ class UserAPIKey(models.Model):
|
||||
|
||||
KEY_TYPE_CHOICES = [
|
||||
("api", "API Key"),
|
||||
("mcp", "MCP Server"),
|
||||
("dav", "DAV Credentials"),
|
||||
("token", "Access Token"),
|
||||
("secret", "Secret Key"),
|
||||
|
||||
@@ -38,6 +38,16 @@
|
||||
API Keys
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="{% url 'mcp_server:mcp-token-list' %}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4" fill="none"
|
||||
viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||
d="M9 12l2 2 4-4m5.618-4.016A11.955 11.955 0 0112 2.944a11.955 11.955 0 01-8.618 3.04A12.02 12.02 0 003 9c0 5.591 3.824 10.29 9 11.622 5.176-1.332 9-6.03 9-11.622 0-1.042-.133-2.052-.382-3.016z" />
|
||||
</svg>
|
||||
MCP Tokens
|
||||
</a>
|
||||
</li>
|
||||
<div class="divider my-0"></div>
|
||||
<li>
|
||||
<form method="post" action="{% url 'logout' %}">
|
||||
|
||||
@@ -248,7 +248,7 @@ class APIKeyEditFormTest(TestCase):
|
||||
form = APIKeyEditForm(
|
||||
data={
|
||||
"service_name": "Updated Service",
|
||||
"key_type": "mcp",
|
||||
"key_type": "token",
|
||||
"label": "Updated",
|
||||
"instructions": "",
|
||||
"help_url": "",
|
||||
@@ -260,6 +260,6 @@ class APIKeyEditFormTest(TestCase):
|
||||
form.save()
|
||||
self.key.refresh_from_db()
|
||||
self.assertEqual(self.key.service_name, "Updated Service")
|
||||
self.assertEqual(self.key.key_type, "mcp")
|
||||
self.assertEqual(self.key.key_type, "token")
|
||||
self.assertFalse(self.key.is_active)
|
||||
self.assertEqual(self.key.encrypted_value, original_encrypted)
|
||||
|
||||
@@ -209,7 +209,6 @@ class UserAPIKeyModelTest(TestCase):
|
||||
"""KEY_TYPE_CHOICES contains expected types."""
|
||||
type_keys = [t[0] for t in UserAPIKey.KEY_TYPE_CHOICES]
|
||||
self.assertIn("api", type_keys)
|
||||
self.assertIn("mcp", type_keys)
|
||||
self.assertIn("dav", type_keys)
|
||||
self.assertIn("token", type_keys)
|
||||
self.assertIn("secret", type_keys)
|
||||
|
||||
86
nginx/mnemosyne.conf
Normal file
86
nginx/mnemosyne.conf
Normal file
@@ -0,0 +1,86 @@
|
||||
# Mnemosyne nginx — single virtual host that fronts the Django web app
|
||||
# and the FastMCP server. HAProxy on Titania terminates TLS and routes by
|
||||
# hostname; this nginx is plain HTTP on the internal network.
|
||||
|
||||
# Map of upstreams to give us readable proxy_pass targets and easy retries.
|
||||
upstream mnemosyne_web {
|
||||
server web:8000 max_fails=3 fail_timeout=30s;
|
||||
}
|
||||
|
||||
upstream mnemosyne_mcp {
|
||||
server mcp:22091 max_fails=3 fail_timeout=30s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80 default_server;
|
||||
server_name _;
|
||||
|
||||
# Reasonable limits — file uploads to the ingest endpoint can be big,
|
||||
# but the bulk path is S3-direct from Daedalus. 64 MB covers admin
|
||||
# uploads and direct REST POST /library/api/items/upload.
|
||||
client_max_body_size 64m;
|
||||
client_body_timeout 120s;
|
||||
|
||||
# Mnemosyne's REST API — Django REST Framework views + admin.
|
||||
# Under /library/api/* per mnemosyne/urls.py and /admin/* per Django.
|
||||
location /library/ {
|
||||
proxy_pass http://mnemosyne_web;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 300s;
|
||||
}
|
||||
|
||||
location /admin/ {
|
||||
proxy_pass http://mnemosyne_web;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 300s;
|
||||
}
|
||||
|
||||
# FastMCP Streamable HTTP at /mcp/ and SSE at /mcp/sse/.
|
||||
# Long-running streams need disabled buffering and a generous timeout.
|
||||
location /mcp/ {
|
||||
proxy_pass http://mnemosyne_mcp;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
proxy_buffering off;
|
||||
proxy_cache off;
|
||||
proxy_read_timeout 600s;
|
||||
}
|
||||
|
||||
# Static files baked into the image at /app/staticfiles, mounted into
|
||||
# this nginx via a named volume populated by the web service.
|
||||
location /static/ {
|
||||
alias /var/www/static/;
|
||||
access_log off;
|
||||
expires 30d;
|
||||
}
|
||||
|
||||
# Prometheus scrape endpoint — internal networks only.
|
||||
# Allows: localhost + RFC1918 private ranges (10/8, 172.16/12, 192.168/16).
|
||||
location /metrics {
|
||||
allow 127.0.0.0/8;
|
||||
allow 10.0.0.0/8;
|
||||
allow 172.16.0.0/12;
|
||||
allow 192.168.0.0/16;
|
||||
deny all;
|
||||
proxy_pass http://mnemosyne_web;
|
||||
access_log off;
|
||||
}
|
||||
|
||||
# Liveness probe — proxies through to the MCP health endpoint.
|
||||
location = /healthz {
|
||||
proxy_pass http://mnemosyne_mcp/mcp/health;
|
||||
access_log off;
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,17 @@ dev = [
|
||||
"django-debug-toolbar>=4.0,<5.0",
|
||||
"docker>=7.0,<8.0",
|
||||
]
|
||||
test = [
|
||||
"pytest>=8.0,<9.0",
|
||||
"pytest-django>=4.8,<5.0",
|
||||
]
|
||||
lint = [
|
||||
"ruff>=0.6,<1.0",
|
||||
]
|
||||
docs = [
|
||||
"mkdocs>=1.6,<2.0",
|
||||
"mkdocs-material>=9.5,<10.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=68.0"]
|
||||
|
||||
6
validator/.gitignore
vendored
Normal file
6
validator/.gitignore
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
fastagent.secrets.yaml
|
||||
fastagent.jsonl
|
||||
.venv/
|
||||
*.egg-info/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
101
validator/README.md
Normal file
101
validator/README.md
Normal file
@@ -0,0 +1,101 @@
|
||||
# Mnemosyne Validator
|
||||
|
||||
A bare [FastAgent](https://github.com/evalstate/fast-agent) + [Pallas](https://git.helu.ca/r/pallas) project whose only purpose is to exercise Mnemosyne's MCP server end-to-end. Use it to confirm the transport works, every MCP tool registers, args/responses round-trip, and the local LLM can actually drive the tools.
|
||||
|
||||
This is **not** a production agent. It does not represent the long-term Daedalus integration — when Daedalus ships, it will inject `workspace_id` server-side. The validator never sets `workspace_id`, meaning all calls run against the global scope (libraries with `workspace_id IS NULL`).
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
validator/
|
||||
├── pyproject.toml # pallas-mcp + fast-agent-mcp deps
|
||||
├── agents.yaml # one-agent Pallas topology
|
||||
├── fastagent.config.yaml # default_model + mnemosyne MCP server
|
||||
├── fastagent.secrets.yaml.example # schema for the (gitignored) secrets file
|
||||
└── agents/
|
||||
└── mnemosyne_validator.py # the FastAgent definition
|
||||
```
|
||||
|
||||
All configuration lives in the FastAgent YAML files — no `.env` is used. The
|
||||
real `fastagent.secrets.yaml` is gitignored; `fastagent.secrets.yaml.example`
|
||||
documents its shape and is checked in.
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
cd validator/
|
||||
python3.13 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
Copy the secrets template and fill in the bearer token (see next section):
|
||||
|
||||
```bash
|
||||
cp fastagent.secrets.yaml.example fastagent.secrets.yaml
|
||||
```
|
||||
|
||||
## Provision an MCP bearer token
|
||||
|
||||
Mnemosyne requires a bearer token when `MCP_REQUIRE_AUTH=True` (the default). Generate one for your user:
|
||||
|
||||
```bash
|
||||
cd ../mnemosyne
|
||||
python manage.py create_mcp_token --user <username> --name validator
|
||||
```
|
||||
|
||||
The command prints the token **once** — paste it into `validator/fastagent.secrets.yaml` under `mcp.servers.mnemosyne.headers.Authorization` (keep the `Bearer ` prefix).
|
||||
|
||||
## Start Mnemosyne's MCP server
|
||||
|
||||
The validator hits the Mnemosyne ASGI endpoint, so Mnemosyne's MCP server must be running. From the Mnemosyne project:
|
||||
|
||||
```bash
|
||||
cd mnemosyne/
|
||||
uvicorn mnemosyne.asgi:app --host 0.0.0.0 --port 22091 --workers 1
|
||||
```
|
||||
|
||||
By default the validator points at `http://localhost:22091/mcp/` (note the trailing slash — Starlette's `Mount` only matches paths under the mount prefix, so `/mcp` without the slash falls through to Django). If your Mnemosyne is on another host, override `mcp.servers.mnemosyne.url` in `fastagent.secrets.yaml`.
|
||||
|
||||
## Run the validator
|
||||
|
||||
The `mnemosyne-validator` script is a thin alias for `pallas`:
|
||||
|
||||
```bash
|
||||
# Start with the registry (Pallas mode):
|
||||
mnemosyne-validator
|
||||
|
||||
# Or run the agent directly (no registry):
|
||||
mnemosyne-validator --agent mnemosyne_validator
|
||||
```
|
||||
|
||||
To chat with the agent directly without spinning up a Pallas registry, use the `fast-agent` CLI (provided by `fast-agent-mcp`):
|
||||
|
||||
```bash
|
||||
fast-agent go --config-path fastagent.config.yaml --url http://localhost:24301/mcp mnemosyne_validator
|
||||
```
|
||||
|
||||
## What to test
|
||||
|
||||
These prompts exercise every Mnemosyne MCP tool. After each, the agent should call the named tool and surface the result.
|
||||
|
||||
| Prompt | Tool | What to verify |
|
||||
|--------|------|----------------|
|
||||
| "Run a health check on Mnemosyne." | `get_health` | Returns `status: ok` if Neo4j + S3 + embedding model are all reachable. `degraded` if one is down. |
|
||||
| "List all libraries." | `list_libraries` | Returns the libraries seeded by `load_library_types`, each with `library_type` set. |
|
||||
| "List collections in library `<uid>`." | `list_collections` | Returns collections inside the named library. |
|
||||
| "List items in collection `<uid>`." | `list_items` | Returns items with `chunk_count` and `embedding_status`. |
|
||||
| "Search the technical libraries for `<query>`." | `search` | Returns ranked candidates with `chunk_uid`, `score`, `text_preview`, `library_type`. |
|
||||
| "Fetch the full text of chunk `<chunk_uid>`." | `get_chunk` | Returns the full chunk text from S3. |
|
||||
|
||||
If a call errors, the agent surfaces it verbatim — that's the failure mode you want.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"Invalid MCP token"** — token wasn't provisioned, was provisioned for a different user, or got mangled when pasted. Re-run `create_mcp_token` and paste again. Tokens are SHA-256 hashed at rest and can't be retrieved later.
|
||||
|
||||
**"Couldn't connect to Mnemosyne"** — the ASGI server isn't running, or it's bound to a different host/port than `mcp.servers.mnemosyne.url` says. Check `curl http://localhost:22091/mcp/health` returns `{"status":"ok"}`.
|
||||
|
||||
**"No system embedding model configured" in `get_health`** — `LLMModel.get_system_embedding_model()` returns nothing. Configure the embedding model via the Mnemosyne admin or `manage.py` before searches will work.
|
||||
|
||||
**Search returns zero candidates with no error** — Mnemosyne is reachable but has no embedded content yet. Upload an item and run `embed_item`, or use the Daedalus ingest endpoint, before re-testing search.
|
||||
17
validator/agents.yaml
Normal file
17
validator/agents.yaml
Normal file
@@ -0,0 +1,17 @@
|
||||
# Mnemosyne Validator — Pallas deployment topology
|
||||
#
|
||||
# A single-agent Pallas project whose only purpose is to validate the
|
||||
# Mnemosyne MCP server end-to-end. Not a production deployment.
|
||||
|
||||
name: mnemosyne-validator
|
||||
version: "0.1.0"
|
||||
host: localhost
|
||||
namespace: ca.helu.mnemosyne-validator
|
||||
registry_port: 24300
|
||||
|
||||
agents:
|
||||
mnemosyne_validator:
|
||||
module: agents.mnemosyne_validator
|
||||
port: 24301
|
||||
title: Mnemosyne Validator
|
||||
description: "Exercises Mnemosyne's MCP tools: search, get_chunk, list_*, get_health"
|
||||
0
validator/agents/__init__.py
Normal file
0
validator/agents/__init__.py
Normal file
56
validator/agents/mnemosyne_validator.py
Normal file
56
validator/agents/mnemosyne_validator.py
Normal file
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Mnemosyne Validator Agent
|
||||
|
||||
A bare FastAgent that wraps the Mnemosyne MCP server. Exists solely to
|
||||
exercise the Mnemosyne MCP transport, tool registration, and round-trip
|
||||
serialization — no production role.
|
||||
|
||||
Drive it from the CLI to confirm:
|
||||
- search works against the running Mnemosyne (vector + fulltext + graph)
|
||||
- get_chunk fetches full chunk text from S3
|
||||
- list_libraries / list_collections / list_items return the expected shape
|
||||
- get_health returns ok/degraded with the right dependency breakdown
|
||||
|
||||
When the Daedalus integration ships, the workspace_id parameter will be
|
||||
injected by Daedalus's chat path (force-overwritten before the call leaves
|
||||
Daedalus). This validator never sets it — meaning all calls go to the
|
||||
GLOBAL scope (libraries with workspace_id IS NULL).
|
||||
"""
|
||||
from fast_agent import FastAgent
|
||||
|
||||
fast = FastAgent("Mnemosyne Validator", parse_cli_args=False)
|
||||
|
||||
|
||||
@fast.agent(
|
||||
name="mnemosyne_validator",
|
||||
instruction="""You are a validator for the Mnemosyne knowledge base. Your
|
||||
job is to exercise its MCP tools when asked, report what you saw, and surface
|
||||
errors clearly.
|
||||
|
||||
You have direct access to Mnemosyne via these tools:
|
||||
|
||||
- search(query, library_uid?, library_type?, collection_uid?, limit?, rerank?, include_images?, search_types?)
|
||||
Hybrid retrieval. Returns ranked chunks with text_preview (~500 chars),
|
||||
chunk_uid, item_uid, item_title, library_type, score, source.
|
||||
- get_chunk(chunk_uid)
|
||||
Fetch the full text of a chunk by uid (typically obtained from search).
|
||||
- list_libraries(limit?, offset?)
|
||||
List libraries (uid, name, library_type, description).
|
||||
- list_collections(library_uid?, limit?, offset?)
|
||||
List collections, optionally filtered by parent library.
|
||||
- list_items(collection_uid?, library_uid?, limit?, offset?)
|
||||
List items (documents) with chunk_count, embedding_status, etc.
|
||||
- get_health()
|
||||
Health check: {status: ok|degraded|error, checks: {neo4j, s3, embedding}}.
|
||||
|
||||
When the user asks "what libraries exist", call list_libraries and report.
|
||||
When they ask a research question, call search and surface chunk_uid + score
|
||||
+ item_title for each candidate. If they want full text, call get_chunk.
|
||||
Show raw structured output, not flowery prose — this is a validation tool,
|
||||
not a chat assistant.
|
||||
|
||||
If a tool errors, paste the error message verbatim.""",
|
||||
servers=["mnemosyne"],
|
||||
)
|
||||
async def mnemosyne_validator():
|
||||
pass
|
||||
32
validator/fastagent.config.yaml
Normal file
32
validator/fastagent.config.yaml
Normal file
@@ -0,0 +1,32 @@
|
||||
# Mnemosyne Validator — FastAgent + MCP configuration
|
||||
#
|
||||
# Secrets (api_key, MCP bearer tokens) live in fastagent.secrets.yaml
|
||||
# (gitignored) and merge with this file at runtime.
|
||||
|
||||
# Local llama.cpp on Nyx (OpenAI-compatible). Override via
|
||||
# fastagent.secrets.yaml if you want to point at a different model server.
|
||||
default_model: openai.Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf
|
||||
|
||||
# Capabilities for the model — Pallas registers it with fast-agent's
|
||||
# ModelDatabase using these values. vision: true so we can validate image
|
||||
# round-trip later (search returns image candidates by default).
|
||||
model_capabilities:
|
||||
vision: true
|
||||
context_window: 192000
|
||||
max_output_tokens: 16384
|
||||
|
||||
# ── LLM Providers ───────────────────────────────────────────────────────────
|
||||
openai:
|
||||
base_url: "http://nyx.helu.ca:22079/v1"
|
||||
|
||||
# ── MCP Servers ─────────────────────────────────────────────────────────────
|
||||
mcp:
|
||||
servers:
|
||||
# Mnemosyne MCP server — Streamable HTTP at /mcp.
|
||||
# Default assumes the validator runs on the same host as Mnemosyne;
|
||||
# override the URL in fastagent.secrets.yaml or via Ansible if remote.
|
||||
mnemosyne:
|
||||
transport: http
|
||||
url: "http://localhost:22091/mcp/"
|
||||
# Bearer token in fastagent.secrets.yaml (provisioned via
|
||||
# `python manage.py create_mcp_token <user>`).
|
||||
22
validator/fastagent.secrets.yaml.example
Normal file
22
validator/fastagent.secrets.yaml.example
Normal file
@@ -0,0 +1,22 @@
|
||||
# Mnemosyne Validator — secrets template
|
||||
#
|
||||
# Copy to fastagent.secrets.yaml and fill in real values. The .yaml is
|
||||
# gitignored; the .yaml.example is committed.
|
||||
|
||||
# ── LLM provider keys ───────────────────────────────────────────────────────
|
||||
# Local llama.cpp doesn't authenticate, but fast-agent requires the key field
|
||||
# to be present. "0000" or any non-empty string is fine.
|
||||
openai:
|
||||
api_key: "0000"
|
||||
|
||||
# ── MCP server bearer tokens ────────────────────────────────────────────────
|
||||
mcp:
|
||||
servers:
|
||||
mnemosyne:
|
||||
headers:
|
||||
# Mnemosyne MCP server requires a bearer token when MCP_REQUIRE_AUTH=True.
|
||||
# Provision one with:
|
||||
# cd ../mnemosyne
|
||||
# python manage.py create_mcp_token --user <username> --name validator
|
||||
# then paste the printed token here (it is shown once and not retrievable).
|
||||
Authorization: "Bearer paste-mcp-token-here"
|
||||
23
validator/pyproject.toml
Normal file
23
validator/pyproject.toml
Normal file
@@ -0,0 +1,23 @@
|
||||
[project]
|
||||
name = "mnemosyne-validator"
|
||||
version = "0.1.0"
|
||||
description = "FastAgent + Pallas validator that talks to Mnemosyne's MCP server end-to-end"
|
||||
requires-python = ">=3.13"
|
||||
dependencies = [
|
||||
"pallas-mcp @ git+ssh://git@git.helu.ca:22022/r/pallas.git",
|
||||
"fast-agent-mcp>=0.6.10",
|
||||
"pyyaml>=6.0",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
mnemosyne-validator = "pallas.server:main"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.metadata]
|
||||
allow-direct-references = true
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["agents"]
|
||||
Reference in New Issue
Block a user