Compare commits
10 Commits
2df22941d2
...
236d9e2e74
| Author | SHA1 | Date | |
|---|---|---|---|
| 236d9e2e74 | |||
| 1cd556c3f6 | |||
| e2a6d45b77 | |||
| 97a14fb03a | |||
| 2a8a3d75b4 | |||
| 5527cf6bdb | |||
| f2af28d96d | |||
| c485a8560c | |||
| 33658fbc8d | |||
| 81426327bf |
120
.gitea/workflows/cve-scan-docker-build.yml
Normal file
120
.gitea/workflows/cve-scan-docker-build.yml
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
name: CVE Scan & Docker Build
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
pull_request:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
env:
|
||||||
|
REGISTRY: git.helu.ca
|
||||||
|
IMAGE_NAME: ${{ gitea.repository }}
|
||||||
|
TRIVY_SEVERITY: MEDIUM,HIGH,CRITICAL
|
||||||
|
TRIVY_NO_PROGRESS: "true"
|
||||||
|
TRIVY_DISABLE_VEX_NOTICE: "true"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
security-scan:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Install Trivy
|
||||||
|
run: |
|
||||||
|
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin
|
||||||
|
trivy --version
|
||||||
|
|
||||||
|
- name: Resolve full dependency set (incl. dev/test/lint/docs extras)
|
||||||
|
run: |
|
||||||
|
python3 -m venv /tmp/scanenv
|
||||||
|
/tmp/scanenv/bin/pip install --quiet pip-tools
|
||||||
|
/tmp/scanenv/bin/pip-compile pyproject.toml \
|
||||||
|
--extra dev --extra test --extra lint --extra docs \
|
||||||
|
-o requirements.txt --no-header --quiet --allow-unsafe
|
||||||
|
echo "Resolved $(grep -cv '^\s*\(#\|$\)' requirements.txt) pinned packages."
|
||||||
|
|
||||||
|
- name: Scan Python dependencies for CVEs
|
||||||
|
run: |
|
||||||
|
trivy fs \
|
||||||
|
--scanners vuln \
|
||||||
|
--severity ${TRIVY_SEVERITY} \
|
||||||
|
--format table \
|
||||||
|
--exit-code 0 \
|
||||||
|
requirements.txt
|
||||||
|
|
||||||
|
- name: Scan repository for secrets
|
||||||
|
run: |
|
||||||
|
trivy fs \
|
||||||
|
--scanners secret \
|
||||||
|
--format table \
|
||||||
|
--exit-code 0 \
|
||||||
|
.
|
||||||
|
|
||||||
|
build-and-push:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: security-scan
|
||||||
|
if: always()
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
|
- name: Log in to Gitea Container Registry
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
registry: ${{ env.REGISTRY }}
|
||||||
|
username: ${{ gitea.actor }}
|
||||||
|
password: ${{ secrets.PACKAGE_TOKEN }}
|
||||||
|
|
||||||
|
- name: Extract metadata for Docker
|
||||||
|
id: meta
|
||||||
|
uses: docker/metadata-action@v5
|
||||||
|
with:
|
||||||
|
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||||
|
tags: |
|
||||||
|
type=ref,event=branch
|
||||||
|
type=ref,event=pr
|
||||||
|
type=sha,prefix=
|
||||||
|
type=raw,value=latest,enable=${{ gitea.ref == 'refs/heads/main' }}
|
||||||
|
|
||||||
|
- name: Build and push Docker image
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: Dockerfile
|
||||||
|
push: true
|
||||||
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
|
cache-from: type=gha
|
||||||
|
cache-to: type=gha,mode=max
|
||||||
|
|
||||||
|
- name: Install Trivy
|
||||||
|
run: |
|
||||||
|
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin
|
||||||
|
trivy --version
|
||||||
|
|
||||||
|
- name: Scan built Docker image (OS + Python + system libs)
|
||||||
|
run: |
|
||||||
|
IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1)
|
||||||
|
echo "🔍 Scanning image: ${IMAGE_TAG}"
|
||||||
|
trivy image \
|
||||||
|
--scanners vuln \
|
||||||
|
--severity ${TRIVY_SEVERITY} \
|
||||||
|
--format table \
|
||||||
|
--pkg-types os,library \
|
||||||
|
--exit-code 0 \
|
||||||
|
"${IMAGE_TAG}"
|
||||||
|
|
||||||
|
- name: Scan built Docker image for misconfigurations
|
||||||
|
continue-on-error: true
|
||||||
|
run: |
|
||||||
|
IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1)
|
||||||
|
trivy image \
|
||||||
|
--scanners misconfig \
|
||||||
|
--severity ${TRIVY_SEVERITY} \
|
||||||
|
--format table \
|
||||||
|
--exit-code 0 \
|
||||||
|
"${IMAGE_TAG}"
|
||||||
93
Dockerfile
Normal file
93
Dockerfile
Normal file
@@ -0,0 +1,93 @@
|
|||||||
|
# =============================================================================
|
||||||
|
# Mnemosyne — production image
|
||||||
|
# =============================================================================
|
||||||
|
# Multi-stage:
|
||||||
|
# builder installs Python deps and runs `collectstatic` once.
|
||||||
|
# runtime copies only the artifacts the running process needs.
|
||||||
|
#
|
||||||
|
# The same image runs three different processes (Django web, MCP server,
|
||||||
|
# Celery worker) — the compose file picks the command per service.
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
# ── Stage 1: builder ────────────────────────────────────────────────────────
|
||||||
|
FROM python:3.12-slim AS builder
|
||||||
|
|
||||||
|
# Build deps for psycopg, PyMuPDF, Pillow, cryptography, etc.
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
build-essential \
|
||||||
|
libpq-dev \
|
||||||
|
libffi-dev \
|
||||||
|
libssl-dev \
|
||||||
|
libjpeg-dev \
|
||||||
|
zlib1g-dev \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
PYTHONUNBUFFERED=1 \
|
||||||
|
PIP_NO_CACHE_DIR=1 \
|
||||||
|
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||||
|
|
||||||
|
WORKDIR /build
|
||||||
|
|
||||||
|
# Install dependencies first (better layer caching).
|
||||||
|
COPY pyproject.toml README.md ./
|
||||||
|
COPY mnemosyne/ ./mnemosyne/
|
||||||
|
|
||||||
|
RUN pip install --upgrade pip \
|
||||||
|
&& pip install .
|
||||||
|
|
||||||
|
# Bake static files into the image. The env vars below are build-time-only
|
||||||
|
# stubs needed for settings.py to import without real infrastructure — they
|
||||||
|
# never reach the runtime image because this is the builder stage.
|
||||||
|
# Inlined into the RUN command (rather than ENV/ARG) so static analysis
|
||||||
|
# tools (Trivy) don't flag them as baked-in secrets.
|
||||||
|
ENV DJANGO_SETTINGS_MODULE=mnemosyne.settings \
|
||||||
|
DEBUG=False \
|
||||||
|
USE_LOCAL_STORAGE=True \
|
||||||
|
APP_DB_NAME=collectstatic \
|
||||||
|
APP_DB_USER=collectstatic
|
||||||
|
|
||||||
|
WORKDIR /build/mnemosyne
|
||||||
|
RUN SECRET_KEY=collectstatic-stub \
|
||||||
|
APP_DB_PASSWORD=collectstatic-stub \
|
||||||
|
python manage.py collectstatic --noinput --clear
|
||||||
|
|
||||||
|
# ── Stage 2: runtime ────────────────────────────────────────────────────────
|
||||||
|
FROM python:3.12-slim AS runtime
|
||||||
|
|
||||||
|
# Runtime libs for psycopg + PyMuPDF + Pillow + cryptography.
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
libpq5 \
|
||||||
|
libjpeg62-turbo \
|
||||||
|
zlib1g \
|
||||||
|
libssl3 \
|
||||||
|
ca-certificates \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
PYTHONUNBUFFERED=1 \
|
||||||
|
DJANGO_SETTINGS_MODULE=mnemosyne.settings \
|
||||||
|
PATH=/usr/local/bin:$PATH
|
||||||
|
|
||||||
|
# Copy installed packages from the builder.
|
||||||
|
COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
|
||||||
|
COPY --from=builder /usr/local/bin /usr/local/bin
|
||||||
|
|
||||||
|
# Application code + collected statics.
|
||||||
|
WORKDIR /app
|
||||||
|
COPY --from=builder /build/mnemosyne /app
|
||||||
|
COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||||
|
RUN chmod +x /usr/local/bin/entrypoint.sh
|
||||||
|
|
||||||
|
# Non-root user for everything that runs in this image. uid:gid 1000:1000
|
||||||
|
# matches the convention for a single-application container.
|
||||||
|
RUN groupadd --gid 1000 mnemosyne \
|
||||||
|
&& useradd --uid 1000 --gid mnemosyne --home /app --no-create-home --shell /sbin/nologin mnemosyne \
|
||||||
|
&& mkdir -p /app/media /app/logs \
|
||||||
|
&& chown -R mnemosyne:mnemosyne /app
|
||||||
|
USER mnemosyne
|
||||||
|
|
||||||
|
# The compose file overrides this per service. Default = Django web.
|
||||||
|
EXPOSE 8000 22091
|
||||||
|
ENTRYPOINT ["entrypoint.sh"]
|
||||||
|
CMD ["web"]
|
||||||
159
README.md
159
README.md
@@ -37,11 +37,14 @@ This **content-type awareness** flows through every layer: chunking strategy, em
|
|||||||
| Library | Example Content | Multimodal? | Graph Relationships |
|
| Library | Example Content | Multimodal? | Graph Relationships |
|
||||||
|---------|----------------|-------------|-------------------|
|
|---------|----------------|-------------|-------------------|
|
||||||
| **Fiction** | Novels, short stories | Cover art | Author → Book → Character → Theme |
|
| **Fiction** | Novels, short stories | Cover art | Author → Book → Character → Theme |
|
||||||
|
| **Nonfiction** | History, biography, science writing | Photos, charts | Author → Work → Topic → Person/Place |
|
||||||
| **Technical** | Textbooks, manuals, docs | Diagrams, screenshots | Product → Manual → Section → Procedure |
|
| **Technical** | Textbooks, manuals, docs | Diagrams, screenshots | Product → Manual → Section → Procedure |
|
||||||
| **Music** | Lyrics, liner notes | Album artwork | Artist → Album → Track → Genre |
|
| **Music** | Lyrics, liner notes | Album artwork | Artist → Album → Track → Genre |
|
||||||
| **Film** | Scripts, synopses | Stills, posters | Director → Film → Scene → Actor |
|
| **Film** | Scripts, synopses | Stills, posters | Director → Film → Scene → Actor |
|
||||||
| **Art** | Descriptions, catalogs | The artwork itself | Artist → Piece → Style → Movement |
|
| **Art** | Descriptions, catalogs | The artwork itself | Artist → Piece → Style → Movement |
|
||||||
| **Journals** | Personal entries | Photos | Date → Entry → Topic → Person/Place |
|
| **Journal** | Personal entries, plans, observations | Photos | Date → Entry → Topic → Person/Place |
|
||||||
|
| **Business** | Proposals, marketing, strategy | Logos, charts | Client → Engagement → Deliverable |
|
||||||
|
| **Finance** | Statements, tax, market commentary | Charts, statement scans | Account → Instrument → Period |
|
||||||
|
|
||||||
## Search Pipeline
|
## Search Pipeline
|
||||||
|
|
||||||
@@ -55,32 +58,160 @@ Query → Vector Search (Neo4j) + Graph Traversal (Cypher) + Full-Text Search
|
|||||||
|
|
||||||
Mnemosyne's RAG pipeline architecture is inspired by [Spelunker](https://git.helu.ca/r/spelunker), an enterprise RFP response platform. The proven patterns — hybrid search, two-stage RAG (responder + reviewer), citation-based retrieval, and async document processing — are carried forward and enhanced with multimodal capabilities and knowledge graph relationships.
|
Mnemosyne's RAG pipeline architecture is inspired by [Spelunker](https://git.helu.ca/r/spelunker), an enterprise RFP response platform. The proven patterns — hybrid search, two-stage RAG (responder + reviewer), citation-based retrieval, and async document processing — are carried forward and enhanced with multimodal capabilities and knowledge graph relationships.
|
||||||
|
|
||||||
## Running Celery Workers
|
## Running Mnemosyne
|
||||||
|
|
||||||
Mnemosyne uses Celery with RabbitMQ for async document embedding. From the `mnemosyne/` directory:
|
Mnemosyne runs as three cooperating processes: the Django web app (REST API + admin), the MCP server (LLM-facing tools), and one or more Celery workers (async embedding + ingest). All three read configuration from `mnemosyne/.env` (copy from `mnemosyne/.env example` and fill in secrets).
|
||||||
|
|
||||||
|
Hosts in the Ouranos lab:
|
||||||
|
- **Postgres** — `portia.incus:5432` (Django ORM: users, IngestJob)
|
||||||
|
- **Neo4j** — `ariel.incus:25554` (knowledge graph + vectors)
|
||||||
|
- **RabbitMQ** — `oberon.incus:5672` (Celery broker)
|
||||||
|
- **MinIO** — `nyx.helu.ca:8555` (S3-compatible; `mnemosyne-content` and `daedalus` buckets)
|
||||||
|
- **Memcached** — `127.0.0.1:11211` (task progress)
|
||||||
|
|
||||||
|
### One-time setup
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Development — single worker, all queues
|
cd mnemosyne/
|
||||||
|
python manage.py migrate # Apply Django ORM migrations
|
||||||
|
python manage.py setup_neo4j_indexes # Create Neo4j vector + full-text indexes
|
||||||
|
python manage.py load_library_types # Load LIBRARY_TYPE_DEFAULTS into Neo4j
|
||||||
|
```
|
||||||
|
|
||||||
|
### Start the web app
|
||||||
|
|
||||||
|
The Django REST API serves `/library/api/*` (libraries, collections, items, search, workspaces, ingest) and Django admin. Use Gunicorn in production; `runserver` for dev.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd mnemosyne/
|
||||||
|
|
||||||
|
# Development
|
||||||
|
python manage.py runserver 0.0.0.0:8000
|
||||||
|
|
||||||
|
# Production
|
||||||
|
gunicorn --bind 0.0.0.0:8000 --workers 3 mnemosyne.wsgi:application
|
||||||
|
```
|
||||||
|
|
||||||
|
### Start the MCP server
|
||||||
|
|
||||||
|
The MCP server exposes the LLM-facing tools (`search`, `get_chunk`, `list_libraries`, `list_collections`, `list_items`, `get_health`) over Streamable HTTP at `/mcp` and SSE at `/mcp/sse`. Run as a separate Uvicorn process, on its own port, so it can be reverse-proxied or scaled independently of the Django app.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd mnemosyne/
|
||||||
|
|
||||||
|
# Single command: ASGI server hosting the FastMCP app
|
||||||
|
uvicorn mnemosyne.asgi:app --host 0.0.0.0 --port 22091 --workers 1
|
||||||
|
```
|
||||||
|
|
||||||
|
The `mcp_server/asgi.py` mounts FastMCP at `/mcp` (Streamable HTTP) and `/mcp/sse` (SSE), with a `/mcp/health` JSON probe for HAProxy/Pallas.
|
||||||
|
|
||||||
|
### Start a Celery worker
|
||||||
|
|
||||||
|
A single worker that handles all queues (development) plus the focused command Daedalus depends on (the `embedding` queue, where the Daedalus ingest task lives).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd mnemosyne/
|
||||||
|
|
||||||
|
# Development — one worker, all queues
|
||||||
celery -A mnemosyne worker -l info -Q celery,embedding,batch
|
celery -A mnemosyne worker -l info -Q celery,embedding,batch
|
||||||
|
|
||||||
# Or skip workers entirely with eager mode (.env):
|
# Production — embedding queue (handles Daedalus ingest + embed_item)
|
||||||
CELERY_TASK_ALWAYS_EAGER=True
|
celery -A mnemosyne worker -l info -Q embedding -c 1 -n embedding@%h
|
||||||
|
|
||||||
|
# Production — batch queue (collection/library bulk operations)
|
||||||
|
celery -A mnemosyne worker -l info -Q batch -c 2 -n batch@%h
|
||||||
|
|
||||||
|
# Production — default queue (LLM validation, misc)
|
||||||
|
celery -A mnemosyne worker -l info -Q celery -c 2 -n default@%h
|
||||||
```
|
```
|
||||||
|
|
||||||
**Production — separate workers:**
|
Daedalus's `POST /library/api/ingest/` dispatches `library.tasks.ingest_from_daedalus` to the **embedding** queue. If you only run one worker, make sure it consumes `embedding` or that task will sit in the broker.
|
||||||
```bash
|
|
||||||
celery -A mnemosyne worker -l info -Q embedding -c 1 -n embedding@%h # GPU-bound embedding
|
|
||||||
celery -A mnemosyne worker -l info -Q batch -c 2 -n batch@%h # Batch orchestration
|
|
||||||
celery -A mnemosyne worker -l info -Q celery -c 2 -n default@%h # LLM API validation
|
|
||||||
```
|
|
||||||
|
|
||||||
**Scheduler & Monitoring:**
|
To bypass workers in dev/test, set `CELERY_TASK_ALWAYS_EAGER=True` in `.env`.
|
||||||
|
|
||||||
|
**Scheduler & monitoring (optional):**
|
||||||
```bash
|
```bash
|
||||||
celery -A mnemosyne beat -l info # Periodic task scheduler
|
celery -A mnemosyne beat -l info # Periodic task scheduler
|
||||||
celery -A mnemosyne flower --port=5555 # Web monitoring UI
|
celery -A mnemosyne flower --port=5555 # Web monitoring UI
|
||||||
```
|
```
|
||||||
|
|
||||||
See [Phase 2: Celery Workers & Scheduler](docs/PHASE_2_EMBEDDING_PIPELINE.md#celery-workers--scheduler) for full details on queues, reliability settings, and task progress tracking.
|
See [Phase 2: Celery Workers & Scheduler](docs/PHASE_2_EMBEDDING_PIPELINE.md#celery-workers--scheduler) for queue tuning, reliability settings, and task progress tracking.
|
||||||
|
|
||||||
|
### Daedalus integration endpoints
|
||||||
|
|
||||||
|
These endpoints are used by the Daedalus FastAPI backend (HTTP Basic auth). All under `/library/api/`:
|
||||||
|
|
||||||
|
| Method | Route | Purpose |
|
||||||
|
|--------|-------|---------|
|
||||||
|
| POST | `/workspaces/` | Create a workspace (idempotent on `workspace_id`); body: `{workspace_id, name, library_type, description?}` |
|
||||||
|
| GET | `/workspaces/{workspace_id}/` | Workspace status (item/chunk counts) |
|
||||||
|
| DELETE | `/workspaces/{workspace_id}/` | Delete workspace + reachable content; preserves shared concepts |
|
||||||
|
| POST | `/ingest/` | Queue a file for ingestion + embedding |
|
||||||
|
| GET | `/jobs/{job_id}/` | Poll ingest job status |
|
||||||
|
| POST | `/jobs/{job_id}/retry/` | Re-dispatch a failed job |
|
||||||
|
| GET | `/jobs/?status=&library_uid=` | List recent jobs |
|
||||||
|
|
||||||
|
See [docs/mnemosyne_integration.md](docs/mnemosyne_integration.md) for the full Daedalus contract.
|
||||||
|
|
||||||
|
## Production Deployment
|
||||||
|
|
||||||
|
Production runs as four containers from a single image (built and pushed by [`.gitea/workflows/cve-scan-docker-build.yml`](.gitea/workflows/cve-scan-docker-build.yml) on every push to `main`):
|
||||||
|
|
||||||
|
| Service | Role | Port |
|
||||||
|
|---------|------|------|
|
||||||
|
| `web` | Django REST API + admin (gunicorn) | internal :8000 |
|
||||||
|
| `mcp` | FastMCP server (uvicorn) | internal :22091 |
|
||||||
|
| `worker` | Celery worker — embedding/ingest/batch | — |
|
||||||
|
| `nginx` | Reverse proxy + static files | host :23090 |
|
||||||
|
|
||||||
|
Plus a one-shot `static-init` service that copies `/app/staticfiles` (baked into the image at build time via `collectstatic`) into the shared volume nginx reads from. It runs to completion on every `up`, so static-file changes propagate on each deploy without manual intervention.
|
||||||
|
|
||||||
|
External services (NOT spun up by compose): Postgres on Portia, Neo4j on Ariel, RabbitMQ on Oberon, S3/MinIO on Nyx, Memcached, embedder + reranker. All reached over the internal 10.10.0.0/24 network. URLs and credentials live in `mnemosyne/.env`.
|
||||||
|
|
||||||
|
### First-time bring-up
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Pull the image (or build locally with `docker compose build`)
|
||||||
|
docker compose pull
|
||||||
|
|
||||||
|
# DB migrations (one-shot)
|
||||||
|
docker compose run --rm web migrate
|
||||||
|
|
||||||
|
# Neo4j indexes + library_type defaults (one-shot)
|
||||||
|
docker compose run --rm web setup
|
||||||
|
|
||||||
|
# Bring the stack up
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
### Day-to-day
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose ps # service status + health
|
||||||
|
docker compose logs -f web # tail web logs
|
||||||
|
docker compose logs -f worker # tail Celery worker logs
|
||||||
|
docker compose restart mcp # restart just the MCP server
|
||||||
|
|
||||||
|
# After a new image is published:
|
||||||
|
docker compose pull && docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
### Things to verify in `mnemosyne/.env` before bringing up
|
||||||
|
|
||||||
|
The development `.env` has a few values that need adjusting for production:
|
||||||
|
|
||||||
|
- `DEBUG=False`
|
||||||
|
- `USE_LOCAL_STORAGE=False` (already set; just confirm)
|
||||||
|
- `KVDB_LOCATION=<external-memcached-host>:11211` — `127.0.0.1` does not resolve from inside containers
|
||||||
|
- `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` filled in
|
||||||
|
- `DAEDALUS_S3_*` filled in for cross-bucket reads from the Daedalus bucket
|
||||||
|
- `ALLOWED_HOSTS` includes the public hostname HAProxy routes to (e.g. `mnemosyne.ouranos.helu.ca`)
|
||||||
|
- `LLM_API_SECRETS_ENCRYPTION_KEY` set to a real Fernet key
|
||||||
|
|
||||||
|
### Health probes
|
||||||
|
|
||||||
|
- `GET http://nginx-host:23090/healthz` → proxies to `/mcp/health`, returns `{"status":"ok"}` when the MCP server is up
|
||||||
|
- `GET http://nginx-host:23090/metrics` → Prometheus scrape endpoint, internal-network-only
|
||||||
|
|
||||||
## Architecture Note: Retrieval, Not Synthesis
|
## Architecture Note: Retrieval, Not Synthesis
|
||||||
|
|
||||||
|
|||||||
111
docker-compose.yaml
Normal file
111
docker-compose.yaml
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
# =============================================================================
|
||||||
|
# Mnemosyne — production deployment
|
||||||
|
# =============================================================================
|
||||||
|
# Four services, all from the same image:
|
||||||
|
# web — Django REST API + admin (gunicorn, port 8000)
|
||||||
|
# mcp — FastMCP server (uvicorn, port 22091)
|
||||||
|
# worker — Celery worker (embedding/ingest/batch queues)
|
||||||
|
# nginx — reverse proxy, public port 23090
|
||||||
|
#
|
||||||
|
# External services (NOT spun up here): Postgres on Portia, Neo4j on Ariel,
|
||||||
|
# RabbitMQ on Oberon, S3/MinIO on Nyx, Memcached on its own host, embedder
|
||||||
|
# and reranker on Nyx, smtp4dev on Oberon. All reached over the internal
|
||||||
|
# 10.10.0.0/24 network.
|
||||||
|
#
|
||||||
|
# Run:
|
||||||
|
# docker compose up -d
|
||||||
|
# docker compose run --rm web migrate # one-shot DB migrate
|
||||||
|
# docker compose run --rm web setup # Neo4j indexes + library types
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
services:
|
||||||
|
# ── Static-file seeder: copies /app/staticfiles into the shared volume on
|
||||||
|
# every `up`. Runs once and exits. Without this, the named volume is only
|
||||||
|
# seeded the first time it's empty, so static updates between deploys
|
||||||
|
# would not propagate to nginx.
|
||||||
|
static-init:
|
||||||
|
image: git.helu.ca/r/mnemosyne:latest
|
||||||
|
command: ["sh", "-c", "cp -a /app/staticfiles/. /shared-static/"]
|
||||||
|
user: "0:0"
|
||||||
|
volumes:
|
||||||
|
- mnemosyne-static:/shared-static
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
# ── Web app: Django REST API + admin ───────────────────────────────────────
|
||||||
|
web:
|
||||||
|
image: git.helu.ca/r/mnemosyne:latest
|
||||||
|
command: ["web"]
|
||||||
|
env_file: mnemosyne/.env
|
||||||
|
restart: unless-stopped
|
||||||
|
depends_on:
|
||||||
|
static-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
volumes:
|
||||||
|
- mnemosyne-media:/app/media
|
||||||
|
expose:
|
||||||
|
- "8000"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/admin/login/').read()"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
start_period: 30s
|
||||||
|
|
||||||
|
# ── MCP server: FastMCP Streamable HTTP at /mcp/ ───────────────────────────
|
||||||
|
mcp:
|
||||||
|
image: git.helu.ca/r/mnemosyne:latest
|
||||||
|
command: ["mcp"]
|
||||||
|
env_file: mnemosyne/.env
|
||||||
|
restart: unless-stopped
|
||||||
|
volumes:
|
||||||
|
- mnemosyne-media:/app/media
|
||||||
|
expose:
|
||||||
|
- "22091"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:22091/mcp/health').read()"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
start_period: 30s
|
||||||
|
|
||||||
|
# ── Celery worker: embedding + ingest + batch queues ───────────────────────
|
||||||
|
worker:
|
||||||
|
image: git.helu.ca/r/mnemosyne:latest
|
||||||
|
command: ["worker"]
|
||||||
|
env_file: mnemosyne/.env
|
||||||
|
restart: unless-stopped
|
||||||
|
volumes:
|
||||||
|
- mnemosyne-media:/app/media
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "celery", "-A", "mnemosyne", "inspect", "ping", "-d", "celery@$$HOSTNAME"]
|
||||||
|
interval: 60s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 3
|
||||||
|
start_period: 60s
|
||||||
|
|
||||||
|
# ── nginx: reverse proxy, public port 23090 ────────────────────────────────
|
||||||
|
nginx:
|
||||||
|
image: nginx:alpine
|
||||||
|
restart: unless-stopped
|
||||||
|
depends_on:
|
||||||
|
- web
|
||||||
|
- mcp
|
||||||
|
ports:
|
||||||
|
- "23090:80"
|
||||||
|
volumes:
|
||||||
|
- ./nginx/mnemosyne.conf:/etc/nginx/conf.d/default.conf:ro
|
||||||
|
- mnemosyne-static:/var/www/static:ro
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "-qO-", "http://localhost/healthz"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
# Static files baked into the image at /app/staticfiles. The web service
|
||||||
|
# mounts this volume, populating it on first start; nginx reads from it.
|
||||||
|
mnemosyne-static:
|
||||||
|
# Local FileSystemStorage fallback. Production uses USE_LOCAL_STORAGE=False
|
||||||
|
# so this is mostly empty — kept for parity with dev and for any path
|
||||||
|
# that writes to MEDIA_ROOT directly.
|
||||||
|
mnemosyne-media:
|
||||||
66
docker/entrypoint.sh
Normal file
66
docker/entrypoint.sh
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Mnemosyne container entrypoint.
|
||||||
|
#
|
||||||
|
# The same image runs all three processes — the compose service supplies
|
||||||
|
# `web`, `mcp`, `worker`, or `migrate` as CMD.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
case "$1" in
|
||||||
|
web)
|
||||||
|
# Django REST API + admin (gunicorn → wsgi).
|
||||||
|
exec gunicorn \
|
||||||
|
--bind 0.0.0.0:8000 \
|
||||||
|
--workers "${GUNICORN_WORKERS:-3}" \
|
||||||
|
--access-logfile - \
|
||||||
|
--error-logfile - \
|
||||||
|
mnemosyne.wsgi:application
|
||||||
|
;;
|
||||||
|
|
||||||
|
mcp)
|
||||||
|
# FastMCP over Streamable HTTP at /mcp/, mounted by mnemosyne.asgi.
|
||||||
|
exec uvicorn \
|
||||||
|
--host 0.0.0.0 \
|
||||||
|
--port 22091 \
|
||||||
|
--workers "${UVICORN_WORKERS:-1}" \
|
||||||
|
mnemosyne.asgi:app
|
||||||
|
;;
|
||||||
|
|
||||||
|
worker)
|
||||||
|
# Celery worker covering embedding + ingest + batch + default queues.
|
||||||
|
# In production you may want to split these onto separate worker
|
||||||
|
# services for queue-level isolation; one process is fine to start.
|
||||||
|
exec celery -A mnemosyne worker \
|
||||||
|
--loglevel="${CELERY_LOG_LEVEL:-info}" \
|
||||||
|
--queues="${CELERY_QUEUES:-celery,embedding,batch}" \
|
||||||
|
--concurrency="${CELERY_CONCURRENCY:-2}"
|
||||||
|
;;
|
||||||
|
|
||||||
|
beat)
|
||||||
|
# Celery scheduled tasks (only needed if/when periodic jobs are wired).
|
||||||
|
exec celery -A mnemosyne beat \
|
||||||
|
--loglevel="${CELERY_LOG_LEVEL:-info}"
|
||||||
|
;;
|
||||||
|
|
||||||
|
migrate)
|
||||||
|
# One-shot DB migration runner — invoke before bringing services up
|
||||||
|
# for the first time or after a deploy.
|
||||||
|
exec python manage.py migrate --noinput
|
||||||
|
;;
|
||||||
|
|
||||||
|
setup)
|
||||||
|
# One-shot init — Neo4j indexes + library_type seed data.
|
||||||
|
python manage.py setup_neo4j_indexes
|
||||||
|
python manage.py load_library_types
|
||||||
|
;;
|
||||||
|
|
||||||
|
shell)
|
||||||
|
# Drop into the management shell for ad-hoc work.
|
||||||
|
exec python manage.py shell
|
||||||
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
# Fall through: run whatever was passed (e.g. `manage.py <cmd>`).
|
||||||
|
exec "$@"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
@@ -40,6 +40,23 @@ AWS_S3_REGION_NAME=us-east-1
|
|||||||
# Set to True to use local FileSystemStorage instead of S3 (dev/test)
|
# Set to True to use local FileSystemStorage instead of S3 (dev/test)
|
||||||
USE_LOCAL_STORAGE=True
|
USE_LOCAL_STORAGE=True
|
||||||
|
|
||||||
|
# --- Daedalus S3 (cross-bucket reads for ingest) ---
|
||||||
|
# Mnemosyne ingests files from the Daedalus S3 bucket. These vars
|
||||||
|
# configure read access; the file is copied into AWS_STORAGE_BUCKET_NAME
|
||||||
|
# (Mnemosyne's own bucket) by the ingest Celery task.
|
||||||
|
DAEDALUS_S3_ENDPOINT_URL=
|
||||||
|
DAEDALUS_S3_ACCESS_KEY_ID=
|
||||||
|
DAEDALUS_S3_SECRET_ACCESS_KEY=
|
||||||
|
DAEDALUS_S3_BUCKET_NAME=daedalus
|
||||||
|
DAEDALUS_S3_REGION_NAME=us-east-1
|
||||||
|
DAEDALUS_S3_USE_SSL=False
|
||||||
|
DAEDALUS_S3_VERIFY=True
|
||||||
|
|
||||||
|
# --- MCP Server ---
|
||||||
|
# Set to False for internal-only deployments where the MCP transport is
|
||||||
|
# already on a trusted network (10.10.0.0/24).
|
||||||
|
MCP_REQUIRE_AUTH=True
|
||||||
|
|
||||||
# --- Email (smtp4dev on Oberon) ---
|
# --- Email (smtp4dev on Oberon) ---
|
||||||
EMAIL_HOST=oberon.incus
|
EMAIL_HOST=oberon.incus
|
||||||
EMAIL_PORT=22025
|
EMAIL_PORT=22025
|
||||||
|
|||||||
@@ -7,12 +7,23 @@ Serialize Neo4j neomodel nodes into JSON for the REST API.
|
|||||||
from rest_framework import serializers
|
from rest_framework import serializers
|
||||||
|
|
||||||
|
|
||||||
|
LIBRARY_TYPE_CHOICES = [
|
||||||
|
"fiction",
|
||||||
|
"nonfiction",
|
||||||
|
"technical",
|
||||||
|
"music",
|
||||||
|
"film",
|
||||||
|
"art",
|
||||||
|
"journal",
|
||||||
|
"business",
|
||||||
|
"finance",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
class LibrarySerializer(serializers.Serializer):
|
class LibrarySerializer(serializers.Serializer):
|
||||||
uid = serializers.CharField(read_only=True)
|
uid = serializers.CharField(read_only=True)
|
||||||
name = serializers.CharField(max_length=200)
|
name = serializers.CharField(max_length=200)
|
||||||
library_type = serializers.ChoiceField(
|
library_type = serializers.ChoiceField(choices=LIBRARY_TYPE_CHOICES)
|
||||||
choices=["fiction", "nonfiction", "technical", "music", "film", "art", "journal"]
|
|
||||||
)
|
|
||||||
description = serializers.CharField(required=False, allow_blank=True, default="")
|
description = serializers.CharField(required=False, allow_blank=True, default="")
|
||||||
chunking_config = serializers.JSONField(required=False, default=dict)
|
chunking_config = serializers.JSONField(required=False, default=dict)
|
||||||
embedding_instruction = serializers.CharField(
|
embedding_instruction = serializers.CharField(
|
||||||
@@ -24,6 +35,7 @@ class LibrarySerializer(serializers.Serializer):
|
|||||||
llm_context_prompt = serializers.CharField(
|
llm_context_prompt = serializers.CharField(
|
||||||
required=False, allow_blank=True, default=""
|
required=False, allow_blank=True, default=""
|
||||||
)
|
)
|
||||||
|
workspace_id = serializers.CharField(read_only=True)
|
||||||
created_at = serializers.DateTimeField(read_only=True)
|
created_at = serializers.DateTimeField(read_only=True)
|
||||||
|
|
||||||
|
|
||||||
@@ -90,12 +102,11 @@ class SearchRequestSerializer(serializers.Serializer):
|
|||||||
query = serializers.CharField(max_length=2000)
|
query = serializers.CharField(max_length=2000)
|
||||||
library_uid = serializers.CharField(required=False, allow_blank=True)
|
library_uid = serializers.CharField(required=False, allow_blank=True)
|
||||||
library_type = serializers.ChoiceField(
|
library_type = serializers.ChoiceField(
|
||||||
choices=[
|
choices=LIBRARY_TYPE_CHOICES,
|
||||||
"fiction", "nonfiction", "technical", "music", "film", "art", "journal",
|
|
||||||
],
|
|
||||||
required=False,
|
required=False,
|
||||||
)
|
)
|
||||||
collection_uid = serializers.CharField(required=False, allow_blank=True)
|
collection_uid = serializers.CharField(required=False, allow_blank=True)
|
||||||
|
workspace_id = serializers.CharField(required=False, allow_blank=True)
|
||||||
search_types = serializers.ListField(
|
search_types = serializers.ListField(
|
||||||
child=serializers.ChoiceField(choices=["vector", "fulltext", "graph"]),
|
child=serializers.ChoiceField(choices=["vector", "fulltext", "graph"]),
|
||||||
required=False,
|
required=False,
|
||||||
@@ -139,3 +150,73 @@ class SearchResponseSerializer(serializers.Serializer):
|
|||||||
reranker_used = serializers.BooleanField()
|
reranker_used = serializers.BooleanField()
|
||||||
reranker_model = serializers.CharField(allow_null=True)
|
reranker_model = serializers.CharField(allow_null=True)
|
||||||
search_types_used = serializers.ListField(child=serializers.CharField())
|
search_types_used = serializers.ListField(child=serializers.CharField())
|
||||||
|
|
||||||
|
|
||||||
|
# --- Workspace lifecycle (Daedalus integration) ---
|
||||||
|
|
||||||
|
|
||||||
|
class WorkspaceCreateSerializer(serializers.Serializer):
|
||||||
|
"""Inbound payload for POST /api/v1/workspaces/."""
|
||||||
|
|
||||||
|
workspace_id = serializers.CharField(max_length=64)
|
||||||
|
name = serializers.CharField(max_length=200)
|
||||||
|
library_type = serializers.ChoiceField(choices=LIBRARY_TYPE_CHOICES)
|
||||||
|
description = serializers.CharField(required=False, allow_blank=True, default="")
|
||||||
|
|
||||||
|
|
||||||
|
class WorkspaceStatusSerializer(serializers.Serializer):
|
||||||
|
"""Outbound payload for workspace lifecycle endpoints."""
|
||||||
|
|
||||||
|
workspace_id = serializers.CharField()
|
||||||
|
library_uid = serializers.CharField()
|
||||||
|
name = serializers.CharField()
|
||||||
|
library_type = serializers.CharField()
|
||||||
|
description = serializers.CharField(allow_blank=True)
|
||||||
|
item_count = serializers.IntegerField()
|
||||||
|
chunk_count = serializers.IntegerField()
|
||||||
|
created_at = serializers.DateTimeField()
|
||||||
|
|
||||||
|
|
||||||
|
# --- Ingest (Daedalus integration) ---
|
||||||
|
|
||||||
|
|
||||||
|
class IngestRequestSerializer(serializers.Serializer):
|
||||||
|
"""Inbound payload for POST /api/v1/library/ingest/."""
|
||||||
|
|
||||||
|
s3_key = serializers.CharField(max_length=500)
|
||||||
|
title = serializers.CharField(max_length=500)
|
||||||
|
library_uid = serializers.CharField(required=False, allow_blank=True)
|
||||||
|
workspace_id = serializers.CharField(required=False, allow_blank=True)
|
||||||
|
collection_uid = serializers.CharField(required=False, allow_blank=True)
|
||||||
|
file_type = serializers.CharField(required=False, allow_blank=True, default="")
|
||||||
|
file_size = serializers.IntegerField(required=False, default=0)
|
||||||
|
content_hash = serializers.CharField(max_length=64)
|
||||||
|
source = serializers.CharField(required=False, allow_blank=True, default="")
|
||||||
|
source_ref = serializers.CharField(required=False, allow_blank=True, default="")
|
||||||
|
|
||||||
|
def validate(self, data):
|
||||||
|
if not data.get("library_uid") and not data.get("workspace_id"):
|
||||||
|
raise serializers.ValidationError(
|
||||||
|
"Either library_uid or workspace_id is required."
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
class IngestJobSerializer(serializers.Serializer):
|
||||||
|
"""Outbound payload for ingest job status."""
|
||||||
|
|
||||||
|
job_id = serializers.CharField(source="id")
|
||||||
|
item_uid = serializers.CharField(allow_blank=True)
|
||||||
|
library_uid = serializers.CharField()
|
||||||
|
status = serializers.CharField()
|
||||||
|
progress = serializers.CharField()
|
||||||
|
error = serializers.CharField(allow_null=True)
|
||||||
|
chunks_created = serializers.IntegerField()
|
||||||
|
concepts_extracted = serializers.IntegerField()
|
||||||
|
embedding_model = serializers.CharField(allow_blank=True)
|
||||||
|
content_hash = serializers.CharField(allow_blank=True)
|
||||||
|
source = serializers.CharField(allow_blank=True)
|
||||||
|
source_ref = serializers.CharField(allow_blank=True)
|
||||||
|
created_at = serializers.DateTimeField()
|
||||||
|
started_at = serializers.DateTimeField(allow_null=True)
|
||||||
|
completed_at = serializers.DateTimeField(allow_null=True)
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ URL patterns for the library DRF API.
|
|||||||
|
|
||||||
from django.urls import path
|
from django.urls import path
|
||||||
|
|
||||||
from . import views
|
from . import views, workspaces
|
||||||
|
|
||||||
app_name = "library-api"
|
app_name = "library-api"
|
||||||
|
|
||||||
@@ -28,4 +28,16 @@ urlpatterns = [
|
|||||||
# Concepts (Phase 3)
|
# Concepts (Phase 3)
|
||||||
path("concepts/", views.concept_list, name="concept-list"),
|
path("concepts/", views.concept_list, name="concept-list"),
|
||||||
path("concepts/<str:uid>/graph/", views.concept_graph, name="concept-graph"),
|
path("concepts/<str:uid>/graph/", views.concept_graph, name="concept-graph"),
|
||||||
|
# Workspaces (Daedalus integration)
|
||||||
|
path("workspaces/", workspaces.workspace_create, name="workspace-create"),
|
||||||
|
path(
|
||||||
|
"workspaces/<str:workspace_id>/",
|
||||||
|
workspaces.workspace_detail_or_delete,
|
||||||
|
name="workspace-detail",
|
||||||
|
),
|
||||||
|
# Ingest (Daedalus integration)
|
||||||
|
path("ingest/", views.ingest_create, name="ingest-create"),
|
||||||
|
path("jobs/", views.ingest_job_list, name="ingest-job-list"),
|
||||||
|
path("jobs/<str:job_id>/", views.ingest_job_detail, name="ingest-job-detail"),
|
||||||
|
path("jobs/<str:job_id>/retry/", views.ingest_job_retry, name="ingest-job-retry"),
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ from library.content_types import get_library_type_config
|
|||||||
from .serializers import (
|
from .serializers import (
|
||||||
CollectionSerializer,
|
CollectionSerializer,
|
||||||
ConceptSerializer,
|
ConceptSerializer,
|
||||||
|
IngestJobSerializer,
|
||||||
|
IngestRequestSerializer,
|
||||||
ItemSerializer,
|
ItemSerializer,
|
||||||
LibrarySerializer,
|
LibrarySerializer,
|
||||||
SearchRequestSerializer,
|
SearchRequestSerializer,
|
||||||
@@ -456,6 +458,7 @@ def search(request):
|
|||||||
library_uid=data.get("library_uid") or None,
|
library_uid=data.get("library_uid") or None,
|
||||||
library_type=data.get("library_type") or None,
|
library_type=data.get("library_type") or None,
|
||||||
collection_uid=data.get("collection_uid") or None,
|
collection_uid=data.get("collection_uid") or None,
|
||||||
|
workspace_id=data.get("workspace_id") or None,
|
||||||
search_types=data.get("search_types", ["vector", "fulltext", "graph"]),
|
search_types=data.get("search_types", ["vector", "fulltext", "graph"]),
|
||||||
limit=data.get("limit", getattr(django_settings, "SEARCH_DEFAULT_LIMIT", 20)),
|
limit=data.get("limit", getattr(django_settings, "SEARCH_DEFAULT_LIMIT", 20)),
|
||||||
vector_top_k=getattr(django_settings, "SEARCH_VECTOR_TOP_K", 50),
|
vector_top_k=getattr(django_settings, "SEARCH_VECTOR_TOP_K", 50),
|
||||||
@@ -487,6 +490,7 @@ def search_vector(request):
|
|||||||
library_uid=data.get("library_uid") or None,
|
library_uid=data.get("library_uid") or None,
|
||||||
library_type=data.get("library_type") or None,
|
library_type=data.get("library_type") or None,
|
||||||
collection_uid=data.get("collection_uid") or None,
|
collection_uid=data.get("collection_uid") or None,
|
||||||
|
workspace_id=data.get("workspace_id") or None,
|
||||||
search_types=["vector"],
|
search_types=["vector"],
|
||||||
limit=data.get("limit", 20),
|
limit=data.get("limit", 20),
|
||||||
vector_top_k=getattr(django_settings, "SEARCH_VECTOR_TOP_K", 50),
|
vector_top_k=getattr(django_settings, "SEARCH_VECTOR_TOP_K", 50),
|
||||||
@@ -517,6 +521,7 @@ def search_fulltext(request):
|
|||||||
library_uid=data.get("library_uid") or None,
|
library_uid=data.get("library_uid") or None,
|
||||||
library_type=data.get("library_type") or None,
|
library_type=data.get("library_type") or None,
|
||||||
collection_uid=data.get("collection_uid") or None,
|
collection_uid=data.get("collection_uid") or None,
|
||||||
|
workspace_id=data.get("workspace_id") or None,
|
||||||
search_types=["fulltext"],
|
search_types=["fulltext"],
|
||||||
limit=data.get("limit", 20),
|
limit=data.get("limit", 20),
|
||||||
fulltext_top_k=getattr(django_settings, "SEARCH_FULLTEXT_TOP_K", 30),
|
fulltext_top_k=getattr(django_settings, "SEARCH_FULLTEXT_TOP_K", 30),
|
||||||
@@ -635,3 +640,196 @@ def concept_graph(request, uid):
|
|||||||
{"detail": f"Failed: {exc}"},
|
{"detail": f"Failed: {exc}"},
|
||||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Ingest API (Daedalus integration)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@api_view(["POST"])
|
||||||
|
@permission_classes([IsAuthenticated])
|
||||||
|
def ingest_create(request):
|
||||||
|
"""
|
||||||
|
Accept a file (already in S3) for ingestion + embedding.
|
||||||
|
|
||||||
|
Daedalus calls this after committing a file to its own S3 bucket.
|
||||||
|
We resolve the target Library (by workspace_id or library_uid),
|
||||||
|
enforce idempotency on (library, source_ref, content_hash), create
|
||||||
|
an IngestJob row, and dispatch a Celery task.
|
||||||
|
|
||||||
|
Idempotency:
|
||||||
|
- Same source_ref + same content_hash → return existing completed job
|
||||||
|
(no new task dispatched).
|
||||||
|
- Same source_ref + different content_hash → dispatch a new task that
|
||||||
|
will supersede the prior Item.
|
||||||
|
- New source_ref → fresh ingest.
|
||||||
|
"""
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from library.models import IngestJob, Library
|
||||||
|
from library.tasks import ingest_from_daedalus
|
||||||
|
|
||||||
|
serializer = IngestRequestSerializer(data=request.data)
|
||||||
|
serializer.is_valid(raise_exception=True)
|
||||||
|
data = serializer.validated_data
|
||||||
|
|
||||||
|
# --- Resolve target Library ---
|
||||||
|
workspace_id = data.get("workspace_id") or ""
|
||||||
|
library_uid = data.get("library_uid") or ""
|
||||||
|
if workspace_id:
|
||||||
|
try:
|
||||||
|
lib = Library.nodes.get(workspace_id=workspace_id)
|
||||||
|
except Library.DoesNotExist:
|
||||||
|
return Response(
|
||||||
|
{"detail": f"Workspace '{workspace_id}' not registered."},
|
||||||
|
status=status.HTTP_404_NOT_FOUND,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
lib = Library.nodes.get(uid=library_uid)
|
||||||
|
except Library.DoesNotExist:
|
||||||
|
return Response(
|
||||||
|
{"detail": f"Library '{library_uid}' not found."},
|
||||||
|
status=status.HTTP_404_NOT_FOUND,
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Idempotency check on (library, source_ref, content_hash) ---
|
||||||
|
source_ref = data.get("source_ref") or ""
|
||||||
|
content_hash = data["content_hash"]
|
||||||
|
|
||||||
|
if source_ref:
|
||||||
|
existing = (
|
||||||
|
IngestJob.objects
|
||||||
|
.filter(
|
||||||
|
library_uid=lib.uid,
|
||||||
|
source_ref=source_ref,
|
||||||
|
content_hash=content_hash,
|
||||||
|
status__in=["pending", "processing", "completed"],
|
||||||
|
)
|
||||||
|
.order_by("-created_at")
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if existing is not None:
|
||||||
|
logger.info(
|
||||||
|
"Ingest idempotent hit job_id=%s source_ref=%s status=%s",
|
||||||
|
existing.id, source_ref, existing.status,
|
||||||
|
)
|
||||||
|
return Response(
|
||||||
|
IngestJobSerializer(existing).data,
|
||||||
|
status=status.HTTP_200_OK,
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Create job + dispatch ---
|
||||||
|
job = IngestJob.objects.create(
|
||||||
|
id=f"job_{uuid.uuid4().hex[:24]}",
|
||||||
|
library_uid=lib.uid,
|
||||||
|
s3_key=data["s3_key"],
|
||||||
|
title=data["title"],
|
||||||
|
file_type=data.get("file_type", ""),
|
||||||
|
file_size=data.get("file_size", 0),
|
||||||
|
content_hash=content_hash,
|
||||||
|
source=data.get("source", ""),
|
||||||
|
source_ref=source_ref,
|
||||||
|
collection_uid=data.get("collection_uid", ""),
|
||||||
|
status="pending",
|
||||||
|
progress="queued",
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
async_result = ingest_from_daedalus.delay(job.id)
|
||||||
|
job.celery_task_id = async_result.id
|
||||||
|
job.save(update_fields=["celery_task_id"])
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error("Failed to dispatch ingest task job_id=%s: %s", job.id, exc)
|
||||||
|
job.status = "failed"
|
||||||
|
job.error = f"dispatch failed: {exc}"
|
||||||
|
job.save(update_fields=["status", "error"])
|
||||||
|
return Response(
|
||||||
|
IngestJobSerializer(job).data,
|
||||||
|
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Ingest dispatched job_id=%s library_uid=%s source_ref=%s task_id=%s",
|
||||||
|
job.id, lib.uid, source_ref, job.celery_task_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
return Response(
|
||||||
|
IngestJobSerializer(job).data,
|
||||||
|
status=status.HTTP_202_ACCEPTED,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@api_view(["GET"])
|
||||||
|
@permission_classes([IsAuthenticated])
|
||||||
|
def ingest_job_detail(request, job_id):
|
||||||
|
"""Get the current status of an IngestJob."""
|
||||||
|
from library.models import IngestJob
|
||||||
|
|
||||||
|
try:
|
||||||
|
job = IngestJob.objects.get(pk=job_id)
|
||||||
|
except IngestJob.DoesNotExist:
|
||||||
|
return Response(
|
||||||
|
{"detail": "Job not found."}, status=status.HTTP_404_NOT_FOUND
|
||||||
|
)
|
||||||
|
|
||||||
|
return Response(IngestJobSerializer(job).data)
|
||||||
|
|
||||||
|
|
||||||
|
@api_view(["POST"])
|
||||||
|
@permission_classes([IsAuthenticated])
|
||||||
|
def ingest_job_retry(request, job_id):
|
||||||
|
"""Re-dispatch a failed IngestJob."""
|
||||||
|
from library.models import IngestJob
|
||||||
|
from library.tasks import ingest_from_daedalus
|
||||||
|
|
||||||
|
try:
|
||||||
|
job = IngestJob.objects.get(pk=job_id)
|
||||||
|
except IngestJob.DoesNotExist:
|
||||||
|
return Response(
|
||||||
|
{"detail": "Job not found."}, status=status.HTTP_404_NOT_FOUND
|
||||||
|
)
|
||||||
|
|
||||||
|
if job.status not in ("failed", "completed"):
|
||||||
|
return Response(
|
||||||
|
{"detail": f"Job is currently {job.status}; cannot retry."},
|
||||||
|
status=status.HTTP_409_CONFLICT,
|
||||||
|
)
|
||||||
|
|
||||||
|
job.status = "pending"
|
||||||
|
job.progress = "queued"
|
||||||
|
job.error = ""
|
||||||
|
job.save(update_fields=["status", "progress", "error"])
|
||||||
|
|
||||||
|
async_result = ingest_from_daedalus.delay(job.id)
|
||||||
|
job.celery_task_id = async_result.id
|
||||||
|
job.save(update_fields=["celery_task_id"])
|
||||||
|
|
||||||
|
logger.info("Ingest retry dispatched job_id=%s task_id=%s", job.id, async_result.id)
|
||||||
|
return Response(IngestJobSerializer(job).data, status=status.HTTP_202_ACCEPTED)
|
||||||
|
|
||||||
|
|
||||||
|
@api_view(["GET"])
|
||||||
|
@permission_classes([IsAuthenticated])
|
||||||
|
def ingest_job_list(request):
|
||||||
|
"""List recent IngestJob rows, optionally filtered by status / library_uid."""
|
||||||
|
from library.models import IngestJob
|
||||||
|
|
||||||
|
qs = IngestJob.objects.all()
|
||||||
|
status_filter = request.query_params.get("status")
|
||||||
|
library_uid = request.query_params.get("library_uid")
|
||||||
|
limit = min(int(request.query_params.get("limit", 50)), 200)
|
||||||
|
|
||||||
|
if status_filter:
|
||||||
|
qs = qs.filter(status=status_filter)
|
||||||
|
if library_uid:
|
||||||
|
qs = qs.filter(library_uid=library_uid)
|
||||||
|
|
||||||
|
jobs = list(qs.order_by("-created_at")[:limit])
|
||||||
|
return Response(
|
||||||
|
{
|
||||||
|
"jobs": IngestJobSerializer(jobs, many=True).data,
|
||||||
|
"count": len(jobs),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|||||||
217
mnemosyne/library/api/workspaces.py
Normal file
217
mnemosyne/library/api/workspaces.py
Normal file
@@ -0,0 +1,217 @@
|
|||||||
|
"""
|
||||||
|
Workspace lifecycle endpoints for the Daedalus integration.
|
||||||
|
|
||||||
|
A "workspace" in Mnemosyne is a Library scoped to a Daedalus workspace UUID.
|
||||||
|
It uses the same Library node as a global library; the difference is that
|
||||||
|
`workspace_id` is set, and search must filter on it.
|
||||||
|
|
||||||
|
These endpoints are called by the Daedalus backend (HTTP Basic auth as
|
||||||
|
the `daedalus-service` user). Daedalus owns the workspace_id; Mnemosyne
|
||||||
|
just persists what Daedalus tells it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from neomodel import db
|
||||||
|
from rest_framework import status
|
||||||
|
from rest_framework.decorators import api_view, permission_classes
|
||||||
|
from rest_framework.permissions import IsAuthenticated
|
||||||
|
from rest_framework.response import Response
|
||||||
|
|
||||||
|
from library.content_types import get_library_type_config
|
||||||
|
|
||||||
|
from .serializers import WorkspaceCreateSerializer, WorkspaceStatusSerializer
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_workspace(lib):
|
||||||
|
"""Build a WorkspaceStatus payload from a Library node + chunk/item counts."""
|
||||||
|
counts, _ = db.cypher_query(
|
||||||
|
"MATCH (l:Library {workspace_id: $wsid}) "
|
||||||
|
"OPTIONAL MATCH (l)-[:CONTAINS]->(:Collection)-[:CONTAINS]->(i:Item) "
|
||||||
|
"OPTIONAL MATCH (i)-[:HAS_CHUNK]->(c:Chunk) "
|
||||||
|
"RETURN count(DISTINCT i) AS item_count, count(DISTINCT c) AS chunk_count",
|
||||||
|
{"wsid": lib.workspace_id},
|
||||||
|
)
|
||||||
|
item_count = counts[0][0] if counts else 0
|
||||||
|
chunk_count = counts[0][1] if counts else 0
|
||||||
|
|
||||||
|
return {
|
||||||
|
"workspace_id": lib.workspace_id,
|
||||||
|
"library_uid": lib.uid,
|
||||||
|
"name": lib.name,
|
||||||
|
"library_type": lib.library_type,
|
||||||
|
"description": lib.description or "",
|
||||||
|
"item_count": item_count,
|
||||||
|
"chunk_count": chunk_count,
|
||||||
|
"created_at": lib.created_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@api_view(["POST"])
|
||||||
|
@permission_classes([IsAuthenticated])
|
||||||
|
def workspace_create(request):
|
||||||
|
"""
|
||||||
|
Create a workspace Library, idempotently.
|
||||||
|
|
||||||
|
A POST with a `workspace_id` already in use returns the existing
|
||||||
|
workspace (200) — not an error. The library_type is frozen at first
|
||||||
|
create; subsequent calls are not allowed to change it.
|
||||||
|
"""
|
||||||
|
from library.models import Library
|
||||||
|
|
||||||
|
serializer = WorkspaceCreateSerializer(data=request.data)
|
||||||
|
serializer.is_valid(raise_exception=True)
|
||||||
|
data = serializer.validated_data
|
||||||
|
|
||||||
|
# Idempotent path: workspace already exists.
|
||||||
|
try:
|
||||||
|
existing = Library.nodes.get(workspace_id=data["workspace_id"])
|
||||||
|
except Library.DoesNotExist:
|
||||||
|
existing = None
|
||||||
|
|
||||||
|
if existing is not None:
|
||||||
|
if existing.library_type != data["library_type"]:
|
||||||
|
return Response(
|
||||||
|
{
|
||||||
|
"detail": (
|
||||||
|
"library_type is immutable for an existing workspace "
|
||||||
|
f"(have '{existing.library_type}', "
|
||||||
|
f"got '{data['library_type']}')."
|
||||||
|
)
|
||||||
|
},
|
||||||
|
status=status.HTTP_409_CONFLICT,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"Workspace already exists workspace_id=%s library_uid=%s",
|
||||||
|
data["workspace_id"], existing.uid,
|
||||||
|
)
|
||||||
|
return Response(
|
||||||
|
WorkspaceStatusSerializer(_serialize_workspace(existing)).data,
|
||||||
|
status=status.HTTP_200_OK,
|
||||||
|
)
|
||||||
|
|
||||||
|
defaults = get_library_type_config(data["library_type"])
|
||||||
|
lib = Library(
|
||||||
|
name=data["name"],
|
||||||
|
library_type=data["library_type"],
|
||||||
|
description=data.get("description", ""),
|
||||||
|
workspace_id=data["workspace_id"],
|
||||||
|
chunking_config=defaults["chunking_config"],
|
||||||
|
embedding_instruction=defaults["embedding_instruction"],
|
||||||
|
reranker_instruction=defaults["reranker_instruction"],
|
||||||
|
llm_context_prompt=defaults["llm_context_prompt"],
|
||||||
|
)
|
||||||
|
lib.save()
|
||||||
|
logger.info(
|
||||||
|
"Workspace created workspace_id=%s library_uid=%s library_type=%s",
|
||||||
|
data["workspace_id"], lib.uid, lib.library_type,
|
||||||
|
)
|
||||||
|
|
||||||
|
return Response(
|
||||||
|
WorkspaceStatusSerializer(_serialize_workspace(lib)).data,
|
||||||
|
status=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@api_view(["GET", "DELETE"])
|
||||||
|
@permission_classes([IsAuthenticated])
|
||||||
|
def workspace_detail_or_delete(request, workspace_id):
|
||||||
|
"""
|
||||||
|
GET: return workspace status (item/chunk counts, metadata).
|
||||||
|
|
||||||
|
DELETE: delete the workspace Library and everything reachable AND unique
|
||||||
|
to it. Concept-safe: orphan-only Concept GC happens at the end.
|
||||||
|
Concepts referenced by other libraries (workspace or global) are preserved.
|
||||||
|
"""
|
||||||
|
from library.models import Library
|
||||||
|
|
||||||
|
if request.method == "GET":
|
||||||
|
try:
|
||||||
|
lib = Library.nodes.get(workspace_id=workspace_id)
|
||||||
|
except Library.DoesNotExist:
|
||||||
|
return Response(
|
||||||
|
{"detail": "Workspace not found."},
|
||||||
|
status=status.HTTP_404_NOT_FOUND,
|
||||||
|
)
|
||||||
|
|
||||||
|
return Response(WorkspaceStatusSerializer(_serialize_workspace(lib)).data)
|
||||||
|
|
||||||
|
# DELETE — idempotent: a missing workspace returns 204.
|
||||||
|
try:
|
||||||
|
lib = Library.nodes.get(workspace_id=workspace_id)
|
||||||
|
except Library.DoesNotExist:
|
||||||
|
return Response(status=status.HTTP_204_NO_CONTENT)
|
||||||
|
|
||||||
|
library_uid = lib.uid
|
||||||
|
library_name = lib.name
|
||||||
|
|
||||||
|
# Step 1-4: delete chunks, items, collections, then the library itself.
|
||||||
|
# We collect Item s3_keys first so the caller can clean up S3
|
||||||
|
# asynchronously (a future enhancement — for now, the keys are logged).
|
||||||
|
s3_rows, _ = db.cypher_query(
|
||||||
|
"MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(:Collection)"
|
||||||
|
"-[:CONTAINS]->(i:Item) RETURN i.uid, i.s3_key",
|
||||||
|
{"wsid": workspace_id},
|
||||||
|
)
|
||||||
|
item_s3_keys = [(r[0], r[1]) for r in s3_rows if r[1]]
|
||||||
|
|
||||||
|
db.cypher_query(
|
||||||
|
"""
|
||||||
|
MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(:Collection)
|
||||||
|
-[:CONTAINS]->(i:Item)-[:HAS_CHUNK]->(c:Chunk)
|
||||||
|
DETACH DELETE c
|
||||||
|
""",
|
||||||
|
{"wsid": workspace_id},
|
||||||
|
)
|
||||||
|
db.cypher_query(
|
||||||
|
"""
|
||||||
|
MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(:Collection)
|
||||||
|
-[:CONTAINS]->(i:Item)-[:HAS_IMAGE]->(img:Image)
|
||||||
|
OPTIONAL MATCH (img)-[:HAS_EMBEDDING]->(emb:ImageEmbedding)
|
||||||
|
DETACH DELETE img, emb
|
||||||
|
""",
|
||||||
|
{"wsid": workspace_id},
|
||||||
|
)
|
||||||
|
db.cypher_query(
|
||||||
|
"""
|
||||||
|
MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(:Collection)
|
||||||
|
-[:CONTAINS]->(i:Item)
|
||||||
|
DETACH DELETE i
|
||||||
|
""",
|
||||||
|
{"wsid": workspace_id},
|
||||||
|
)
|
||||||
|
db.cypher_query(
|
||||||
|
"""
|
||||||
|
MATCH (l:Library {workspace_id: $wsid})-[:CONTAINS]->(col:Collection)
|
||||||
|
DETACH DELETE col
|
||||||
|
""",
|
||||||
|
{"wsid": workspace_id},
|
||||||
|
)
|
||||||
|
db.cypher_query(
|
||||||
|
"MATCH (l:Library {workspace_id: $wsid}) DETACH DELETE l",
|
||||||
|
{"wsid": workspace_id},
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 5: orphan Concept garbage collection.
|
||||||
|
orphan_result, _ = db.cypher_query(
|
||||||
|
"""
|
||||||
|
MATCH (con:Concept)
|
||||||
|
WHERE NOT (con)<-[:REFERENCES]-() AND NOT (con)<-[:MENTIONS]-()
|
||||||
|
AND NOT (con)<-[:DEPICTS]-()
|
||||||
|
WITH con
|
||||||
|
DETACH DELETE con
|
||||||
|
RETURN count(con) AS deleted
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
orphans_deleted = orphan_result[0][0] if orphan_result else 0
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Workspace deleted workspace_id=%s library_uid=%s name=%s "
|
||||||
|
"items=%d orphans_deleted=%d",
|
||||||
|
workspace_id, library_uid, library_name,
|
||||||
|
len(item_s3_keys), orphans_deleted,
|
||||||
|
)
|
||||||
|
|
||||||
|
return Response(status=status.HTTP_204_NO_CONTENT)
|
||||||
@@ -210,6 +210,69 @@ LIBRARY_TYPE_DEFAULTS = {
|
|||||||
"4) Context clues about when and where this was taken or created."
|
"4) Context clues about when and where this was taken or created."
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
|
"business": {
|
||||||
|
"chunking_config": {
|
||||||
|
"strategy": "section_aware",
|
||||||
|
"chunk_size": 640,
|
||||||
|
"chunk_overlap": 96,
|
||||||
|
"respect_boundaries": ["section", "subsection", "list", "table"],
|
||||||
|
},
|
||||||
|
"embedding_instruction": (
|
||||||
|
"Represent this passage from a business document for retrieval. "
|
||||||
|
"Focus on value propositions, positioning, pricing, scope of work, "
|
||||||
|
"client outcomes, and commercial commitments."
|
||||||
|
),
|
||||||
|
"reranker_instruction": (
|
||||||
|
"Re-rank passages from business documents based on commercial relevance. "
|
||||||
|
"Prioritize value framing, deliverables, client outcomes, and specific "
|
||||||
|
"pricing or scope language."
|
||||||
|
),
|
||||||
|
"llm_context_prompt": (
|
||||||
|
"The following excerpts are from business documents (proposals, marketing, "
|
||||||
|
"sales, strategy). Interpret in commercial context. Distinguish positioning "
|
||||||
|
"claims from committed deliverables. Preserve numbers, scope language, and "
|
||||||
|
"client names exactly as written."
|
||||||
|
),
|
||||||
|
"vision_prompt": (
|
||||||
|
"Analyze this image from a business document. Identify:\n"
|
||||||
|
"1) Image type (logo, chart, diagram, screenshot, photograph, table).\n"
|
||||||
|
"2) What it depicts — brand marks, data, organizational structure, products.\n"
|
||||||
|
"3) Any visible text — company names, figures, captions, headings.\n"
|
||||||
|
"4) The commercial purpose — positioning, pricing, capability demonstration."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"finance": {
|
||||||
|
"chunking_config": {
|
||||||
|
"strategy": "section_aware",
|
||||||
|
"chunk_size": 512,
|
||||||
|
"chunk_overlap": 64,
|
||||||
|
"respect_boundaries": ["section", "table", "row", "paragraph"],
|
||||||
|
},
|
||||||
|
"embedding_instruction": (
|
||||||
|
"Represent this passage from a financial document for retrieval. "
|
||||||
|
"Focus on accounts, instruments, dates, amounts, balances, and "
|
||||||
|
"analytical commentary."
|
||||||
|
),
|
||||||
|
"reranker_instruction": (
|
||||||
|
"Re-rank passages from financial documents based on relevance to the query. "
|
||||||
|
"Prioritize the matching account, instrument, time period, and figures."
|
||||||
|
),
|
||||||
|
"llm_context_prompt": (
|
||||||
|
"The following excerpts are from financial documents (statements, tax "
|
||||||
|
"documents, market commentary, planning). Distinguish factual figures "
|
||||||
|
"(statements, transactions, balances) from opinion (forecasts, commentary). "
|
||||||
|
"Quote numbers, dates, and account identifiers exactly as they appear. "
|
||||||
|
"Do not infer, round, or fabricate financial figures. If a figure is not "
|
||||||
|
"present in the excerpts, say so explicitly."
|
||||||
|
),
|
||||||
|
"vision_prompt": (
|
||||||
|
"Analyze this image from a financial document. Identify:\n"
|
||||||
|
"1) Image type (chart, table, statement scan, dashboard screenshot, receipt).\n"
|
||||||
|
"2) What it depicts — account, instrument, time period, data series.\n"
|
||||||
|
"3) Any visible text — figures, dates, account identifiers, labels.\n"
|
||||||
|
"4) Whether the data is factual (statement) or analytical (forecast/commentary)."
|
||||||
|
),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -219,7 +282,7 @@ def get_library_type_config(library_type):
|
|||||||
|
|
||||||
Args:
|
Args:
|
||||||
library_type: One of 'fiction', 'nonfiction', 'technical', 'music',
|
library_type: One of 'fiction', 'nonfiction', 'technical', 'music',
|
||||||
'film', 'art', 'journal'
|
'film', 'art', 'journal', 'business', 'finance'
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
dict with keys: chunking_config, embedding_instruction,
|
dict with keys: chunking_config, embedding_instruction,
|
||||||
|
|||||||
45
mnemosyne/library/migrations/0001_initial.py
Normal file
45
mnemosyne/library/migrations/0001_initial.py
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
# Generated by Django 5.2.13 on 2026-04-28 12:36
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
initial = True
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name='IngestJob',
|
||||||
|
fields=[
|
||||||
|
('id', models.CharField(max_length=64, primary_key=True, serialize=False)),
|
||||||
|
('item_uid', models.CharField(blank=True, db_index=True, max_length=64)),
|
||||||
|
('library_uid', models.CharField(db_index=True, max_length=64)),
|
||||||
|
('celery_task_id', models.CharField(blank=True, max_length=255)),
|
||||||
|
('status', models.CharField(choices=[('pending', 'Pending'), ('processing', 'Processing'), ('completed', 'Completed'), ('failed', 'Failed')], db_index=True, default='pending', max_length=20)),
|
||||||
|
('progress', models.CharField(default='queued', max_length=50)),
|
||||||
|
('error', models.TextField(blank=True, null=True)),
|
||||||
|
('retry_count', models.PositiveIntegerField(default=0)),
|
||||||
|
('chunks_created', models.PositiveIntegerField(default=0)),
|
||||||
|
('concepts_extracted', models.PositiveIntegerField(default=0)),
|
||||||
|
('embedding_model', models.CharField(blank=True, max_length=100)),
|
||||||
|
('content_hash', models.CharField(blank=True, db_index=True, max_length=64)),
|
||||||
|
('source', models.CharField(default='', max_length=50)),
|
||||||
|
('source_ref', models.CharField(blank=True, db_index=True, max_length=200)),
|
||||||
|
('s3_key', models.CharField(max_length=500)),
|
||||||
|
('title', models.CharField(blank=True, max_length=500)),
|
||||||
|
('file_type', models.CharField(blank=True, max_length=50)),
|
||||||
|
('file_size', models.PositiveBigIntegerField(default=0)),
|
||||||
|
('collection_uid', models.CharField(blank=True, max_length=64)),
|
||||||
|
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||||
|
('started_at', models.DateTimeField(blank=True, null=True)),
|
||||||
|
('completed_at', models.DateTimeField(blank=True, null=True)),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
'ordering': ['-created_at'],
|
||||||
|
'indexes': [models.Index(fields=['status', '-created_at'], name='library_ing_status_9c95b2_idx'), models.Index(fields=['source', 'source_ref'], name='library_ing_source_a48684_idx')],
|
||||||
|
},
|
||||||
|
),
|
||||||
|
]
|
||||||
@@ -1,11 +1,16 @@
|
|||||||
"""
|
"""
|
||||||
Neo4j graph models for the Mnemosyne content library.
|
Models for the Mnemosyne content library.
|
||||||
|
|
||||||
All content data (libraries, collections, items, chunks, concepts, images)
|
Most content (libraries, collections, items, chunks, concepts, images)
|
||||||
lives in Neo4j as a knowledge graph. These models use neomodel's StructuredNode
|
lives in Neo4j as a knowledge graph via neomodel StructuredNode. These do
|
||||||
OGM — they do NOT participate in Django's ORM or migrations.
|
NOT participate in Django's ORM or migrations.
|
||||||
|
|
||||||
|
The IngestJob model at the bottom of this file is the exception: it tracks
|
||||||
|
the lifecycle of asynchronous ingestion requests (file → embedding pipeline)
|
||||||
|
in PostgreSQL via Django's ORM.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from django.db import models
|
||||||
from neomodel import (
|
from neomodel import (
|
||||||
ArrayProperty,
|
ArrayProperty,
|
||||||
DateTimeProperty,
|
DateTimeProperty,
|
||||||
@@ -50,8 +55,14 @@ class Library(StructuredNode):
|
|||||||
"""
|
"""
|
||||||
Top-level container representing a content library.
|
Top-level container representing a content library.
|
||||||
|
|
||||||
Each library has a type (fiction, technical, music, film, art, journal)
|
Each library has a type (fiction, nonfiction, technical, music, film,
|
||||||
that drives chunking strategy, embedding instructions, and LLM prompts.
|
art, journal, business, finance) that drives chunking strategy,
|
||||||
|
embedding instructions, and LLM prompts.
|
||||||
|
|
||||||
|
A library may be either *global* (workspace_id is null — searchable
|
||||||
|
across the whole instance) or *workspace-scoped* (workspace_id set —
|
||||||
|
visible only to agents inside that Daedalus workspace). Scoping is
|
||||||
|
enforced structurally by every search query.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
uid = UniqueIdProperty()
|
uid = UniqueIdProperty()
|
||||||
@@ -66,10 +77,16 @@ class Library(StructuredNode):
|
|||||||
"film": "Film",
|
"film": "Film",
|
||||||
"art": "Art",
|
"art": "Art",
|
||||||
"journal": "Journal",
|
"journal": "Journal",
|
||||||
|
"business": "Business",
|
||||||
|
"finance": "Finance",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
description = StringProperty(default="")
|
description = StringProperty(default="")
|
||||||
|
|
||||||
|
# Daedalus workspace UUID this library is scoped to. Null for global
|
||||||
|
# libraries. Unique-indexed so a workspace cannot have two libraries.
|
||||||
|
workspace_id = StringProperty(unique_index=True, required=False)
|
||||||
|
|
||||||
# Content-type configuration
|
# Content-type configuration
|
||||||
chunking_config = JSONProperty(default={})
|
chunking_config = JSONProperty(default={})
|
||||||
embedding_instruction = StringProperty(default="")
|
embedding_instruction = StringProperty(default="")
|
||||||
@@ -270,3 +287,78 @@ class ImageEmbedding(StructuredNode):
|
|||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return f"ImageEmbedding ({self.uid})"
|
return f"ImageEmbedding ({self.uid})"
|
||||||
|
|
||||||
|
|
||||||
|
# --- Django ORM models (PostgreSQL) ---
|
||||||
|
|
||||||
|
|
||||||
|
class IngestJob(models.Model):
|
||||||
|
"""
|
||||||
|
Tracks the lifecycle of an asynchronous ingestion + embedding job.
|
||||||
|
|
||||||
|
Created when an external client (e.g. Daedalus) posts a file via the
|
||||||
|
REST ingest API. The Celery worker reads and updates this row as the
|
||||||
|
job moves through fetch / chunk / embed / graph stages.
|
||||||
|
|
||||||
|
Idempotency: a (library, source_ref, content_hash) triple uniquely
|
||||||
|
identifies a piece of content. A second POST with the same triple
|
||||||
|
returns the existing job; a POST with the same source_ref but a new
|
||||||
|
content_hash supersedes the prior Item.
|
||||||
|
"""
|
||||||
|
|
||||||
|
STATUS_CHOICES = [
|
||||||
|
("pending", "Pending"),
|
||||||
|
("processing", "Processing"),
|
||||||
|
("completed", "Completed"),
|
||||||
|
("failed", "Failed"),
|
||||||
|
]
|
||||||
|
|
||||||
|
id = models.CharField(max_length=64, primary_key=True)
|
||||||
|
item_uid = models.CharField(max_length=64, db_index=True, blank=True)
|
||||||
|
library_uid = models.CharField(max_length=64, db_index=True)
|
||||||
|
celery_task_id = models.CharField(max_length=255, blank=True)
|
||||||
|
|
||||||
|
status = models.CharField(
|
||||||
|
max_length=20,
|
||||||
|
choices=STATUS_CHOICES,
|
||||||
|
default="pending",
|
||||||
|
db_index=True,
|
||||||
|
)
|
||||||
|
progress = models.CharField(max_length=50, default="queued")
|
||||||
|
error = models.TextField(blank=True, null=True)
|
||||||
|
retry_count = models.PositiveIntegerField(default=0)
|
||||||
|
|
||||||
|
chunks_created = models.PositiveIntegerField(default=0)
|
||||||
|
concepts_extracted = models.PositiveIntegerField(default=0)
|
||||||
|
embedding_model = models.CharField(max_length=100, blank=True)
|
||||||
|
|
||||||
|
# The file's content hash (sha256). Used for idempotency: a second
|
||||||
|
# ingest with the same source_ref + same hash is a no-op; a second
|
||||||
|
# ingest with the same source_ref + different hash supersedes.
|
||||||
|
content_hash = models.CharField(max_length=64, db_index=True, blank=True)
|
||||||
|
|
||||||
|
# Where the file came from. For Daedalus: source="daedalus",
|
||||||
|
# source_ref="<workspace_id>/<file_id>".
|
||||||
|
source = models.CharField(max_length=50, default="")
|
||||||
|
source_ref = models.CharField(max_length=200, blank=True, db_index=True)
|
||||||
|
s3_key = models.CharField(max_length=500)
|
||||||
|
|
||||||
|
# Optional metadata carried forward to the Item node.
|
||||||
|
title = models.CharField(max_length=500, blank=True)
|
||||||
|
file_type = models.CharField(max_length=50, blank=True)
|
||||||
|
file_size = models.PositiveBigIntegerField(default=0)
|
||||||
|
collection_uid = models.CharField(max_length=64, blank=True)
|
||||||
|
|
||||||
|
created_at = models.DateTimeField(auto_now_add=True)
|
||||||
|
started_at = models.DateTimeField(null=True, blank=True)
|
||||||
|
completed_at = models.DateTimeField(null=True, blank=True)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
ordering = ["-created_at"]
|
||||||
|
indexes = [
|
||||||
|
models.Index(fields=["status", "-created_at"]),
|
||||||
|
models.Index(fields=["source", "source_ref"]),
|
||||||
|
]
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return f"IngestJob {self.id} [{self.status}]"
|
||||||
|
|||||||
70
mnemosyne/library/services/daedalus_s3.py
Normal file
70
mnemosyne/library/services/daedalus_s3.py
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
"""
|
||||||
|
Cross-bucket S3 helper for ingesting files from the Daedalus S3 bucket
|
||||||
|
into Mnemosyne's own bucket.
|
||||||
|
|
||||||
|
Daedalus uploads files to its own bucket (configured per Daedalus deployment)
|
||||||
|
and posts an ingest request to Mnemosyne with the s3_key. This module fetches
|
||||||
|
that file using read-only Daedalus credentials and writes it to Mnemosyne's
|
||||||
|
bucket via the standard `default_storage` backend so the rest of the pipeline
|
||||||
|
(parsing, chunking, embedding) works unchanged.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import boto3
|
||||||
|
from django.conf import settings
|
||||||
|
from django.core.files.base import ContentFile
|
||||||
|
from django.core.files.storage import default_storage
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _daedalus_s3_client():
|
||||||
|
"""Build a boto3 S3 client pointed at the Daedalus bucket."""
|
||||||
|
return boto3.client(
|
||||||
|
"s3",
|
||||||
|
endpoint_url=settings.DAEDALUS_S3_ENDPOINT_URL or None,
|
||||||
|
aws_access_key_id=settings.DAEDALUS_S3_ACCESS_KEY_ID,
|
||||||
|
aws_secret_access_key=settings.DAEDALUS_S3_SECRET_ACCESS_KEY,
|
||||||
|
region_name=settings.DAEDALUS_S3_REGION_NAME,
|
||||||
|
use_ssl=settings.DAEDALUS_S3_USE_SSL,
|
||||||
|
verify=settings.DAEDALUS_S3_VERIFY,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_from_daedalus(daedalus_s3_key: str) -> bytes:
|
||||||
|
"""
|
||||||
|
Read a file from the Daedalus S3 bucket.
|
||||||
|
|
||||||
|
:param daedalus_s3_key: Object key in the Daedalus bucket.
|
||||||
|
:returns: File bytes.
|
||||||
|
:raises: botocore exceptions on failure (caller decides retry).
|
||||||
|
"""
|
||||||
|
client = _daedalus_s3_client()
|
||||||
|
bucket = settings.DAEDALUS_S3_BUCKET_NAME
|
||||||
|
logger.debug(
|
||||||
|
"Fetching from Daedalus S3 bucket=%s key=%s", bucket, daedalus_s3_key
|
||||||
|
)
|
||||||
|
response = client.get_object(Bucket=bucket, Key=daedalus_s3_key)
|
||||||
|
data = response["Body"].read()
|
||||||
|
logger.info(
|
||||||
|
"Fetched from Daedalus S3 bucket=%s key=%s size=%d",
|
||||||
|
bucket, daedalus_s3_key, len(data),
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def copy_into_mnemosyne(data: bytes, mnemosyne_s3_key: str) -> str:
|
||||||
|
"""
|
||||||
|
Write bytes into Mnemosyne's S3 bucket via the default_storage backend.
|
||||||
|
|
||||||
|
:param data: File bytes (already in memory).
|
||||||
|
:param mnemosyne_s3_key: Target object key in Mnemosyne's bucket.
|
||||||
|
:returns: The actual key written (may differ from requested if
|
||||||
|
`file_overwrite=False` and the key existed).
|
||||||
|
"""
|
||||||
|
saved_key = default_storage.save(mnemosyne_s3_key, ContentFile(data))
|
||||||
|
logger.info(
|
||||||
|
"Wrote to Mnemosyne S3 key=%s size=%d", saved_key, len(data),
|
||||||
|
)
|
||||||
|
return saved_key
|
||||||
@@ -26,15 +26,32 @@ from .fusion import ImageSearchResult, SearchCandidate, reciprocal_rank_fusion
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# Workspace scoping clause appended to every search Cypher query.
|
||||||
|
#
|
||||||
|
# A request with workspace_id set returns ONLY that workspace's content.
|
||||||
|
# A request with workspace_id null returns ONLY global content (libraries
|
||||||
|
# with no workspace_id). There is no third mode.
|
||||||
|
_WORKSPACE_SCOPE_CLAUSE = (
|
||||||
|
" AND ($workspace_id IS NULL AND lib.workspace_id IS NULL OR "
|
||||||
|
"lib.workspace_id = $workspace_id)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class SearchRequest:
|
class SearchRequest:
|
||||||
"""Parameters for a search query."""
|
"""Parameters for a search query.
|
||||||
|
|
||||||
|
Scope is single-mode: a request is either workspace-scoped (workspace_id
|
||||||
|
set) or global (workspace_id is None). There is no parameter combination
|
||||||
|
that returns both workspace and global content in one call.
|
||||||
|
"""
|
||||||
|
|
||||||
query: str
|
query: str
|
||||||
query_image: Optional[bytes] = None
|
query_image: Optional[bytes] = None
|
||||||
library_uid: Optional[str] = None
|
library_uid: Optional[str] = None
|
||||||
library_type: Optional[str] = None
|
library_type: Optional[str] = None
|
||||||
collection_uid: Optional[str] = None
|
collection_uid: Optional[str] = None
|
||||||
|
workspace_id: Optional[str] = None
|
||||||
search_types: list[str] = field(
|
search_types: list[str] = field(
|
||||||
default_factory=lambda: ["vector", "fulltext", "graph"]
|
default_factory=lambda: ["vector", "fulltext", "graph"]
|
||||||
)
|
)
|
||||||
@@ -45,6 +62,18 @@ class SearchRequest:
|
|||||||
rerank: bool = True
|
rerank: bool = True
|
||||||
include_images: bool = True
|
include_images: bool = True
|
||||||
|
|
||||||
|
def __post_init__(self):
|
||||||
|
# Normalize empty strings to None so "" doesn't slip through as
|
||||||
|
# truthy at the Cypher boundary.
|
||||||
|
if self.workspace_id == "":
|
||||||
|
self.workspace_id = None
|
||||||
|
if self.library_uid == "":
|
||||||
|
self.library_uid = None
|
||||||
|
if self.library_type == "":
|
||||||
|
self.library_type = None
|
||||||
|
if self.collection_uid == "":
|
||||||
|
self.collection_uid = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class SearchResponse:
|
class SearchResponse:
|
||||||
@@ -243,7 +272,8 @@ class SearchService:
|
|||||||
top_k = request.vector_top_k
|
top_k = request.vector_top_k
|
||||||
|
|
||||||
# Build Cypher with optional filtering
|
# Build Cypher with optional filtering
|
||||||
cypher = """
|
cypher = (
|
||||||
|
"""
|
||||||
CALL db.index.vector.queryNodes('chunk_embedding_index', $top_k, $query_vector)
|
CALL db.index.vector.queryNodes('chunk_embedding_index', $top_k, $query_vector)
|
||||||
YIELD node AS chunk, score
|
YIELD node AS chunk, score
|
||||||
MATCH (item:Item)-[:HAS_CHUNK]->(chunk)
|
MATCH (item:Item)-[:HAS_CHUNK]->(chunk)
|
||||||
@@ -251,6 +281,9 @@ class SearchService:
|
|||||||
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
||||||
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
||||||
AND ($collection_uid IS NULL OR col.uid = $collection_uid)
|
AND ($collection_uid IS NULL OR col.uid = $collection_uid)
|
||||||
|
"""
|
||||||
|
+ _WORKSPACE_SCOPE_CLAUSE
|
||||||
|
+ """
|
||||||
RETURN chunk.uid AS chunk_uid, chunk.text_preview AS text_preview,
|
RETURN chunk.uid AS chunk_uid, chunk.text_preview AS text_preview,
|
||||||
chunk.chunk_s3_key AS chunk_s3_key, chunk.chunk_index AS chunk_index,
|
chunk.chunk_s3_key AS chunk_s3_key, chunk.chunk_index AS chunk_index,
|
||||||
item.uid AS item_uid, item.title AS item_title,
|
item.uid AS item_uid, item.title AS item_title,
|
||||||
@@ -258,6 +291,7 @@ class SearchService:
|
|||||||
ORDER BY score DESC
|
ORDER BY score DESC
|
||||||
LIMIT $top_k
|
LIMIT $top_k
|
||||||
"""
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
params = {
|
params = {
|
||||||
"top_k": top_k,
|
"top_k": top_k,
|
||||||
@@ -265,6 +299,7 @@ class SearchService:
|
|||||||
"library_uid": request.library_uid,
|
"library_uid": request.library_uid,
|
||||||
"library_type": request.library_type,
|
"library_type": request.library_type,
|
||||||
"collection_uid": request.collection_uid,
|
"collection_uid": request.collection_uid,
|
||||||
|
"workspace_id": request.workspace_id,
|
||||||
}
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -348,7 +383,8 @@ class SearchService:
|
|||||||
candidates: dict[str, SearchCandidate],
|
candidates: dict[str, SearchCandidate],
|
||||||
):
|
):
|
||||||
"""Search chunk_text_fulltext index and add to candidates dict."""
|
"""Search chunk_text_fulltext index and add to candidates dict."""
|
||||||
cypher = """
|
cypher = (
|
||||||
|
"""
|
||||||
CALL db.index.fulltext.queryNodes('chunk_text_fulltext', $query)
|
CALL db.index.fulltext.queryNodes('chunk_text_fulltext', $query)
|
||||||
YIELD node AS chunk, score
|
YIELD node AS chunk, score
|
||||||
MATCH (item:Item)-[:HAS_CHUNK]->(chunk)
|
MATCH (item:Item)-[:HAS_CHUNK]->(chunk)
|
||||||
@@ -356,6 +392,9 @@ class SearchService:
|
|||||||
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
||||||
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
||||||
AND ($collection_uid IS NULL OR col.uid = $collection_uid)
|
AND ($collection_uid IS NULL OR col.uid = $collection_uid)
|
||||||
|
"""
|
||||||
|
+ _WORKSPACE_SCOPE_CLAUSE
|
||||||
|
+ """
|
||||||
RETURN chunk.uid AS chunk_uid, chunk.text_preview AS text_preview,
|
RETURN chunk.uid AS chunk_uid, chunk.text_preview AS text_preview,
|
||||||
chunk.chunk_s3_key AS chunk_s3_key, chunk.chunk_index AS chunk_index,
|
chunk.chunk_s3_key AS chunk_s3_key, chunk.chunk_index AS chunk_index,
|
||||||
item.uid AS item_uid, item.title AS item_title,
|
item.uid AS item_uid, item.title AS item_title,
|
||||||
@@ -363,6 +402,7 @@ class SearchService:
|
|||||||
ORDER BY score DESC
|
ORDER BY score DESC
|
||||||
LIMIT $top_k
|
LIMIT $top_k
|
||||||
"""
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
params = {
|
params = {
|
||||||
"query": request.query,
|
"query": request.query,
|
||||||
@@ -370,6 +410,7 @@ class SearchService:
|
|||||||
"library_uid": request.library_uid,
|
"library_uid": request.library_uid,
|
||||||
"library_type": request.library_type,
|
"library_type": request.library_type,
|
||||||
"collection_uid": request.collection_uid,
|
"collection_uid": request.collection_uid,
|
||||||
|
"workspace_id": request.workspace_id,
|
||||||
}
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -402,7 +443,8 @@ class SearchService:
|
|||||||
candidates: dict[str, SearchCandidate],
|
candidates: dict[str, SearchCandidate],
|
||||||
):
|
):
|
||||||
"""Search concept_name_fulltext and traverse to chunks."""
|
"""Search concept_name_fulltext and traverse to chunks."""
|
||||||
cypher = """
|
cypher = (
|
||||||
|
"""
|
||||||
CALL db.index.fulltext.queryNodes('concept_name_fulltext', $query)
|
CALL db.index.fulltext.queryNodes('concept_name_fulltext', $query)
|
||||||
YIELD node AS concept, score AS concept_score
|
YIELD node AS concept, score AS concept_score
|
||||||
MATCH (chunk:Chunk)-[:MENTIONS]->(concept)
|
MATCH (chunk:Chunk)-[:MENTIONS]->(concept)
|
||||||
@@ -410,6 +452,9 @@ class SearchService:
|
|||||||
MATCH (lib:Library)-[:CONTAINS]->(:Collection)-[:CONTAINS]->(item)
|
MATCH (lib:Library)-[:CONTAINS]->(:Collection)-[:CONTAINS]->(item)
|
||||||
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
||||||
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
||||||
|
"""
|
||||||
|
+ _WORKSPACE_SCOPE_CLAUSE
|
||||||
|
+ """
|
||||||
RETURN chunk.uid AS chunk_uid, chunk.text_preview AS text_preview,
|
RETURN chunk.uid AS chunk_uid, chunk.text_preview AS text_preview,
|
||||||
chunk.chunk_s3_key AS chunk_s3_key, chunk.chunk_index AS chunk_index,
|
chunk.chunk_s3_key AS chunk_s3_key, chunk.chunk_index AS chunk_index,
|
||||||
item.uid AS item_uid, item.title AS item_title,
|
item.uid AS item_uid, item.title AS item_title,
|
||||||
@@ -418,12 +463,14 @@ class SearchService:
|
|||||||
ORDER BY score DESC
|
ORDER BY score DESC
|
||||||
LIMIT $top_k
|
LIMIT $top_k
|
||||||
"""
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
params = {
|
params = {
|
||||||
"query": request.query,
|
"query": request.query,
|
||||||
"top_k": top_k,
|
"top_k": top_k,
|
||||||
"library_uid": request.library_uid,
|
"library_uid": request.library_uid,
|
||||||
"library_type": request.library_type,
|
"library_type": request.library_type,
|
||||||
|
"workspace_id": request.workspace_id,
|
||||||
}
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -465,7 +512,8 @@ class SearchService:
|
|||||||
"""
|
"""
|
||||||
start = time.time()
|
start = time.time()
|
||||||
|
|
||||||
cypher = """
|
cypher = (
|
||||||
|
"""
|
||||||
CALL db.index.fulltext.queryNodes('concept_name_fulltext', $query)
|
CALL db.index.fulltext.queryNodes('concept_name_fulltext', $query)
|
||||||
YIELD node AS concept, score AS concept_score
|
YIELD node AS concept, score AS concept_score
|
||||||
WITH concept, concept_score
|
WITH concept, concept_score
|
||||||
@@ -476,6 +524,9 @@ class SearchService:
|
|||||||
MATCH (lib:Library)-[:CONTAINS]->(:Collection)-[:CONTAINS]->(item)
|
MATCH (lib:Library)-[:CONTAINS]->(:Collection)-[:CONTAINS]->(item)
|
||||||
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
||||||
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
||||||
|
"""
|
||||||
|
+ _WORKSPACE_SCOPE_CLAUSE
|
||||||
|
+ """
|
||||||
WITH chunk, item, lib,
|
WITH chunk, item, lib,
|
||||||
max(concept_score) AS score,
|
max(concept_score) AS score,
|
||||||
collect(DISTINCT concept.name)[..5] AS concept_names
|
collect(DISTINCT concept.name)[..5] AS concept_names
|
||||||
@@ -487,12 +538,14 @@ class SearchService:
|
|||||||
ORDER BY score DESC
|
ORDER BY score DESC
|
||||||
LIMIT $limit
|
LIMIT $limit
|
||||||
"""
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
params = {
|
params = {
|
||||||
"query": request.query,
|
"query": request.query,
|
||||||
"limit": request.fulltext_top_k,
|
"limit": request.fulltext_top_k,
|
||||||
"library_uid": request.library_uid,
|
"library_uid": request.library_uid,
|
||||||
"library_type": request.library_type,
|
"library_type": request.library_type,
|
||||||
|
"workspace_id": request.workspace_id,
|
||||||
}
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -550,7 +603,8 @@ class SearchService:
|
|||||||
"""
|
"""
|
||||||
start = time.time()
|
start = time.time()
|
||||||
|
|
||||||
cypher = """
|
cypher = (
|
||||||
|
"""
|
||||||
CALL db.index.vector.queryNodes('image_embedding_index', $top_k, $query_vector)
|
CALL db.index.vector.queryNodes('image_embedding_index', $top_k, $query_vector)
|
||||||
YIELD node AS emb_node, score
|
YIELD node AS emb_node, score
|
||||||
MATCH (img:Image)-[:HAS_EMBEDDING]->(emb_node)
|
MATCH (img:Image)-[:HAS_EMBEDDING]->(emb_node)
|
||||||
@@ -558,6 +612,9 @@ class SearchService:
|
|||||||
MATCH (lib:Library)-[:CONTAINS]->(:Collection)-[:CONTAINS]->(item)
|
MATCH (lib:Library)-[:CONTAINS]->(:Collection)-[:CONTAINS]->(item)
|
||||||
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
WHERE ($library_uid IS NULL OR lib.uid = $library_uid)
|
||||||
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
AND ($library_type IS NULL OR lib.library_type = $library_type)
|
||||||
|
"""
|
||||||
|
+ _WORKSPACE_SCOPE_CLAUSE
|
||||||
|
+ """
|
||||||
RETURN img.uid AS image_uid, img.image_type AS image_type,
|
RETURN img.uid AS image_uid, img.image_type AS image_type,
|
||||||
img.description AS description, img.s3_key AS s3_key,
|
img.description AS description, img.s3_key AS s3_key,
|
||||||
item.uid AS item_uid, item.title AS item_title,
|
item.uid AS item_uid, item.title AS item_title,
|
||||||
@@ -565,12 +622,14 @@ class SearchService:
|
|||||||
ORDER BY score DESC
|
ORDER BY score DESC
|
||||||
LIMIT 10
|
LIMIT 10
|
||||||
"""
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
params = {
|
params = {
|
||||||
"top_k": 10,
|
"top_k": 10,
|
||||||
"query_vector": query_vector,
|
"query_vector": query_vector,
|
||||||
"library_uid": request.library_uid,
|
"library_uid": request.library_uid,
|
||||||
"library_type": request.library_type,
|
"library_type": request.library_type,
|
||||||
|
"workspace_id": request.workspace_id,
|
||||||
}
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import logging
|
|||||||
|
|
||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
from django.core.cache import cache
|
from django.core.cache import cache
|
||||||
|
from neomodel import db
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -280,3 +281,212 @@ def _resolve_user(user_id: int = None):
|
|||||||
return User.objects.get(pk=user_id)
|
return User.objects.get(pk=user_id)
|
||||||
except Exception:
|
except Exception:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Ingest task (Daedalus integration)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@shared_task(
|
||||||
|
name="library.tasks.ingest_from_daedalus",
|
||||||
|
bind=True,
|
||||||
|
queue="embedding",
|
||||||
|
max_retries=3,
|
||||||
|
default_retry_delay=60,
|
||||||
|
acks_late=True,
|
||||||
|
)
|
||||||
|
def ingest_from_daedalus(self, job_id: str):
|
||||||
|
"""
|
||||||
|
Process a single IngestJob: fetch from Daedalus S3 → create Item →
|
||||||
|
run embedding pipeline → mark complete.
|
||||||
|
|
||||||
|
Idempotent on (library_uid, source_ref, content_hash) — handled in the
|
||||||
|
REST view that creates the IngestJob, so by the time this task runs the
|
||||||
|
job either represents new content or a content_hash-changed re-ingest.
|
||||||
|
|
||||||
|
For a content_hash-changed re-ingest, the prior Item with the same
|
||||||
|
source_ref is deleted before the new one is processed (ensures no
|
||||||
|
stale chunks linger).
|
||||||
|
"""
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from library.models import IngestJob, Item, Library
|
||||||
|
from library.services.daedalus_s3 import (
|
||||||
|
copy_into_mnemosyne,
|
||||||
|
fetch_from_daedalus,
|
||||||
|
)
|
||||||
|
from library.services.pipeline import EmbeddingPipeline
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Task ingest_from_daedalus starting job_id=%s task_id=%s",
|
||||||
|
job_id, self.request.id,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
job = IngestJob.objects.get(pk=job_id)
|
||||||
|
except IngestJob.DoesNotExist:
|
||||||
|
logger.error("IngestJob not found job_id=%s", job_id)
|
||||||
|
return {"success": False, "error": "job_not_found"}
|
||||||
|
|
||||||
|
job.status = "processing"
|
||||||
|
job.progress = "fetching"
|
||||||
|
job.started_at = datetime.now(timezone.utc)
|
||||||
|
job.celery_task_id = self.request.id
|
||||||
|
job.save(update_fields=["status", "progress", "started_at", "celery_task_id"])
|
||||||
|
|
||||||
|
try:
|
||||||
|
# --- 1. Resolve target Library ---
|
||||||
|
try:
|
||||||
|
lib = Library.nodes.get(uid=job.library_uid)
|
||||||
|
except Library.DoesNotExist:
|
||||||
|
raise RuntimeError(f"Library not found: {job.library_uid}")
|
||||||
|
|
||||||
|
# --- 2. Supersede prior Item with same source_ref but different hash ---
|
||||||
|
prior_item_uid = None
|
||||||
|
if job.source_ref:
|
||||||
|
rows, _ = db.cypher_query(
|
||||||
|
"""
|
||||||
|
MATCH (l:Library {uid: $library_uid})-[:CONTAINS]->(:Collection)
|
||||||
|
-[:CONTAINS]->(i:Item)
|
||||||
|
WHERE i.metadata IS NOT NULL
|
||||||
|
AND i.metadata CONTAINS $source_ref_marker
|
||||||
|
RETURN i.uid LIMIT 1
|
||||||
|
""",
|
||||||
|
{
|
||||||
|
"library_uid": lib.uid,
|
||||||
|
"source_ref_marker": f'"source_ref": "{job.source_ref}"',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if rows:
|
||||||
|
prior_item_uid = rows[0][0]
|
||||||
|
logger.info(
|
||||||
|
"Superseding prior Item job_id=%s prior_item_uid=%s",
|
||||||
|
job_id, prior_item_uid,
|
||||||
|
)
|
||||||
|
_delete_item_and_chunks(prior_item_uid)
|
||||||
|
|
||||||
|
# --- 3. Fetch from Daedalus, copy into Mnemosyne bucket ---
|
||||||
|
job.progress = "copying"
|
||||||
|
job.save(update_fields=["progress"])
|
||||||
|
|
||||||
|
data = fetch_from_daedalus(job.s3_key)
|
||||||
|
|
||||||
|
# --- 4. Create Item node ---
|
||||||
|
ext = (job.file_type or "bin").lstrip(".").lower() or "bin"
|
||||||
|
item = Item(
|
||||||
|
title=job.title,
|
||||||
|
file_type=ext,
|
||||||
|
file_size=len(data),
|
||||||
|
content_hash=job.content_hash,
|
||||||
|
embedding_status="pending",
|
||||||
|
metadata={
|
||||||
|
"source": job.source,
|
||||||
|
"source_ref": job.source_ref,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
item.save()
|
||||||
|
|
||||||
|
mnemosyne_s3_key = f"items/{item.uid}/original.{ext}"
|
||||||
|
copy_into_mnemosyne(data, mnemosyne_s3_key)
|
||||||
|
item.s3_key = mnemosyne_s3_key
|
||||||
|
item.save()
|
||||||
|
|
||||||
|
# --- 5. Connect to library/collection ---
|
||||||
|
col = _resolve_or_create_default_collection(lib, job.collection_uid)
|
||||||
|
col.items.connect(item)
|
||||||
|
|
||||||
|
job.item_uid = item.uid
|
||||||
|
job.save(update_fields=["item_uid"])
|
||||||
|
|
||||||
|
# --- 6. Run the embedding pipeline ---
|
||||||
|
job.progress = "embedding"
|
||||||
|
job.save(update_fields=["progress"])
|
||||||
|
|
||||||
|
def progress_cb(percent, message):
|
||||||
|
_update_progress(self, percent, message)
|
||||||
|
|
||||||
|
pipeline = EmbeddingPipeline(user=None)
|
||||||
|
result = pipeline.process_item(item.uid, progress_callback=progress_cb)
|
||||||
|
|
||||||
|
# --- 7. Mark complete ---
|
||||||
|
job.status = "completed"
|
||||||
|
job.progress = "done"
|
||||||
|
job.chunks_created = result.get("chunks_created", 0)
|
||||||
|
job.concepts_extracted = result.get("concepts_extracted", 0)
|
||||||
|
job.embedding_model = result.get("embedding_model", "")
|
||||||
|
job.completed_at = datetime.now(timezone.utc)
|
||||||
|
job.save()
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Task ingest_from_daedalus completed job_id=%s item_uid=%s "
|
||||||
|
"chunks=%d concepts=%d",
|
||||||
|
job_id, item.uid, job.chunks_created, job.concepts_extracted,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"job_id": job_id,
|
||||||
|
"item_uid": item.uid,
|
||||||
|
**result,
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error(
|
||||||
|
"Task ingest_from_daedalus failed job_id=%s: %s",
|
||||||
|
job_id, exc, exc_info=True,
|
||||||
|
)
|
||||||
|
if self.request.retries < self.max_retries:
|
||||||
|
job.retry_count = self.request.retries + 1
|
||||||
|
job.save(update_fields=["retry_count"])
|
||||||
|
raise self.retry(exc=exc)
|
||||||
|
|
||||||
|
job.status = "failed"
|
||||||
|
job.error = str(exc)
|
||||||
|
job.completed_at = datetime.now(timezone.utc)
|
||||||
|
job.save(update_fields=["status", "error", "completed_at"])
|
||||||
|
return {"success": False, "job_id": job_id, "error": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
|
def _delete_item_and_chunks(item_uid: str):
|
||||||
|
"""Delete an Item, its chunks, and its images. Concept GC is workspace-delete only."""
|
||||||
|
db.cypher_query(
|
||||||
|
"""
|
||||||
|
MATCH (i:Item {uid: $uid})
|
||||||
|
OPTIONAL MATCH (i)-[:HAS_CHUNK]->(c:Chunk)
|
||||||
|
OPTIONAL MATCH (i)-[:HAS_IMAGE]->(img:Image)
|
||||||
|
OPTIONAL MATCH (img)-[:HAS_EMBEDDING]->(emb:ImageEmbedding)
|
||||||
|
DETACH DELETE c, img, emb, i
|
||||||
|
""",
|
||||||
|
{"uid": item_uid},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_or_create_default_collection(lib, collection_uid: str = ""):
|
||||||
|
"""
|
||||||
|
Find or create the default Collection for a Library.
|
||||||
|
|
||||||
|
Daedalus integration creates one Collection per Library, named "default".
|
||||||
|
Explicit collection_uid is honored if provided.
|
||||||
|
"""
|
||||||
|
from library.models import Collection
|
||||||
|
|
||||||
|
if collection_uid:
|
||||||
|
try:
|
||||||
|
return Collection.nodes.get(uid=collection_uid)
|
||||||
|
except Collection.DoesNotExist:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Look for an existing "default" collection in this library
|
||||||
|
rows, _ = db.cypher_query(
|
||||||
|
"MATCH (l:Library {uid: $library_uid})-[:CONTAINS]->(c:Collection {name: 'default'}) "
|
||||||
|
"RETURN c.uid LIMIT 1",
|
||||||
|
{"library_uid": lib.uid},
|
||||||
|
)
|
||||||
|
if rows:
|
||||||
|
return Collection.nodes.get(uid=rows[0][0])
|
||||||
|
|
||||||
|
col = Collection(name="default", description="Default collection")
|
||||||
|
col.save()
|
||||||
|
lib.collections.connect(col)
|
||||||
|
col.library.connect(lib)
|
||||||
|
return col
|
||||||
|
|||||||
@@ -13,7 +13,17 @@ from library.content_types import LIBRARY_TYPE_DEFAULTS, get_library_type_config
|
|||||||
class LibraryTypeDefaultsTests(TestCase):
|
class LibraryTypeDefaultsTests(TestCase):
|
||||||
"""Tests for the LIBRARY_TYPE_DEFAULTS registry."""
|
"""Tests for the LIBRARY_TYPE_DEFAULTS registry."""
|
||||||
|
|
||||||
EXPECTED_TYPES = {"fiction", "nonfiction", "technical", "music", "film", "art", "journal"}
|
EXPECTED_TYPES = {
|
||||||
|
"fiction",
|
||||||
|
"nonfiction",
|
||||||
|
"technical",
|
||||||
|
"music",
|
||||||
|
"film",
|
||||||
|
"art",
|
||||||
|
"journal",
|
||||||
|
"business",
|
||||||
|
"finance",
|
||||||
|
}
|
||||||
|
|
||||||
def test_all_expected_types_present(self):
|
def test_all_expected_types_present(self):
|
||||||
for lib_type in self.EXPECTED_TYPES:
|
for lib_type in self.EXPECTED_TYPES:
|
||||||
@@ -105,6 +115,16 @@ class VisionPromptTests(TestCase):
|
|||||||
prompt = config["vision_prompt"].lower()
|
prompt = config["vision_prompt"].lower()
|
||||||
self.assertIn("historical", prompt)
|
self.assertIn("historical", prompt)
|
||||||
|
|
||||||
|
def test_business_vision_prompt_mentions_logo_or_chart(self):
|
||||||
|
config = get_library_type_config("business")
|
||||||
|
prompt = config["vision_prompt"].lower()
|
||||||
|
self.assertTrue("logo" in prompt or "chart" in prompt)
|
||||||
|
|
||||||
|
def test_finance_llm_context_forbids_fabrication(self):
|
||||||
|
config = get_library_type_config("finance")
|
||||||
|
prompt = config["llm_context_prompt"].lower()
|
||||||
|
self.assertIn("fabricate", prompt)
|
||||||
|
|
||||||
|
|
||||||
class GetLibraryTypeConfigTests(TestCase):
|
class GetLibraryTypeConfigTests(TestCase):
|
||||||
"""Tests for the get_library_type_config helper."""
|
"""Tests for the get_library_type_config helper."""
|
||||||
|
|||||||
71
mnemosyne/library/tests/test_search_scoping.py
Normal file
71
mnemosyne/library/tests/test_search_scoping.py
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
"""
|
||||||
|
Tests for workspace scoping in SearchRequest and the Cypher scope clause.
|
||||||
|
|
||||||
|
These exercise the dataclass-level normalization and the construction
|
||||||
|
of Cypher parameter dicts. The actual Cypher execution against Neo4j
|
||||||
|
is validated by the manual end-to-end test plan.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from django.test import TestCase
|
||||||
|
|
||||||
|
from library.services.search import _WORKSPACE_SCOPE_CLAUSE, SearchRequest
|
||||||
|
|
||||||
|
|
||||||
|
class SearchRequestScopingTests(TestCase):
|
||||||
|
"""SearchRequest workspace_id behavior."""
|
||||||
|
|
||||||
|
def test_default_workspace_id_is_none(self):
|
||||||
|
req = SearchRequest(query="hello")
|
||||||
|
self.assertIsNone(req.workspace_id)
|
||||||
|
|
||||||
|
def test_explicit_workspace_id_preserved(self):
|
||||||
|
req = SearchRequest(query="hello", workspace_id="ws_abc")
|
||||||
|
self.assertEqual(req.workspace_id, "ws_abc")
|
||||||
|
|
||||||
|
def test_empty_string_workspace_id_normalized_to_none(self):
|
||||||
|
"""Empty strings must NOT slip through as a truthy filter at the Cypher boundary."""
|
||||||
|
req = SearchRequest(query="hello", workspace_id="")
|
||||||
|
self.assertIsNone(req.workspace_id)
|
||||||
|
|
||||||
|
def test_empty_string_library_uid_normalized_to_none(self):
|
||||||
|
req = SearchRequest(query="hello", library_uid="")
|
||||||
|
self.assertIsNone(req.library_uid)
|
||||||
|
|
||||||
|
def test_empty_string_library_type_normalized_to_none(self):
|
||||||
|
req = SearchRequest(query="hello", library_type="")
|
||||||
|
self.assertIsNone(req.library_type)
|
||||||
|
|
||||||
|
def test_empty_string_collection_uid_normalized_to_none(self):
|
||||||
|
req = SearchRequest(query="hello", collection_uid="")
|
||||||
|
self.assertIsNone(req.collection_uid)
|
||||||
|
|
||||||
|
|
||||||
|
class WorkspaceScopeClauseTests(TestCase):
|
||||||
|
"""Sanity checks on the Cypher snippet itself.
|
||||||
|
|
||||||
|
The clause must produce two distinct, non-overlapping result sets:
|
||||||
|
1. workspace_id IS NULL → only global libraries (lib.workspace_id IS NULL)
|
||||||
|
2. workspace_id = X → only libraries with workspace_id = X
|
||||||
|
|
||||||
|
A "leaks both" bug would be a Cypher OR that fails to bracket properly.
|
||||||
|
Verifying the literal string here is a cheap regression guard against
|
||||||
|
refactors that accidentally change the operator precedence.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_clause_references_lib_workspace_id(self):
|
||||||
|
self.assertIn("lib.workspace_id", _WORKSPACE_SCOPE_CLAUSE)
|
||||||
|
|
||||||
|
def test_clause_references_workspace_id_param(self):
|
||||||
|
self.assertIn("$workspace_id", _WORKSPACE_SCOPE_CLAUSE)
|
||||||
|
|
||||||
|
def test_clause_handles_both_modes(self):
|
||||||
|
"""Both 'IS NULL' and '=' branches must be present."""
|
||||||
|
self.assertIn("IS NULL", _WORKSPACE_SCOPE_CLAUSE)
|
||||||
|
self.assertIn("=", _WORKSPACE_SCOPE_CLAUSE)
|
||||||
|
|
||||||
|
def test_clause_starts_with_AND_so_it_appends_safely(self):
|
||||||
|
"""The clause is appended to existing WHERE filters."""
|
||||||
|
self.assertTrue(
|
||||||
|
_WORKSPACE_SCOPE_CLAUSE.lstrip().startswith("AND"),
|
||||||
|
f"Clause must start with AND: {_WORKSPACE_SCOPE_CLAUSE!r}",
|
||||||
|
)
|
||||||
210
mnemosyne/library/tests/test_workspaces_api.py
Normal file
210
mnemosyne/library/tests/test_workspaces_api.py
Normal file
@@ -0,0 +1,210 @@
|
|||||||
|
"""
|
||||||
|
Tests for workspace and ingest REST endpoints.
|
||||||
|
|
||||||
|
These exercise serializer validation, idempotency rules, and Django ORM
|
||||||
|
behavior for IngestJob. The Cypher-touching paths (Library node CRUD,
|
||||||
|
search scoping) require Neo4j and are validated by the manual end-to-end
|
||||||
|
test plan, not these unit tests.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from django.test import TestCase
|
||||||
|
from rest_framework.test import APIClient
|
||||||
|
|
||||||
|
from library.api.serializers import (
|
||||||
|
IngestRequestSerializer,
|
||||||
|
WorkspaceCreateSerializer,
|
||||||
|
)
|
||||||
|
from library.models import IngestJob
|
||||||
|
|
||||||
|
|
||||||
|
class WorkspaceCreateSerializerTests(TestCase):
|
||||||
|
"""Validation rules for the create-workspace payload."""
|
||||||
|
|
||||||
|
def test_minimal_payload_validates(self):
|
||||||
|
s = WorkspaceCreateSerializer(
|
||||||
|
data={
|
||||||
|
"workspace_id": "ws_abc",
|
||||||
|
"name": "My Workspace",
|
||||||
|
"library_type": "technical",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertTrue(s.is_valid(), s.errors)
|
||||||
|
|
||||||
|
def test_business_type_accepted(self):
|
||||||
|
s = WorkspaceCreateSerializer(
|
||||||
|
data={
|
||||||
|
"workspace_id": "ws_abc",
|
||||||
|
"name": "Sales",
|
||||||
|
"library_type": "business",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertTrue(s.is_valid(), s.errors)
|
||||||
|
|
||||||
|
def test_finance_type_accepted(self):
|
||||||
|
s = WorkspaceCreateSerializer(
|
||||||
|
data={
|
||||||
|
"workspace_id": "ws_abc",
|
||||||
|
"name": "Money",
|
||||||
|
"library_type": "finance",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertTrue(s.is_valid(), s.errors)
|
||||||
|
|
||||||
|
def test_unknown_type_rejected(self):
|
||||||
|
s = WorkspaceCreateSerializer(
|
||||||
|
data={
|
||||||
|
"workspace_id": "ws_abc",
|
||||||
|
"name": "x",
|
||||||
|
"library_type": "miscellaneous",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertFalse(s.is_valid())
|
||||||
|
self.assertIn("library_type", s.errors)
|
||||||
|
|
||||||
|
def test_workspace_id_required(self):
|
||||||
|
s = WorkspaceCreateSerializer(
|
||||||
|
data={"name": "x", "library_type": "technical"}
|
||||||
|
)
|
||||||
|
self.assertFalse(s.is_valid())
|
||||||
|
self.assertIn("workspace_id", s.errors)
|
||||||
|
|
||||||
|
|
||||||
|
class IngestRequestSerializerTests(TestCase):
|
||||||
|
"""Validation rules for the ingest payload."""
|
||||||
|
|
||||||
|
BASE_PAYLOAD = {
|
||||||
|
"s3_key": "workspaces/ws/files/f/x.pdf",
|
||||||
|
"title": "Q4 Report",
|
||||||
|
"content_hash": "a" * 64,
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_workspace_id_only_validates(self):
|
||||||
|
s = IngestRequestSerializer(
|
||||||
|
data={**self.BASE_PAYLOAD, "workspace_id": "ws_abc"}
|
||||||
|
)
|
||||||
|
self.assertTrue(s.is_valid(), s.errors)
|
||||||
|
|
||||||
|
def test_library_uid_only_validates(self):
|
||||||
|
s = IngestRequestSerializer(
|
||||||
|
data={**self.BASE_PAYLOAD, "library_uid": "lib_xyz"}
|
||||||
|
)
|
||||||
|
self.assertTrue(s.is_valid(), s.errors)
|
||||||
|
|
||||||
|
def test_neither_workspace_nor_library_rejected(self):
|
||||||
|
s = IngestRequestSerializer(data=self.BASE_PAYLOAD)
|
||||||
|
self.assertFalse(s.is_valid())
|
||||||
|
# ValidationError on the whole payload, not a specific field
|
||||||
|
self.assertIn("non_field_errors", s.errors)
|
||||||
|
|
||||||
|
def test_content_hash_required(self):
|
||||||
|
s = IngestRequestSerializer(
|
||||||
|
data={
|
||||||
|
"s3_key": "x",
|
||||||
|
"title": "y",
|
||||||
|
"workspace_id": "ws_abc",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertFalse(s.is_valid())
|
||||||
|
self.assertIn("content_hash", s.errors)
|
||||||
|
|
||||||
|
|
||||||
|
class IngestJobModelTests(TestCase):
|
||||||
|
"""IngestJob persists and queries correctly."""
|
||||||
|
|
||||||
|
def test_create_with_minimal_fields(self):
|
||||||
|
job = IngestJob.objects.create(
|
||||||
|
id="job_test1",
|
||||||
|
library_uid="lib_xyz",
|
||||||
|
s3_key="x.pdf",
|
||||||
|
)
|
||||||
|
self.assertEqual(job.status, "pending")
|
||||||
|
self.assertEqual(job.progress, "queued")
|
||||||
|
self.assertEqual(job.retry_count, 0)
|
||||||
|
self.assertEqual(job.chunks_created, 0)
|
||||||
|
|
||||||
|
def test_idempotency_query_pattern(self):
|
||||||
|
"""The idempotency query in ingest_create uses (library, source_ref, hash)."""
|
||||||
|
IngestJob.objects.create(
|
||||||
|
id="job_a",
|
||||||
|
library_uid="lib_xyz",
|
||||||
|
source_ref="ws_a/file_1",
|
||||||
|
content_hash="h1",
|
||||||
|
s3_key="a.pdf",
|
||||||
|
status="completed",
|
||||||
|
)
|
||||||
|
IngestJob.objects.create(
|
||||||
|
id="job_b",
|
||||||
|
library_uid="lib_xyz",
|
||||||
|
source_ref="ws_a/file_1",
|
||||||
|
content_hash="h2", # different hash — supersedes
|
||||||
|
s3_key="a.pdf",
|
||||||
|
status="completed",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Same library + source_ref + h1 → finds job_a
|
||||||
|
match = IngestJob.objects.filter(
|
||||||
|
library_uid="lib_xyz",
|
||||||
|
source_ref="ws_a/file_1",
|
||||||
|
content_hash="h1",
|
||||||
|
).first()
|
||||||
|
self.assertIsNotNone(match)
|
||||||
|
self.assertEqual(match.id, "job_a")
|
||||||
|
|
||||||
|
# h2 → finds job_b
|
||||||
|
match = IngestJob.objects.filter(
|
||||||
|
library_uid="lib_xyz",
|
||||||
|
source_ref="ws_a/file_1",
|
||||||
|
content_hash="h2",
|
||||||
|
).first()
|
||||||
|
self.assertEqual(match.id, "job_b")
|
||||||
|
|
||||||
|
# Different source_ref → no match
|
||||||
|
match = IngestJob.objects.filter(
|
||||||
|
library_uid="lib_xyz",
|
||||||
|
source_ref="ws_a/file_2",
|
||||||
|
content_hash="h1",
|
||||||
|
).first()
|
||||||
|
self.assertIsNone(match)
|
||||||
|
|
||||||
|
|
||||||
|
class IngestEndpointAuthTests(TestCase):
|
||||||
|
"""Auth boundary on the ingest endpoint (matches the existing convention)."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.client = APIClient()
|
||||||
|
|
||||||
|
def test_ingest_requires_auth(self):
|
||||||
|
response = self.client.post(
|
||||||
|
"/library/api/ingest/",
|
||||||
|
{
|
||||||
|
"s3_key": "x",
|
||||||
|
"title": "y",
|
||||||
|
"workspace_id": "ws_abc",
|
||||||
|
"content_hash": "a" * 64,
|
||||||
|
},
|
||||||
|
format="json",
|
||||||
|
)
|
||||||
|
self.assertIn(response.status_code, [401, 403])
|
||||||
|
|
||||||
|
|
||||||
|
class WorkspaceEndpointAuthTests(TestCase):
|
||||||
|
"""Auth on workspace endpoints."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.client = APIClient()
|
||||||
|
|
||||||
|
def test_workspace_create_requires_auth(self):
|
||||||
|
response = self.client.post(
|
||||||
|
"/library/api/workspaces/",
|
||||||
|
{"workspace_id": "ws_a", "name": "x", "library_type": "technical"},
|
||||||
|
format="json",
|
||||||
|
)
|
||||||
|
self.assertIn(response.status_code, [401, 403])
|
||||||
|
|
||||||
|
def test_workspace_get_requires_auth(self):
|
||||||
|
response = self.client.get("/library/api/workspaces/ws_a/")
|
||||||
|
self.assertIn(response.status_code, [401, 403])
|
||||||
|
|
||||||
|
def test_workspace_delete_requires_auth(self):
|
||||||
|
response = self.client.delete("/library/api/workspaces/ws_a/")
|
||||||
|
self.assertIn(response.status_code, [401, 403])
|
||||||
@@ -16,14 +16,23 @@ class MCPTokenAdmin(admin.ModelAdmin):
|
|||||||
]
|
]
|
||||||
list_filter = ["is_active"]
|
list_filter = ["is_active"]
|
||||||
search_fields = ["name", "user__email", "user__username"]
|
search_fields = ["name", "user__email", "user__username"]
|
||||||
readonly_fields = ["token", "last_used_at", "created_at", "updated_at"]
|
readonly_fields = ["token_hash", "last_used_at", "created_at", "updated_at"]
|
||||||
fieldsets = (
|
fieldsets = (
|
||||||
(None, {"fields": ("user", "name", "is_active")}),
|
(None, {"fields": ("user", "name", "is_active")}),
|
||||||
("Restrictions", {"fields": ("allowed_tools", "expires_at")}),
|
("Restrictions", {"fields": ("allowed_tools", "expires_at")}),
|
||||||
("Token (shown once at creation)", {"fields": ("token",)}),
|
(
|
||||||
|
"Token (hashed at rest — plaintext is shown only once at creation)",
|
||||||
|
{"fields": ("token_hash",)},
|
||||||
|
),
|
||||||
("Audit", {"fields": ("last_used_at", "created_at", "updated_at")}),
|
("Audit", {"fields": ("last_used_at", "created_at", "updated_at")}),
|
||||||
)
|
)
|
||||||
|
|
||||||
@admin.display(description="Token")
|
@admin.display(description="Token")
|
||||||
def masked_token(self, obj):
|
def masked_token(self, obj):
|
||||||
return obj.get_masked_token()
|
return obj.get_masked_token()
|
||||||
|
|
||||||
|
def has_add_permission(self, request):
|
||||||
|
# Tokens must be created via the dashboard or management command
|
||||||
|
# so the plaintext can be surfaced to the user. Adding via admin
|
||||||
|
# would persist a hash with no plaintext ever shown.
|
||||||
|
return False
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from fastmcp.server.dependencies import get_http_request
|
|||||||
from fastmcp.server.middleware import Middleware, MiddlewareContext
|
from fastmcp.server.middleware import Middleware, MiddlewareContext
|
||||||
|
|
||||||
from .metrics import mcp_auth_failures_total
|
from .metrics import mcp_auth_failures_total
|
||||||
from .models import MCPToken
|
from .models import MCPToken, hash_token
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -25,9 +25,17 @@ class MCPAuthError(Exception):
|
|||||||
|
|
||||||
|
|
||||||
def resolve_mcp_user(token_string: str):
|
def resolve_mcp_user(token_string: str):
|
||||||
"""Resolve a bearer token to (user, MCPToken). Raises MCPAuthError on any failure."""
|
"""Resolve a bearer token to (user, MCPToken). Raises MCPAuthError on any failure.
|
||||||
|
|
||||||
|
Hashes the incoming bearer and looks up by the hash — plaintext is never
|
||||||
|
stored or compared directly.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
token = MCPToken.objects.select_related("user").get(token=token_string)
|
token = (
|
||||||
|
MCPToken.objects
|
||||||
|
.select_related("user")
|
||||||
|
.get(token_hash=hash_token(token_string))
|
||||||
|
)
|
||||||
except MCPToken.DoesNotExist:
|
except MCPToken.DoesNotExist:
|
||||||
raise MCPAuthError("Invalid MCP token.")
|
raise MCPAuthError("Invalid MCP token.")
|
||||||
|
|
||||||
|
|||||||
85
mnemosyne/mcp_server/forms.py
Normal file
85
mnemosyne/mcp_server/forms.py
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
"""Forms for the MCP token self-service dashboard."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import functools
|
||||||
|
|
||||||
|
from django import forms
|
||||||
|
|
||||||
|
from .models import MCPToken
|
||||||
|
|
||||||
|
|
||||||
|
@functools.lru_cache(maxsize=1)
|
||||||
|
def _registered_tool_names() -> list[str]:
|
||||||
|
"""Pull the list of registered MCP tool names from the FastMCP instance.
|
||||||
|
|
||||||
|
Cached at module level — the tool registry is fixed at server startup.
|
||||||
|
Importing here (rather than at module import) avoids circulars when the
|
||||||
|
server module imports the dashboard pieces transitively.
|
||||||
|
"""
|
||||||
|
from .server import mcp
|
||||||
|
|
||||||
|
tools = asyncio.run(mcp.get_tools())
|
||||||
|
return sorted(tools.keys())
|
||||||
|
|
||||||
|
|
||||||
|
def _tool_choices() -> list[tuple[str, str]]:
|
||||||
|
return [(name, name) for name in _registered_tool_names()]
|
||||||
|
|
||||||
|
|
||||||
|
class MCPTokenCreateForm(forms.Form):
|
||||||
|
"""Generate a new bearer token. The token value itself is server-generated."""
|
||||||
|
|
||||||
|
name = forms.CharField(
|
||||||
|
max_length=100,
|
||||||
|
widget=forms.TextInput(attrs={
|
||||||
|
"class": "input input-bordered w-full",
|
||||||
|
"placeholder": "e.g. Claude Desktop, CI script",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
expires_at = forms.DateTimeField(
|
||||||
|
required=False,
|
||||||
|
widget=forms.DateTimeInput(attrs={
|
||||||
|
"class": "input input-bordered w-full",
|
||||||
|
"type": "datetime-local",
|
||||||
|
}),
|
||||||
|
help_text="Leave blank for no expiry.",
|
||||||
|
)
|
||||||
|
allowed_tools = forms.MultipleChoiceField(
|
||||||
|
required=False,
|
||||||
|
widget=forms.CheckboxSelectMultiple(attrs={"class": "checkbox checkbox-sm"}),
|
||||||
|
help_text="Leave all unchecked to permit every tool.",
|
||||||
|
)
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self.fields["allowed_tools"].choices = _tool_choices()
|
||||||
|
|
||||||
|
|
||||||
|
class MCPTokenEditForm(forms.ModelForm):
|
||||||
|
"""Edit token metadata. The hashed token itself cannot be edited."""
|
||||||
|
|
||||||
|
allowed_tools = forms.MultipleChoiceField(
|
||||||
|
required=False,
|
||||||
|
widget=forms.CheckboxSelectMultiple(attrs={"class": "checkbox checkbox-sm"}),
|
||||||
|
help_text="Leave all unchecked to permit every tool.",
|
||||||
|
)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
model = MCPToken
|
||||||
|
fields = ["name", "is_active", "expires_at", "allowed_tools"]
|
||||||
|
widgets = {
|
||||||
|
"name": forms.TextInput(attrs={"class": "input input-bordered w-full"}),
|
||||||
|
"is_active": forms.CheckboxInput(attrs={"class": "toggle toggle-primary"}),
|
||||||
|
"expires_at": forms.DateTimeInput(attrs={
|
||||||
|
"class": "input input-bordered w-full",
|
||||||
|
"type": "datetime-local",
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self.fields["allowed_tools"].choices = _tool_choices()
|
||||||
|
if self.instance and self.instance.pk:
|
||||||
|
self.fields["allowed_tools"].initial = self.instance.allowed_tools or []
|
||||||
@@ -57,7 +57,7 @@ class Command(BaseCommand):
|
|||||||
raise CommandError("--expires-days must be at least 1.")
|
raise CommandError("--expires-days must be at least 1.")
|
||||||
expires_at = timezone.now() + timedelta(days=options["expires_days"])
|
expires_at = timezone.now() + timedelta(days=options["expires_days"])
|
||||||
|
|
||||||
token = MCPToken.objects.create(
|
token, plaintext = MCPToken.objects.create_token(
|
||||||
user=user,
|
user=user,
|
||||||
name=options["name"],
|
name=options["name"],
|
||||||
allowed_tools=allowed_tools,
|
allowed_tools=allowed_tools,
|
||||||
@@ -73,5 +73,5 @@ class Command(BaseCommand):
|
|||||||
self.stdout.write(" Tools: (all)")
|
self.stdout.write(" Tools: (all)")
|
||||||
if expires_at:
|
if expires_at:
|
||||||
self.stdout.write(f" Expires: {expires_at.isoformat()}")
|
self.stdout.write(f" Expires: {expires_at.isoformat()}")
|
||||||
self.stdout.write(self.style.WARNING(" Token (shown once):"))
|
self.stdout.write(self.style.WARNING(" Token (shown once — store it now):"))
|
||||||
self.stdout.write(f" {token.token}")
|
self.stdout.write(f" {plaintext}")
|
||||||
|
|||||||
43
mnemosyne/mcp_server/migrations/0002_hash_token.py
Normal file
43
mnemosyne/mcp_server/migrations/0002_hash_token.py
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
"""Hash MCPToken values at rest.
|
||||||
|
|
||||||
|
Renames ``token`` → ``token_hash`` and rewrites any pre-existing plaintext
|
||||||
|
values into SHA-256 hex digests in-place. Forward-only: hashing is one-way,
|
||||||
|
so no reverse migration is provided.
|
||||||
|
|
||||||
|
Existing tokens issued before this migration keep working only because
|
||||||
|
``resolve_mcp_user`` hashes the incoming bearer before lookup; the original
|
||||||
|
plaintext the client holds still hashes to what we just wrote.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
def hash_existing_tokens(apps, schema_editor):
|
||||||
|
MCPToken = apps.get_model("mcp_server", "MCPToken")
|
||||||
|
for token in MCPToken.objects.all():
|
||||||
|
plaintext = token.token_hash # post-rename, still holds original plaintext
|
||||||
|
token.token_hash = hashlib.sha256(plaintext.encode("utf-8")).hexdigest()
|
||||||
|
token.save(update_fields=["token_hash"])
|
||||||
|
|
||||||
|
|
||||||
|
def noop_reverse(apps, schema_editor):
|
||||||
|
# Cannot reverse a hash. Leaving as no-op so the schema can be rolled
|
||||||
|
# back, but operators must understand any hashed rows are unrecoverable.
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
dependencies = [
|
||||||
|
("mcp_server", "0001_initial"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RenameField(
|
||||||
|
model_name="mcptoken",
|
||||||
|
old_name="token",
|
||||||
|
new_name="token_hash",
|
||||||
|
),
|
||||||
|
migrations.RunPython(hash_existing_tokens, noop_reverse),
|
||||||
|
]
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import hashlib
|
||||||
import secrets
|
import secrets
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
@@ -5,15 +6,44 @@ from django.db import models
|
|||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
|
|
||||||
|
def hash_token(plaintext: str) -> str:
|
||||||
|
"""SHA-256 hex digest of an MCP bearer token. 64 chars."""
|
||||||
|
return hashlib.sha256(plaintext.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
class MCPTokenManager(models.Manager):
|
||||||
|
def create_token(self, *, user, name, allowed_tools=None, expires_at=None):
|
||||||
|
"""Generate a new bearer token, store its hash, and return (instance, plaintext).
|
||||||
|
|
||||||
|
The plaintext is returned exactly once and is never persisted. Callers
|
||||||
|
must surface it to the human and rely on the user to copy it; after
|
||||||
|
this method returns, the plaintext is unrecoverable from the database.
|
||||||
|
"""
|
||||||
|
plaintext = secrets.token_urlsafe(48)
|
||||||
|
instance = self.create(
|
||||||
|
user=user,
|
||||||
|
name=name,
|
||||||
|
token_hash=hash_token(plaintext),
|
||||||
|
allowed_tools=list(allowed_tools or []),
|
||||||
|
expires_at=expires_at,
|
||||||
|
)
|
||||||
|
return instance, plaintext
|
||||||
|
|
||||||
|
|
||||||
class MCPToken(models.Model):
|
class MCPToken(models.Model):
|
||||||
"""Bearer token for authenticating MCP tool calls. See docs/Pattern_Django-MCP_V1-00.md."""
|
"""Bearer token for authenticating MCP tool calls.
|
||||||
|
|
||||||
|
Tokens are hashed at rest (SHA-256, 64-char hex). Plaintext exists only in
|
||||||
|
memory at creation time, on the wire to the client, and in the user's own
|
||||||
|
storage. A leaked database backup discloses no usable credentials.
|
||||||
|
"""
|
||||||
|
|
||||||
user = models.ForeignKey(
|
user = models.ForeignKey(
|
||||||
settings.AUTH_USER_MODEL,
|
settings.AUTH_USER_MODEL,
|
||||||
on_delete=models.CASCADE,
|
on_delete=models.CASCADE,
|
||||||
related_name="mcp_tokens",
|
related_name="mcp_tokens",
|
||||||
)
|
)
|
||||||
token = models.CharField(max_length=64, unique=True, db_index=True)
|
token_hash = models.CharField(max_length=64, unique=True, db_index=True)
|
||||||
name = models.CharField(max_length=100)
|
name = models.CharField(max_length=100)
|
||||||
is_active = models.BooleanField(default=True)
|
is_active = models.BooleanField(default=True)
|
||||||
expires_at = models.DateTimeField(null=True, blank=True)
|
expires_at = models.DateTimeField(null=True, blank=True)
|
||||||
@@ -22,17 +52,14 @@ class MCPToken(models.Model):
|
|||||||
created_at = models.DateTimeField(auto_now_add=True)
|
created_at = models.DateTimeField(auto_now_add=True)
|
||||||
updated_at = models.DateTimeField(auto_now=True)
|
updated_at = models.DateTimeField(auto_now=True)
|
||||||
|
|
||||||
|
objects = MCPTokenManager()
|
||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
ordering = ["-created_at"]
|
ordering = ["-created_at"]
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return f"{self.name} ({self.user})"
|
return f"{self.name} ({self.user})"
|
||||||
|
|
||||||
def save(self, **kwargs):
|
|
||||||
if not self.token:
|
|
||||||
self.token = secrets.token_urlsafe(48)
|
|
||||||
super().save(**kwargs)
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_valid(self) -> bool:
|
def is_valid(self) -> bool:
|
||||||
if not self.is_active:
|
if not self.is_active:
|
||||||
@@ -51,6 +78,9 @@ class MCPToken(models.Model):
|
|||||||
self.save(update_fields=["last_used_at"])
|
self.save(update_fields=["last_used_at"])
|
||||||
|
|
||||||
def get_masked_token(self) -> str:
|
def get_masked_token(self) -> str:
|
||||||
if len(self.token) > 8:
|
"""Token-id-style display for admin and dashboard.
|
||||||
return f"{'*' * (len(self.token) - 8)}{self.token[-8:]}"
|
|
||||||
return "*" * len(self.token)
|
Plaintext is unrecoverable, so we display the first 8 chars of the
|
||||||
|
hash prefixed with `mcp_…`. Stable per token, never reveals plaintext.
|
||||||
|
"""
|
||||||
|
return f"mcp_…{self.token_hash[:8]}"
|
||||||
|
|||||||
@@ -5,7 +5,11 @@ from __future__ import annotations
|
|||||||
from fastmcp import FastMCP
|
from fastmcp import FastMCP
|
||||||
|
|
||||||
from .auth import MCPAuthMiddleware
|
from .auth import MCPAuthMiddleware
|
||||||
from .tools import register_discovery_tools, register_search_tools
|
from .tools import (
|
||||||
|
register_discovery_tools,
|
||||||
|
register_health_tools,
|
||||||
|
register_search_tools,
|
||||||
|
)
|
||||||
|
|
||||||
INSTRUCTIONS = """\
|
INSTRUCTIONS = """\
|
||||||
Mnemosyne is a content-type-aware, multimodal knowledge base. It indexes
|
Mnemosyne is a content-type-aware, multimodal knowledge base. It indexes
|
||||||
@@ -23,6 +27,8 @@ shapes how content is chunked, embedded, and re-ranked:
|
|||||||
- film — Scripts, synopses, stills.
|
- film — Scripts, synopses, stills.
|
||||||
- art — Catalogs, descriptions, artwork itself.
|
- art — Catalogs, descriptions, artwork itself.
|
||||||
- journal — Personal entries; temporal/reflective.
|
- journal — Personal entries; temporal/reflective.
|
||||||
|
- business — Proposals, marketing, sales, strategy. Commercial context.
|
||||||
|
- finance — Statements, tax, market commentary. Quote figures exactly.
|
||||||
|
|
||||||
Tools:
|
Tools:
|
||||||
- search Hybrid retrieval. Filter by library_uid, library_type,
|
- search Hybrid retrieval. Filter by library_uid, library_type,
|
||||||
@@ -32,6 +38,7 @@ Tools:
|
|||||||
- list_libraries Discover libraries (and their library_type).
|
- list_libraries Discover libraries (and their library_type).
|
||||||
- list_collections Discover collections, optionally per library.
|
- list_collections Discover collections, optionally per library.
|
||||||
- list_items Discover indexed items (documents).
|
- list_items Discover indexed items (documents).
|
||||||
|
- get_health Health check (used by Pallas/Daedalus pollers).
|
||||||
|
|
||||||
Workflow: list_libraries → search(query, library_type=...) → get_chunk(chunk_uid)
|
Workflow: list_libraries → search(query, library_type=...) → get_chunk(chunk_uid)
|
||||||
when the preview isn't enough. The calling LLM is responsible for synthesis
|
when the preview isn't enough. The calling LLM is responsible for synthesis
|
||||||
@@ -47,6 +54,7 @@ def build_server() -> FastMCP:
|
|||||||
mcp.add_middleware(MCPAuthMiddleware())
|
mcp.add_middleware(MCPAuthMiddleware())
|
||||||
register_search_tools(mcp)
|
register_search_tools(mcp)
|
||||||
register_discovery_tools(mcp)
|
register_discovery_tools(mcp)
|
||||||
|
register_health_tools(mcp)
|
||||||
return mcp
|
return mcp
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
63
mnemosyne/mcp_server/templates/mcp_server/tokens/create.html
Normal file
63
mnemosyne/mcp_server/templates/mcp_server/tokens/create.html
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
{% extends "themis/base.html" %}
|
||||||
|
|
||||||
|
{% block title %}Generate MCP Token — {{ themis_app_name }}{% endblock %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<div class="max-w-2xl mx-auto">
|
||||||
|
<h1 class="text-2xl font-bold mb-6">Generate MCP Token</h1>
|
||||||
|
|
||||||
|
<div class="alert alert-info mb-6">
|
||||||
|
<span>The token will be displayed once after creation. Save it before leaving the page — it cannot be recovered.</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form method="post" action="{% url 'mcp_server:mcp-token-create' %}">
|
||||||
|
{% csrf_token %}
|
||||||
|
|
||||||
|
<div class="card bg-base-200 mb-6">
|
||||||
|
<div class="card-body">
|
||||||
|
<div class="form-control">
|
||||||
|
<label class="label" for="id_name">
|
||||||
|
<span class="label-text">Name</span>
|
||||||
|
</label>
|
||||||
|
{{ form.name }}
|
||||||
|
<label class="label">
|
||||||
|
<span class="label-text-alt opacity-60">A friendly label so you can identify this token later (e.g. “Claude Desktop”).</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<div class="form-control mt-4">
|
||||||
|
<label class="label" for="id_expires_at">
|
||||||
|
<span class="label-text">Expires at (optional)</span>
|
||||||
|
</label>
|
||||||
|
{{ form.expires_at }}
|
||||||
|
<label class="label">
|
||||||
|
<span class="label-text-alt opacity-60">{{ form.expires_at.help_text }}</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card bg-base-200 mb-6">
|
||||||
|
<div class="card-body">
|
||||||
|
<h2 class="card-title text-lg">Allowed Tools</h2>
|
||||||
|
<p class="text-sm opacity-60 mb-2">{{ form.allowed_tools.help_text }}</p>
|
||||||
|
<div class="space-y-1">
|
||||||
|
{% for choice in form.allowed_tools %}
|
||||||
|
<label class="label cursor-pointer justify-start gap-3">
|
||||||
|
{{ choice.tag }}
|
||||||
|
<span class="label-text font-mono">{{ choice.choice_label }}</span>
|
||||||
|
</label>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
{% if form.allowed_tools.errors %}
|
||||||
|
<div class="text-error text-sm mt-2">{{ form.allowed_tools.errors }}</div>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex justify-between">
|
||||||
|
<a href="{% url 'mcp_server:mcp-token-list' %}" class="btn btn-ghost">Cancel</a>
|
||||||
|
<button type="submit" class="btn btn-primary">Generate Token</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
{% extends "themis/base.html" %}
|
||||||
|
|
||||||
|
{% block title %}MCP Token Created — {{ themis_app_name }}{% endblock %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<div class="max-w-2xl mx-auto">
|
||||||
|
<h1 class="text-2xl font-bold mb-2">MCP Token Created</h1>
|
||||||
|
<p class="opacity-60 mb-6">{{ token.name }}</p>
|
||||||
|
|
||||||
|
<div class="alert alert-warning mb-6">
|
||||||
|
<span><strong>Save this token now.</strong> Once you leave this page it cannot be retrieved — only the hash is stored. If lost, you will need to generate a new one.</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card bg-base-200 mb-6">
|
||||||
|
<div class="card-body">
|
||||||
|
<h2 class="card-title text-lg mb-2">Token</h2>
|
||||||
|
<div class="font-mono break-all bg-base-300 p-4 rounded select-all" id="mcp-plaintext">{{ plaintext }}</div>
|
||||||
|
<div class="card-actions justify-end mt-3">
|
||||||
|
<button type="button" id="mcp-copy-btn" class="btn btn-sm btn-primary"
|
||||||
|
onclick="(()=>{const t=document.getElementById('mcp-plaintext').textContent;navigator.clipboard.writeText(t).then(()=>{const b=document.getElementById('mcp-copy-btn');b.textContent='Copied!';setTimeout(()=>b.textContent='Copy to clipboard',2000);});})()">
|
||||||
|
Copy to clipboard
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card bg-base-200 mb-6">
|
||||||
|
<div class="card-body">
|
||||||
|
<h2 class="card-title text-lg">Use it</h2>
|
||||||
|
<p class="text-sm opacity-80 mb-3">
|
||||||
|
Add the token to your MCP client config as a Bearer credential. Example for Claude Desktop:
|
||||||
|
</p>
|
||||||
|
<pre class="bg-base-300 p-4 rounded text-xs overflow-x-auto"><code>{
|
||||||
|
"mcpServers": {
|
||||||
|
"mnemosyne": {
|
||||||
|
"url": "http://localhost:8001/mcp/",
|
||||||
|
"headers": {
|
||||||
|
"Authorization": "Bearer <paste token here>"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}</code></pre>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex justify-end">
|
||||||
|
<a href="{% url 'mcp_server:mcp-token-list' %}" class="btn btn-primary">I’ve saved it — go to token list</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
81
mnemosyne/mcp_server/templates/mcp_server/tokens/detail.html
Normal file
81
mnemosyne/mcp_server/templates/mcp_server/tokens/detail.html
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
{% extends "themis/base.html" %}
|
||||||
|
{% load humanize %}
|
||||||
|
|
||||||
|
{% block title %}{{ token.name }} — {{ themis_app_name }}{% endblock %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<div class="max-w-2xl mx-auto">
|
||||||
|
<div class="flex items-center justify-between mb-6">
|
||||||
|
<h1 class="text-2xl font-bold">{{ token.name }}</h1>
|
||||||
|
<div class="flex gap-2">
|
||||||
|
<a href="{% url 'mcp_server:mcp-token-edit' pk=token.pk %}" class="btn btn-ghost btn-sm">Edit</a>
|
||||||
|
{% if token.is_active %}
|
||||||
|
<form method="post" action="{% url 'mcp_server:mcp-token-revoke' pk=token.pk %}"
|
||||||
|
onsubmit="return confirm('Revoke this token? It will no longer authenticate MCP requests.');">
|
||||||
|
{% csrf_token %}
|
||||||
|
<button type="submit" class="btn btn-warning btn-sm btn-outline">Revoke</button>
|
||||||
|
</form>
|
||||||
|
{% endif %}
|
||||||
|
<form method="post" action="{% url 'mcp_server:mcp-token-delete' pk=token.pk %}"
|
||||||
|
onsubmit="return confirm('Delete this token permanently? This removes the audit trail.');">
|
||||||
|
{% csrf_token %}
|
||||||
|
<button type="submit" class="btn btn-error btn-sm btn-outline">Delete</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card bg-base-200 mb-6">
|
||||||
|
<div class="card-body">
|
||||||
|
<div class="grid grid-cols-2 gap-y-3">
|
||||||
|
<div class="opacity-60">Token ID</div>
|
||||||
|
<div class="font-mono">{{ token.get_masked_token }}</div>
|
||||||
|
|
||||||
|
<div class="opacity-60">Status</div>
|
||||||
|
<div>
|
||||||
|
{% if token.is_valid %}
|
||||||
|
<span class="badge badge-success badge-sm">Active</span>
|
||||||
|
{% elif not token.is_active %}
|
||||||
|
<span class="badge badge-error badge-sm">Revoked</span>
|
||||||
|
{% else %}
|
||||||
|
<span class="badge badge-error badge-sm">Expired</span>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{% if token.expires_at %}
|
||||||
|
<div class="opacity-60">Expires</div>
|
||||||
|
<div>{{ token.expires_at }}</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
<div class="opacity-60">Last Used</div>
|
||||||
|
<div>
|
||||||
|
{% if token.last_used_at %}
|
||||||
|
{{ token.last_used_at }} <span class="opacity-60">({{ token.last_used_at|naturaltime }})</span>
|
||||||
|
{% else %}
|
||||||
|
<span class="opacity-60">Never</span>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="opacity-60">Created</div>
|
||||||
|
<div>{{ token.created_at }}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card bg-base-200 mb-6">
|
||||||
|
<div class="card-body">
|
||||||
|
<h2 class="card-title text-lg">Allowed Tools</h2>
|
||||||
|
{% if token.allowed_tools %}
|
||||||
|
<div class="flex flex-wrap gap-2">
|
||||||
|
{% for tool in token.allowed_tools %}
|
||||||
|
<span class="badge badge-outline font-mono">{{ tool }}</span>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
{% else %}
|
||||||
|
<p class="opacity-60">All tools permitted.</p>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<a href="{% url 'mcp_server:mcp-token-list' %}" class="btn btn-ghost btn-sm">← Back to Tokens</a>
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
60
mnemosyne/mcp_server/templates/mcp_server/tokens/edit.html
Normal file
60
mnemosyne/mcp_server/templates/mcp_server/tokens/edit.html
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
{% extends "themis/base.html" %}
|
||||||
|
|
||||||
|
{% block title %}Edit {{ token.name }} — {{ themis_app_name }}{% endblock %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<div class="max-w-2xl mx-auto">
|
||||||
|
<h1 class="text-2xl font-bold mb-6">Edit Token: {{ token.name }}</h1>
|
||||||
|
|
||||||
|
<div class="alert alert-info mb-6">
|
||||||
|
<span>You can edit metadata below. The token value itself cannot be changed — generate a new token if needed.</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form method="post" action="{% url 'mcp_server:mcp-token-edit' pk=token.pk %}">
|
||||||
|
{% csrf_token %}
|
||||||
|
|
||||||
|
<div class="card bg-base-200 mb-6">
|
||||||
|
<div class="card-body">
|
||||||
|
<div class="form-control">
|
||||||
|
<label class="label" for="id_name">
|
||||||
|
<span class="label-text">Name</span>
|
||||||
|
</label>
|
||||||
|
{{ form.name }}
|
||||||
|
</div>
|
||||||
|
<div class="form-control mt-4">
|
||||||
|
<label class="label cursor-pointer justify-start gap-3">
|
||||||
|
{{ form.is_active }}
|
||||||
|
<span class="label-text">Active</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<div class="form-control mt-4">
|
||||||
|
<label class="label" for="id_expires_at">
|
||||||
|
<span class="label-text">Expires at (optional)</span>
|
||||||
|
</label>
|
||||||
|
{{ form.expires_at }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card bg-base-200 mb-6">
|
||||||
|
<div class="card-body">
|
||||||
|
<h2 class="card-title text-lg">Allowed Tools</h2>
|
||||||
|
<p class="text-sm opacity-60 mb-2">{{ form.allowed_tools.help_text }}</p>
|
||||||
|
<div class="space-y-1">
|
||||||
|
{% for choice in form.allowed_tools %}
|
||||||
|
<label class="label cursor-pointer justify-start gap-3">
|
||||||
|
{{ choice.tag }}
|
||||||
|
<span class="label-text font-mono">{{ choice.choice_label }}</span>
|
||||||
|
</label>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex justify-between">
|
||||||
|
<a href="{% url 'mcp_server:mcp-token-detail' pk=token.pk %}" class="btn btn-ghost">Cancel</a>
|
||||||
|
<button type="submit" class="btn btn-primary">Save Changes</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
61
mnemosyne/mcp_server/templates/mcp_server/tokens/list.html
Normal file
61
mnemosyne/mcp_server/templates/mcp_server/tokens/list.html
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
{% extends "themis/base.html" %}
|
||||||
|
{% load humanize %}
|
||||||
|
|
||||||
|
{% block title %}MCP Tokens — {{ themis_app_name }}{% endblock %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<div class="max-w-3xl mx-auto">
|
||||||
|
<div class="flex items-center justify-between mb-6">
|
||||||
|
<h1 class="text-2xl font-bold">MCP Tokens</h1>
|
||||||
|
<a href="{% url 'mcp_server:mcp-token-create' %}" class="btn btn-primary btn-sm">Generate Token</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p class="text-sm opacity-60 mb-4">
|
||||||
|
Bearer tokens used by MCP clients (Claude Desktop, Cursor, etc.) to call Mnemosyne. Stored hashed at rest — the plaintext is shown only once at creation.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
{% if tokens %}
|
||||||
|
<div class="space-y-3">
|
||||||
|
{% for token in tokens %}
|
||||||
|
<div class="card bg-base-200">
|
||||||
|
<div class="card-body p-4">
|
||||||
|
<div class="flex items-center justify-between">
|
||||||
|
<div>
|
||||||
|
<a href="{% url 'mcp_server:mcp-token-detail' pk=token.pk %}"
|
||||||
|
class="font-semibold link link-hover">
|
||||||
|
{{ token.name }}
|
||||||
|
</a>
|
||||||
|
<div class="flex items-center gap-2 mt-1 flex-wrap">
|
||||||
|
<span class="text-sm opacity-60 font-mono">{{ token.get_masked_token }}</span>
|
||||||
|
{% if not token.is_valid %}
|
||||||
|
<span class="badge badge-sm badge-error">
|
||||||
|
{% if not token.is_active %}Revoked{% else %}Expired{% endif %}
|
||||||
|
</span>
|
||||||
|
{% else %}
|
||||||
|
<span class="badge badge-sm badge-success">Active</span>
|
||||||
|
{% endif %}
|
||||||
|
<span class="text-xs opacity-60">
|
||||||
|
Last used:
|
||||||
|
{% if token.last_used_at %}{{ token.last_used_at|naturaltime }}{% else %}never{% endif %}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="flex gap-1">
|
||||||
|
<a href="{% url 'mcp_server:mcp-token-edit' pk=token.pk %}"
|
||||||
|
class="btn btn-ghost btn-xs">Edit</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
{% else %}
|
||||||
|
<div class="card bg-base-200">
|
||||||
|
<div class="card-body items-center text-center py-12">
|
||||||
|
<p class="opacity-60 mb-4">No MCP tokens yet.</p>
|
||||||
|
<a href="{% url 'mcp_server:mcp-token-create' %}" class="btn btn-primary btn-sm">Generate Your First Token</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
@@ -17,16 +17,18 @@ class ResolveMCPUserTest(TestCase):
|
|||||||
self.user = User.objects.create_user(
|
self.user = User.objects.create_user(
|
||||||
username="bob", email="bob@example.com", password="pw"
|
username="bob", email="bob@example.com", password="pw"
|
||||||
)
|
)
|
||||||
self.token = MCPToken.objects.create(user=self.user, name="t")
|
self.token, self.plaintext = MCPToken.objects.create_token(
|
||||||
|
user=self.user, name="t"
|
||||||
|
)
|
||||||
|
|
||||||
def test_resolves_valid_token(self):
|
def test_resolves_valid_token(self):
|
||||||
user, token = resolve_mcp_user(self.token.token)
|
user, token = resolve_mcp_user(self.plaintext)
|
||||||
self.assertEqual(user.pk, self.user.pk)
|
self.assertEqual(user.pk, self.user.pk)
|
||||||
self.assertEqual(token.pk, self.token.pk)
|
self.assertEqual(token.pk, self.token.pk)
|
||||||
|
|
||||||
def test_records_usage(self):
|
def test_records_usage(self):
|
||||||
self.assertIsNone(self.token.last_used_at)
|
self.assertIsNone(self.token.last_used_at)
|
||||||
resolve_mcp_user(self.token.token)
|
resolve_mcp_user(self.plaintext)
|
||||||
self.token.refresh_from_db()
|
self.token.refresh_from_db()
|
||||||
self.assertIsNotNone(self.token.last_used_at)
|
self.assertIsNotNone(self.token.last_used_at)
|
||||||
|
|
||||||
@@ -38,16 +40,31 @@ class ResolveMCPUserTest(TestCase):
|
|||||||
self.token.is_active = False
|
self.token.is_active = False
|
||||||
self.token.save()
|
self.token.save()
|
||||||
with self.assertRaises(MCPAuthError):
|
with self.assertRaises(MCPAuthError):
|
||||||
resolve_mcp_user(self.token.token)
|
resolve_mcp_user(self.plaintext)
|
||||||
|
|
||||||
def test_expired_token_raises(self):
|
def test_expired_token_raises(self):
|
||||||
self.token.expires_at = timezone.now() - timedelta(hours=1)
|
self.token.expires_at = timezone.now() - timedelta(hours=1)
|
||||||
self.token.save()
|
self.token.save()
|
||||||
with self.assertRaises(MCPAuthError):
|
with self.assertRaises(MCPAuthError):
|
||||||
resolve_mcp_user(self.token.token)
|
resolve_mcp_user(self.plaintext)
|
||||||
|
|
||||||
def test_disabled_user_raises(self):
|
def test_disabled_user_raises(self):
|
||||||
self.user.is_active = False
|
self.user.is_active = False
|
||||||
self.user.save()
|
self.user.save()
|
||||||
with self.assertRaises(MCPAuthError):
|
with self.assertRaises(MCPAuthError):
|
||||||
resolve_mcp_user(self.token.token)
|
resolve_mcp_user(self.plaintext)
|
||||||
|
|
||||||
|
def test_plaintext_not_in_db(self):
|
||||||
|
# Defense in depth: scan every column for the plaintext value.
|
||||||
|
from django.db import connection
|
||||||
|
|
||||||
|
plaintext = self.plaintext
|
||||||
|
with connection.cursor() as cur:
|
||||||
|
cur.execute("SELECT * FROM mcp_server_mcptoken")
|
||||||
|
rows = cur.fetchall()
|
||||||
|
for row in rows:
|
||||||
|
for value in row:
|
||||||
|
self.assertNotEqual(
|
||||||
|
value, plaintext,
|
||||||
|
f"Plaintext token leaked into the database: {value!r}",
|
||||||
|
)
|
||||||
|
|||||||
58
mnemosyne/mcp_server/tests/test_forms.py
Normal file
58
mnemosyne/mcp_server/tests/test_forms.py
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
"""Form tests for the MCP token dashboard."""
|
||||||
|
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.test import TestCase
|
||||||
|
|
||||||
|
from mcp_server.forms import MCPTokenCreateForm, MCPTokenEditForm
|
||||||
|
from mcp_server.models import MCPToken
|
||||||
|
|
||||||
|
User = get_user_model()
|
||||||
|
|
||||||
|
|
||||||
|
class CreateFormTest(TestCase):
|
||||||
|
def test_required_fields(self):
|
||||||
|
form = MCPTokenCreateForm(data={})
|
||||||
|
self.assertFalse(form.is_valid())
|
||||||
|
self.assertIn("name", form.errors)
|
||||||
|
|
||||||
|
def test_name_only_is_valid(self):
|
||||||
|
form = MCPTokenCreateForm(data={"name": "Test"})
|
||||||
|
self.assertTrue(form.is_valid(), form.errors)
|
||||||
|
|
||||||
|
def test_tool_choices_match_registered_tools(self):
|
||||||
|
form = MCPTokenCreateForm()
|
||||||
|
choices = {value for value, _ in form.fields["allowed_tools"].choices}
|
||||||
|
# These five must always be present per the FastMCP server.
|
||||||
|
for expected in {"search", "get_chunk", "list_libraries", "list_collections", "list_items"}:
|
||||||
|
self.assertIn(expected, choices)
|
||||||
|
|
||||||
|
|
||||||
|
class EditFormTest(TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(username="alice", password="pw")
|
||||||
|
self.token, _ = MCPToken.objects.create_token(
|
||||||
|
user=self.user, name="t", allowed_tools=["search"]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_initial_allowed_tools_populated(self):
|
||||||
|
form = MCPTokenEditForm(instance=self.token)
|
||||||
|
self.assertEqual(form.fields["allowed_tools"].initial, ["search"])
|
||||||
|
|
||||||
|
def test_save_updates_metadata(self):
|
||||||
|
form = MCPTokenEditForm(
|
||||||
|
data={
|
||||||
|
"name": "Renamed",
|
||||||
|
"is_active": False,
|
||||||
|
"expires_at": "",
|
||||||
|
"allowed_tools": ["search", "get_chunk"],
|
||||||
|
},
|
||||||
|
instance=self.token,
|
||||||
|
)
|
||||||
|
self.assertTrue(form.is_valid(), form.errors)
|
||||||
|
instance = form.save(commit=False)
|
||||||
|
instance.allowed_tools = form.cleaned_data["allowed_tools"]
|
||||||
|
instance.save()
|
||||||
|
self.token.refresh_from_db()
|
||||||
|
self.assertEqual(self.token.name, "Renamed")
|
||||||
|
self.assertFalse(self.token.is_active)
|
||||||
|
self.assertEqual(self.token.allowed_tools, ["search", "get_chunk"])
|
||||||
@@ -7,7 +7,14 @@ from django.test import TestCase
|
|||||||
from mcp_server.server import mcp
|
from mcp_server.server import mcp
|
||||||
|
|
||||||
|
|
||||||
EXPECTED_TOOLS = {"search", "get_chunk", "list_libraries", "list_collections", "list_items"}
|
EXPECTED_TOOLS = {
|
||||||
|
"search",
|
||||||
|
"get_chunk",
|
||||||
|
"list_libraries",
|
||||||
|
"list_collections",
|
||||||
|
"list_items",
|
||||||
|
"get_health",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class ServerRegistrationTest(TestCase):
|
class ServerRegistrationTest(TestCase):
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from django.contrib.auth import get_user_model
|
|||||||
from django.test import TestCase
|
from django.test import TestCase
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
from mcp_server.models import MCPToken
|
from mcp_server.models import MCPToken, hash_token
|
||||||
|
|
||||||
User = get_user_model()
|
User = get_user_model()
|
||||||
|
|
||||||
@@ -17,21 +17,33 @@ class MCPTokenModelTest(TestCase):
|
|||||||
username="alice", email="alice@example.com", password="pw"
|
username="alice", email="alice@example.com", password="pw"
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_token_auto_generated(self):
|
def test_create_token_returns_plaintext_and_stores_hash(self):
|
||||||
token = MCPToken.objects.create(user=self.user, name="t")
|
token, plaintext = MCPToken.objects.create_token(user=self.user, name="t")
|
||||||
self.assertTrue(token.token)
|
self.assertTrue(plaintext)
|
||||||
self.assertGreater(len(token.token), 20)
|
self.assertGreater(len(plaintext), 20)
|
||||||
|
# Database stores hash, not plaintext
|
||||||
|
self.assertEqual(len(token.token_hash), 64)
|
||||||
|
self.assertNotEqual(token.token_hash, plaintext)
|
||||||
|
self.assertEqual(token.token_hash, hash_token(plaintext))
|
||||||
|
|
||||||
|
def test_token_hash_never_equals_plaintext(self):
|
||||||
|
# Regression guard: if anyone ever wires plaintext back into token_hash,
|
||||||
|
# this fails.
|
||||||
|
token, plaintext = MCPToken.objects.create_token(user=self.user, name="t")
|
||||||
|
self.assertNotIn(plaintext, token.token_hash)
|
||||||
|
|
||||||
def test_active_token_is_valid(self):
|
def test_active_token_is_valid(self):
|
||||||
token = MCPToken.objects.create(user=self.user, name="t")
|
token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||||
self.assertTrue(token.is_valid)
|
self.assertTrue(token.is_valid)
|
||||||
|
|
||||||
def test_inactive_token_not_valid(self):
|
def test_inactive_token_not_valid(self):
|
||||||
token = MCPToken.objects.create(user=self.user, name="t", is_active=False)
|
token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||||
|
token.is_active = False
|
||||||
|
token.save()
|
||||||
self.assertFalse(token.is_valid)
|
self.assertFalse(token.is_valid)
|
||||||
|
|
||||||
def test_expired_token_not_valid(self):
|
def test_expired_token_not_valid(self):
|
||||||
token = MCPToken.objects.create(
|
token, _ = MCPToken.objects.create_token(
|
||||||
user=self.user,
|
user=self.user,
|
||||||
name="t",
|
name="t",
|
||||||
expires_at=timezone.now() - timedelta(hours=1),
|
expires_at=timezone.now() - timedelta(hours=1),
|
||||||
@@ -39,25 +51,27 @@ class MCPTokenModelTest(TestCase):
|
|||||||
self.assertFalse(token.is_valid)
|
self.assertFalse(token.is_valid)
|
||||||
|
|
||||||
def test_unrestricted_permits_all(self):
|
def test_unrestricted_permits_all(self):
|
||||||
token = MCPToken.objects.create(user=self.user, name="t")
|
token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||||
self.assertTrue(token.can_use_tool("anything"))
|
self.assertTrue(token.can_use_tool("anything"))
|
||||||
|
|
||||||
def test_tool_whitelist(self):
|
def test_tool_whitelist(self):
|
||||||
token = MCPToken.objects.create(
|
token, _ = MCPToken.objects.create_token(
|
||||||
user=self.user, name="t", allowed_tools=["search"]
|
user=self.user, name="t", allowed_tools=["search"]
|
||||||
)
|
)
|
||||||
self.assertTrue(token.can_use_tool("search"))
|
self.assertTrue(token.can_use_tool("search"))
|
||||||
self.assertFalse(token.can_use_tool("get_chunk"))
|
self.assertFalse(token.can_use_tool("get_chunk"))
|
||||||
|
|
||||||
def test_record_usage(self):
|
def test_record_usage(self):
|
||||||
token = MCPToken.objects.create(user=self.user, name="t")
|
token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||||
self.assertIsNone(token.last_used_at)
|
self.assertIsNone(token.last_used_at)
|
||||||
token.record_usage()
|
token.record_usage()
|
||||||
token.refresh_from_db()
|
token.refresh_from_db()
|
||||||
self.assertIsNotNone(token.last_used_at)
|
self.assertIsNotNone(token.last_used_at)
|
||||||
|
|
||||||
def test_masked_token(self):
|
def test_masked_token_is_hash_prefix(self):
|
||||||
token = MCPToken.objects.create(user=self.user, name="t")
|
token, plaintext = MCPToken.objects.create_token(user=self.user, name="t")
|
||||||
masked = token.get_masked_token()
|
masked = token.get_masked_token()
|
||||||
self.assertTrue(masked.endswith(token.token[-8:]))
|
self.assertTrue(masked.startswith("mcp_…"))
|
||||||
self.assertIn("*", masked)
|
self.assertIn(token.token_hash[:8], masked)
|
||||||
|
# Plaintext must never leak through the masked display
|
||||||
|
self.assertNotIn(plaintext, masked)
|
||||||
|
|||||||
177
mnemosyne/mcp_server/tests/test_views.py
Normal file
177
mnemosyne/mcp_server/tests/test_views.py
Normal file
@@ -0,0 +1,177 @@
|
|||||||
|
"""View tests for the MCP token self-service dashboard."""
|
||||||
|
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.test import TestCase
|
||||||
|
from django.urls import reverse
|
||||||
|
|
||||||
|
from mcp_server.models import MCPToken
|
||||||
|
|
||||||
|
User = get_user_model()
|
||||||
|
|
||||||
|
|
||||||
|
class TokenListViewTest(TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(
|
||||||
|
username="alice", email="alice@example.com", password="pw"
|
||||||
|
)
|
||||||
|
self.url = reverse("mcp_server:mcp-token-list")
|
||||||
|
|
||||||
|
def test_login_required(self):
|
||||||
|
resp = self.client.get(self.url)
|
||||||
|
self.assertEqual(resp.status_code, 302)
|
||||||
|
self.assertIn("/login/", resp.url)
|
||||||
|
|
||||||
|
def test_list_shows_only_own_tokens(self):
|
||||||
|
other = User.objects.create_user(username="bob", password="pw")
|
||||||
|
MCPToken.objects.create_token(user=self.user, name="mine")
|
||||||
|
MCPToken.objects.create_token(user=other, name="theirs")
|
||||||
|
self.client.force_login(self.user)
|
||||||
|
resp = self.client.get(self.url)
|
||||||
|
self.assertContains(resp, "mine")
|
||||||
|
self.assertNotContains(resp, "theirs")
|
||||||
|
|
||||||
|
def test_empty_state(self):
|
||||||
|
self.client.force_login(self.user)
|
||||||
|
resp = self.client.get(self.url)
|
||||||
|
self.assertContains(resp, "No MCP tokens yet.")
|
||||||
|
|
||||||
|
|
||||||
|
class TokenCreateViewTest(TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(username="alice", password="pw")
|
||||||
|
self.client.force_login(self.user)
|
||||||
|
self.url = reverse("mcp_server:mcp-token-create")
|
||||||
|
|
||||||
|
def test_get_renders_form(self):
|
||||||
|
resp = self.client.get(self.url)
|
||||||
|
self.assertEqual(resp.status_code, 200)
|
||||||
|
self.assertContains(resp, "Generate MCP Token")
|
||||||
|
|
||||||
|
def test_post_creates_token_and_shows_plaintext_once(self):
|
||||||
|
resp = self.client.post(self.url, {"name": "Claude Desktop"})
|
||||||
|
self.assertEqual(resp.status_code, 200)
|
||||||
|
self.assertContains(resp, "Save this token now")
|
||||||
|
# Pull the created row, verify the response contained a plaintext that
|
||||||
|
# is NOT what we stored.
|
||||||
|
token = MCPToken.objects.get(user=self.user, name="Claude Desktop")
|
||||||
|
self.assertNotContains(resp, token.token_hash) # hash is not what we display
|
||||||
|
# And the detail page never renders the plaintext.
|
||||||
|
body = resp.content.decode()
|
||||||
|
# Find the plaintext from the response: the only long alphanumeric
|
||||||
|
# block inside the #mcp-plaintext div.
|
||||||
|
import re
|
||||||
|
|
||||||
|
m = re.search(r'id="mcp-plaintext">([A-Za-z0-9_\-]+)<', body)
|
||||||
|
self.assertIsNotNone(m, "plaintext block not found in response")
|
||||||
|
plaintext = m.group(1)
|
||||||
|
# Sanity: round-tripping the plaintext through hash_token reproduces
|
||||||
|
# what's stored.
|
||||||
|
from mcp_server.models import hash_token
|
||||||
|
|
||||||
|
self.assertEqual(hash_token(plaintext), token.token_hash)
|
||||||
|
# Detail page must NOT contain the plaintext.
|
||||||
|
detail_resp = self.client.get(
|
||||||
|
reverse("mcp_server:mcp-token-detail", args=[token.pk])
|
||||||
|
)
|
||||||
|
self.assertNotContains(detail_resp, plaintext)
|
||||||
|
|
||||||
|
def test_post_invalid_renders_form_again(self):
|
||||||
|
resp = self.client.post(self.url, {"name": ""})
|
||||||
|
self.assertEqual(resp.status_code, 200)
|
||||||
|
self.assertContains(resp, "Generate MCP Token")
|
||||||
|
self.assertEqual(MCPToken.objects.count(), 0)
|
||||||
|
|
||||||
|
|
||||||
|
class TokenDetailViewTest(TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(username="alice", password="pw")
|
||||||
|
self.client.force_login(self.user)
|
||||||
|
self.token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||||
|
|
||||||
|
def test_renders_token(self):
|
||||||
|
resp = self.client.get(
|
||||||
|
reverse("mcp_server:mcp-token-detail", args=[self.token.pk])
|
||||||
|
)
|
||||||
|
self.assertContains(resp, self.token.name)
|
||||||
|
self.assertContains(resp, self.token.get_masked_token())
|
||||||
|
|
||||||
|
def test_cannot_view_other_users_token(self):
|
||||||
|
other = User.objects.create_user(username="bob", password="pw")
|
||||||
|
other_token, _ = MCPToken.objects.create_token(user=other, name="theirs")
|
||||||
|
resp = self.client.get(
|
||||||
|
reverse("mcp_server:mcp-token-detail", args=[other_token.pk])
|
||||||
|
)
|
||||||
|
self.assertEqual(resp.status_code, 404)
|
||||||
|
|
||||||
|
|
||||||
|
class TokenEditViewTest(TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(username="alice", password="pw")
|
||||||
|
self.client.force_login(self.user)
|
||||||
|
self.token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||||
|
|
||||||
|
def test_post_updates_metadata(self):
|
||||||
|
resp = self.client.post(
|
||||||
|
reverse("mcp_server:mcp-token-edit", args=[self.token.pk]),
|
||||||
|
{
|
||||||
|
"name": "Renamed",
|
||||||
|
"is_active": "on",
|
||||||
|
"expires_at": "",
|
||||||
|
"allowed_tools": ["search"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.assertEqual(resp.status_code, 302)
|
||||||
|
self.token.refresh_from_db()
|
||||||
|
self.assertEqual(self.token.name, "Renamed")
|
||||||
|
self.assertEqual(self.token.allowed_tools, ["search"])
|
||||||
|
|
||||||
|
def test_cannot_edit_other_users_token(self):
|
||||||
|
other = User.objects.create_user(username="bob", password="pw")
|
||||||
|
other_token, _ = MCPToken.objects.create_token(user=other, name="theirs")
|
||||||
|
resp = self.client.post(
|
||||||
|
reverse("mcp_server:mcp-token-edit", args=[other_token.pk]),
|
||||||
|
{"name": "hacked"},
|
||||||
|
)
|
||||||
|
self.assertEqual(resp.status_code, 404)
|
||||||
|
|
||||||
|
|
||||||
|
class TokenRevokeViewTest(TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(username="alice", password="pw")
|
||||||
|
self.client.force_login(self.user)
|
||||||
|
self.token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||||
|
|
||||||
|
def test_revoke_sets_inactive_keeps_row(self):
|
||||||
|
url = reverse("mcp_server:mcp-token-revoke", args=[self.token.pk])
|
||||||
|
resp = self.client.post(url)
|
||||||
|
self.assertEqual(resp.status_code, 302)
|
||||||
|
self.token.refresh_from_db()
|
||||||
|
self.assertFalse(self.token.is_active)
|
||||||
|
# Row still exists for audit trail.
|
||||||
|
self.assertTrue(MCPToken.objects.filter(pk=self.token.pk).exists())
|
||||||
|
|
||||||
|
def test_get_not_allowed(self):
|
||||||
|
url = reverse("mcp_server:mcp-token-revoke", args=[self.token.pk])
|
||||||
|
resp = self.client.get(url)
|
||||||
|
self.assertEqual(resp.status_code, 405)
|
||||||
|
|
||||||
|
|
||||||
|
class TokenDeleteViewTest(TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(username="alice", password="pw")
|
||||||
|
self.client.force_login(self.user)
|
||||||
|
self.token, _ = MCPToken.objects.create_token(user=self.user, name="t")
|
||||||
|
|
||||||
|
def test_delete_removes_row(self):
|
||||||
|
url = reverse("mcp_server:mcp-token-delete", args=[self.token.pk])
|
||||||
|
resp = self.client.post(url)
|
||||||
|
self.assertEqual(resp.status_code, 302)
|
||||||
|
self.assertFalse(MCPToken.objects.filter(pk=self.token.pk).exists())
|
||||||
|
|
||||||
|
def test_cannot_delete_other_users_token(self):
|
||||||
|
other = User.objects.create_user(username="bob", password="pw")
|
||||||
|
other_token, _ = MCPToken.objects.create_token(user=other, name="theirs")
|
||||||
|
url = reverse("mcp_server:mcp-token-delete", args=[other_token.pk])
|
||||||
|
resp = self.client.post(url)
|
||||||
|
self.assertEqual(resp.status_code, 404)
|
||||||
|
self.assertTrue(MCPToken.objects.filter(pk=other_token.pk).exists())
|
||||||
@@ -1,4 +1,9 @@
|
|||||||
from .discovery import register_discovery_tools
|
from .discovery import register_discovery_tools
|
||||||
|
from .health import register_health_tools
|
||||||
from .search import register_search_tools
|
from .search import register_search_tools
|
||||||
|
|
||||||
__all__ = ["register_search_tools", "register_discovery_tools"]
|
__all__ = [
|
||||||
|
"register_search_tools",
|
||||||
|
"register_discovery_tools",
|
||||||
|
"register_health_tools",
|
||||||
|
]
|
||||||
|
|||||||
@@ -20,16 +20,21 @@ def _clamp(limit: int) -> int:
|
|||||||
|
|
||||||
def register_discovery_tools(mcp):
|
def register_discovery_tools(mcp):
|
||||||
@mcp.tool
|
@mcp.tool
|
||||||
async def list_libraries(limit: int = DEFAULT_LIMIT, offset: int = 0) -> dict[str, Any]:
|
async def list_libraries(
|
||||||
|
limit: int = DEFAULT_LIMIT,
|
||||||
|
offset: int = 0,
|
||||||
|
# System-injected; deliberately absent from the docstring.
|
||||||
|
workspace_id: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
"""List Mnemosyne libraries. Each library has a content-aware library_type
|
"""List Mnemosyne libraries. Each library has a content-aware library_type
|
||||||
(fiction, nonfiction, technical, music, film, art, journal) that drives
|
(fiction, nonfiction, technical, music, film, art, journal, business,
|
||||||
chunking, embedding, and re-ranking. Returns uid, name, library_type,
|
finance) that drives chunking, embedding, and re-ranking. Returns uid,
|
||||||
description for each library — use the uid or library_type to scope a
|
name, library_type, description for each library — use the uid or
|
||||||
subsequent search.
|
library_type to scope a subsequent search.
|
||||||
"""
|
"""
|
||||||
with record_tool_call("list_libraries"):
|
with record_tool_call("list_libraries"):
|
||||||
return await sync_to_async(_query_libraries, thread_sensitive=True)(
|
return await sync_to_async(_query_libraries, thread_sensitive=True)(
|
||||||
_clamp(limit), max(offset, 0)
|
_clamp(limit), max(offset, 0), workspace_id
|
||||||
)
|
)
|
||||||
|
|
||||||
@mcp.tool
|
@mcp.tool
|
||||||
@@ -37,6 +42,8 @@ def register_discovery_tools(mcp):
|
|||||||
library_uid: str | None = None,
|
library_uid: str | None = None,
|
||||||
limit: int = DEFAULT_LIMIT,
|
limit: int = DEFAULT_LIMIT,
|
||||||
offset: int = 0,
|
offset: int = 0,
|
||||||
|
# System-injected; deliberately absent from the docstring.
|
||||||
|
workspace_id: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""List collections, optionally filtered by parent library_uid.
|
"""List collections, optionally filtered by parent library_uid.
|
||||||
Collections group related items inside a library (e.g. a series of novels,
|
Collections group related items inside a library (e.g. a series of novels,
|
||||||
@@ -45,7 +52,7 @@ def register_discovery_tools(mcp):
|
|||||||
"""
|
"""
|
||||||
with record_tool_call("list_collections"):
|
with record_tool_call("list_collections"):
|
||||||
return await sync_to_async(_query_collections, thread_sensitive=True)(
|
return await sync_to_async(_query_collections, thread_sensitive=True)(
|
||||||
library_uid, _clamp(limit), max(offset, 0)
|
library_uid, _clamp(limit), max(offset, 0), workspace_id
|
||||||
)
|
)
|
||||||
|
|
||||||
@mcp.tool
|
@mcp.tool
|
||||||
@@ -54,6 +61,8 @@ def register_discovery_tools(mcp):
|
|||||||
library_uid: str | None = None,
|
library_uid: str | None = None,
|
||||||
limit: int = DEFAULT_LIMIT,
|
limit: int = DEFAULT_LIMIT,
|
||||||
offset: int = 0,
|
offset: int = 0,
|
||||||
|
# System-injected; deliberately absent from the docstring.
|
||||||
|
workspace_id: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""List items (the indexed documents/files), optionally filtered by
|
"""List items (the indexed documents/files), optionally filtered by
|
||||||
collection_uid or library_uid. Returns uid, title, item_type, file_type,
|
collection_uid or library_uid. Returns uid, title, item_type, file_type,
|
||||||
@@ -63,17 +72,27 @@ def register_discovery_tools(mcp):
|
|||||||
"""
|
"""
|
||||||
with record_tool_call("list_items"):
|
with record_tool_call("list_items"):
|
||||||
return await sync_to_async(_query_items, thread_sensitive=True)(
|
return await sync_to_async(_query_items, thread_sensitive=True)(
|
||||||
collection_uid, library_uid, _clamp(limit), max(offset, 0)
|
collection_uid, library_uid, _clamp(limit), max(offset, 0), workspace_id
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _query_libraries(limit: int, offset: int) -> dict[str, Any]:
|
_WORKSPACE_SCOPE = (
|
||||||
|
"($workspace_id IS NULL AND l.workspace_id IS NULL OR "
|
||||||
|
"l.workspace_id = $workspace_id)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _query_libraries(
|
||||||
|
limit: int, offset: int, workspace_id: str | None = None
|
||||||
|
) -> dict[str, Any]:
|
||||||
from neomodel import db
|
from neomodel import db
|
||||||
|
|
||||||
rows, _ = db.cypher_query(
|
rows, _ = db.cypher_query(
|
||||||
"MATCH (l:Library) RETURN l.uid, l.name, l.library_type, l.description "
|
"MATCH (l:Library) "
|
||||||
|
f"WHERE {_WORKSPACE_SCOPE} "
|
||||||
|
"RETURN l.uid, l.name, l.library_type, l.description "
|
||||||
"ORDER BY l.name SKIP $offset LIMIT $limit",
|
"ORDER BY l.name SKIP $offset LIMIT $limit",
|
||||||
{"offset": offset, "limit": limit},
|
{"offset": offset, "limit": limit, "workspace_id": workspace_id},
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
"libraries": [
|
"libraries": [
|
||||||
@@ -91,24 +110,33 @@ def _query_libraries(limit: int, offset: int) -> dict[str, Any]:
|
|||||||
|
|
||||||
|
|
||||||
def _query_collections(
|
def _query_collections(
|
||||||
library_uid: str | None, limit: int, offset: int
|
library_uid: str | None, limit: int, offset: int,
|
||||||
|
workspace_id: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
from neomodel import db
|
from neomodel import db
|
||||||
|
|
||||||
if library_uid:
|
if library_uid:
|
||||||
cypher = (
|
cypher = (
|
||||||
"MATCH (l:Library {uid: $library_uid})-[:CONTAINS]->(c:Collection) "
|
"MATCH (l:Library {uid: $library_uid})-[:CONTAINS]->(c:Collection) "
|
||||||
|
f"WHERE {_WORKSPACE_SCOPE} "
|
||||||
"RETURN c.uid, c.name, c.description, l.uid, l.name "
|
"RETURN c.uid, c.name, c.description, l.uid, l.name "
|
||||||
"ORDER BY c.name SKIP $offset LIMIT $limit"
|
"ORDER BY c.name SKIP $offset LIMIT $limit"
|
||||||
)
|
)
|
||||||
params = {"library_uid": library_uid, "offset": offset, "limit": limit}
|
params = {
|
||||||
|
"library_uid": library_uid, "offset": offset, "limit": limit,
|
||||||
|
"workspace_id": workspace_id,
|
||||||
|
}
|
||||||
else:
|
else:
|
||||||
cypher = (
|
cypher = (
|
||||||
"MATCH (l:Library)-[:CONTAINS]->(c:Collection) "
|
"MATCH (l:Library)-[:CONTAINS]->(c:Collection) "
|
||||||
|
f"WHERE {_WORKSPACE_SCOPE} "
|
||||||
"RETURN c.uid, c.name, c.description, l.uid, l.name "
|
"RETURN c.uid, c.name, c.description, l.uid, l.name "
|
||||||
"ORDER BY l.name, c.name SKIP $offset LIMIT $limit"
|
"ORDER BY l.name, c.name SKIP $offset LIMIT $limit"
|
||||||
)
|
)
|
||||||
params = {"offset": offset, "limit": limit}
|
params = {
|
||||||
|
"offset": offset, "limit": limit,
|
||||||
|
"workspace_id": workspace_id,
|
||||||
|
}
|
||||||
|
|
||||||
rows, _ = db.cypher_query(cypher, params)
|
rows, _ = db.cypher_query(cypher, params)
|
||||||
return {
|
return {
|
||||||
@@ -132,11 +160,15 @@ def _query_items(
|
|||||||
library_uid: str | None,
|
library_uid: str | None,
|
||||||
limit: int,
|
limit: int,
|
||||||
offset: int,
|
offset: int,
|
||||||
|
workspace_id: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
from neomodel import db
|
from neomodel import db
|
||||||
|
|
||||||
where = []
|
where = [_WORKSPACE_SCOPE]
|
||||||
params: dict[str, Any] = {"offset": offset, "limit": limit}
|
params: dict[str, Any] = {
|
||||||
|
"offset": offset, "limit": limit,
|
||||||
|
"workspace_id": workspace_id,
|
||||||
|
}
|
||||||
if collection_uid:
|
if collection_uid:
|
||||||
where.append("c.uid = $collection_uid")
|
where.append("c.uid = $collection_uid")
|
||||||
params["collection_uid"] = collection_uid
|
params["collection_uid"] = collection_uid
|
||||||
@@ -144,7 +176,7 @@ def _query_items(
|
|||||||
where.append("l.uid = $library_uid")
|
where.append("l.uid = $library_uid")
|
||||||
params["library_uid"] = library_uid
|
params["library_uid"] = library_uid
|
||||||
|
|
||||||
where_clause = ("WHERE " + " AND ".join(where)) if where else ""
|
where_clause = "WHERE " + " AND ".join(where)
|
||||||
cypher = (
|
cypher = (
|
||||||
"MATCH (l:Library)-[:CONTAINS]->(c:Collection)-[:CONTAINS]->(i:Item) "
|
"MATCH (l:Library)-[:CONTAINS]->(c:Collection)-[:CONTAINS]->(i:Item) "
|
||||||
f"{where_clause} "
|
f"{where_clause} "
|
||||||
|
|||||||
123
mnemosyne/mcp_server/tools/health.py
Normal file
123
mnemosyne/mcp_server/tools/health.py
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
"""Health-check MCP tool — used by Pallas/Daedalus health pollers.
|
||||||
|
|
||||||
|
Per the Pallas health spec, returns one of:
|
||||||
|
- ok — all dependencies reachable
|
||||||
|
- degraded — non-critical dependency unhealthy (chat allowed)
|
||||||
|
- error — critical dependency unhealthy (chat blocked)
|
||||||
|
|
||||||
|
The tool is intercepted by the FastMCP server and never invokes an LLM —
|
||||||
|
it executes synchronously against Neo4j, S3, and the embedding model
|
||||||
|
endpoint, and returns within the poller's timeout.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from asgiref.sync import sync_to_async
|
||||||
|
|
||||||
|
from ..metrics import record_tool_call
|
||||||
|
|
||||||
|
|
||||||
|
def register_health_tools(mcp):
|
||||||
|
@mcp.tool
|
||||||
|
async def get_health() -> dict[str, Any]:
|
||||||
|
"""Health check for Mnemosyne.
|
||||||
|
|
||||||
|
Returns a status object compatible with the Pallas health spec:
|
||||||
|
{status: "ok"|"degraded"|"error", checks: {neo4j, s3, embedding}}.
|
||||||
|
"""
|
||||||
|
with record_tool_call("get_health"):
|
||||||
|
return await sync_to_async(_run_health_check, thread_sensitive=True)()
|
||||||
|
|
||||||
|
|
||||||
|
def _run_health_check() -> dict[str, Any]:
|
||||||
|
"""Synchronous health check across Neo4j, S3, and embedding model."""
|
||||||
|
checks: dict[str, dict[str, Any]] = {}
|
||||||
|
|
||||||
|
checks["neo4j"] = _check_neo4j()
|
||||||
|
checks["s3"] = _check_s3()
|
||||||
|
checks["embedding"] = _check_embedding_model()
|
||||||
|
|
||||||
|
# Aggregate status: error if any critical check failed; degraded if a
|
||||||
|
# non-critical check failed; ok otherwise.
|
||||||
|
if checks["neo4j"]["status"] == "error" or checks["s3"]["status"] == "error":
|
||||||
|
status = "error"
|
||||||
|
elif any(c["status"] != "ok" for c in checks.values()):
|
||||||
|
status = "degraded"
|
||||||
|
else:
|
||||||
|
status = "ok"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"status": status,
|
||||||
|
"checks": checks,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _check_neo4j() -> dict[str, Any]:
|
||||||
|
start = time.time()
|
||||||
|
try:
|
||||||
|
from neomodel import db
|
||||||
|
|
||||||
|
db.cypher_query("RETURN 1")
|
||||||
|
return {
|
||||||
|
"status": "ok",
|
||||||
|
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||||
|
}
|
||||||
|
except Exception as exc:
|
||||||
|
return {
|
||||||
|
"status": "error",
|
||||||
|
"error": str(exc),
|
||||||
|
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _check_s3() -> dict[str, Any]:
|
||||||
|
start = time.time()
|
||||||
|
try:
|
||||||
|
from django.core.files.storage import default_storage
|
||||||
|
|
||||||
|
# `exists` on a path that won't exist is the cheapest round-trip
|
||||||
|
# we have. It returns False rather than raising on most backends.
|
||||||
|
default_storage.exists("__healthcheck__")
|
||||||
|
return {
|
||||||
|
"status": "ok",
|
||||||
|
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||||
|
}
|
||||||
|
except Exception as exc:
|
||||||
|
return {
|
||||||
|
"status": "error",
|
||||||
|
"error": str(exc),
|
||||||
|
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _check_embedding_model() -> dict[str, Any]:
|
||||||
|
"""Soft check: confirm a system embedding model is configured.
|
||||||
|
|
||||||
|
We don't hit the model endpoint here — that would burn GPU time on
|
||||||
|
every poll. The poller-level check is "is a model registered."
|
||||||
|
"""
|
||||||
|
start = time.time()
|
||||||
|
try:
|
||||||
|
from llm_manager.models import LLMModel
|
||||||
|
|
||||||
|
model = LLMModel.get_system_embedding_model()
|
||||||
|
if model is None:
|
||||||
|
return {
|
||||||
|
"status": "degraded",
|
||||||
|
"error": "no system embedding model configured",
|
||||||
|
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"status": "ok",
|
||||||
|
"model": model.name,
|
||||||
|
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||||
|
}
|
||||||
|
except Exception as exc:
|
||||||
|
return {
|
||||||
|
"status": "degraded",
|
||||||
|
"error": str(exc),
|
||||||
|
"duration_ms": round((time.time() - start) * 1000, 1),
|
||||||
|
}
|
||||||
@@ -27,13 +27,19 @@ def register_search_tools(mcp):
|
|||||||
rerank: bool = True,
|
rerank: bool = True,
|
||||||
include_images: bool = True,
|
include_images: bool = True,
|
||||||
search_types: list[str] | None = None,
|
search_types: list[str] | None = None,
|
||||||
|
# workspace_id is system-injected by Daedalus's chat path. It is
|
||||||
|
# intentionally absent from the docstring so the calling LLM is
|
||||||
|
# never told it exists. Whatever value the LLM produces here is
|
||||||
|
# overwritten by Daedalus before the call reaches Mnemosyne.
|
||||||
|
workspace_id: str | None = None,
|
||||||
ctx: Context | None = None,
|
ctx: Context | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Hybrid retrieval over Mnemosyne: vector + full-text + concept-graph
|
"""Hybrid retrieval over Mnemosyne: vector + full-text + concept-graph
|
||||||
candidates fused by RRF and optionally re-ranked by Synesis.
|
candidates fused by RRF and optionally re-ranked by Synesis.
|
||||||
|
|
||||||
Filters: library_uid (exact library), library_type (one of fiction,
|
Filters: library_uid (exact library), library_type (one of fiction,
|
||||||
nonfiction, technical, music, film, art, journal), or collection_uid.
|
nonfiction, technical, music, film, art, journal, business, finance),
|
||||||
|
or collection_uid.
|
||||||
Set rerank=False to skip re-ranking. search_types defaults to all three.
|
Set rerank=False to skip re-ranking. search_types defaults to all three.
|
||||||
|
|
||||||
Returns ranked candidates with chunk_uid (use get_chunk for full text),
|
Returns ranked candidates with chunk_uid (use get_chunk for full text),
|
||||||
@@ -49,6 +55,7 @@ def register_search_tools(mcp):
|
|||||||
library_uid=library_uid,
|
library_uid=library_uid,
|
||||||
library_type=library_type,
|
library_type=library_type,
|
||||||
collection_uid=collection_uid,
|
collection_uid=collection_uid,
|
||||||
|
workspace_id=workspace_id,
|
||||||
limit=limit,
|
limit=limit,
|
||||||
rerank=rerank,
|
rerank=rerank,
|
||||||
include_images=include_images,
|
include_images=include_images,
|
||||||
@@ -56,7 +63,12 @@ def register_search_tools(mcp):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@mcp.tool
|
@mcp.tool
|
||||||
async def get_chunk(chunk_uid: str, ctx: Context | None = None) -> dict[str, Any]:
|
async def get_chunk(
|
||||||
|
chunk_uid: str,
|
||||||
|
# System-injected; deliberately absent from the docstring.
|
||||||
|
workspace_id: str | None = None,
|
||||||
|
ctx: Context | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
"""Fetch the full text of a chunk by its uid (typically obtained from `search`).
|
"""Fetch the full text of a chunk by its uid (typically obtained from `search`).
|
||||||
|
|
||||||
Returns the chunk text plus parent item context: chunk_uid, chunk_index,
|
Returns the chunk text plus parent item context: chunk_uid, chunk_index,
|
||||||
@@ -64,11 +76,13 @@ def register_search_tools(mcp):
|
|||||||
text_preview from `search` isn't enough.
|
text_preview from `search` isn't enough.
|
||||||
"""
|
"""
|
||||||
with record_tool_call("get_chunk"):
|
with record_tool_call("get_chunk"):
|
||||||
return await sync_to_async(_load_chunk, thread_sensitive=True)(chunk_uid)
|
return await sync_to_async(_load_chunk, thread_sensitive=True)(
|
||||||
|
chunk_uid, workspace_id
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _run_search(*, user, query, library_uid, library_type, collection_uid, limit,
|
def _run_search(*, user, query, library_uid, library_type, collection_uid,
|
||||||
rerank, include_images, search_types) -> dict[str, Any]:
|
workspace_id, limit, rerank, include_images, search_types) -> dict[str, Any]:
|
||||||
from library.services.search import SearchRequest, SearchService
|
from library.services.search import SearchRequest, SearchService
|
||||||
|
|
||||||
req = SearchRequest(
|
req = SearchRequest(
|
||||||
@@ -76,6 +90,7 @@ def _run_search(*, user, query, library_uid, library_type, collection_uid, limit
|
|||||||
library_uid=library_uid,
|
library_uid=library_uid,
|
||||||
library_type=library_type,
|
library_type=library_type,
|
||||||
collection_uid=collection_uid,
|
collection_uid=collection_uid,
|
||||||
|
workspace_id=workspace_id,
|
||||||
search_types=search_types,
|
search_types=search_types,
|
||||||
limit=limit,
|
limit=limit,
|
||||||
vector_top_k=getattr(settings, "SEARCH_VECTOR_TOP_K", 50),
|
vector_top_k=getattr(settings, "SEARCH_VECTOR_TOP_K", 50),
|
||||||
@@ -97,15 +112,17 @@ def _run_search(*, user, query, library_uid, library_type, collection_uid, limit
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _load_chunk(chunk_uid: str) -> dict[str, Any]:
|
def _load_chunk(chunk_uid: str, workspace_id: str | None = None) -> dict[str, Any]:
|
||||||
from neomodel import db
|
from neomodel import db
|
||||||
|
|
||||||
rows, _ = db.cypher_query(
|
rows, _ = db.cypher_query(
|
||||||
"MATCH (l:Library)-[:CONTAINS]->(:Collection)-[:CONTAINS]->"
|
"MATCH (l:Library)-[:CONTAINS]->(:Collection)-[:CONTAINS]->"
|
||||||
"(i:Item)-[:HAS_CHUNK]->(c:Chunk {uid: $uid}) "
|
"(i:Item)-[:HAS_CHUNK]->(c:Chunk {uid: $uid}) "
|
||||||
|
"WHERE ($workspace_id IS NULL AND l.workspace_id IS NULL OR "
|
||||||
|
" l.workspace_id = $workspace_id) "
|
||||||
"RETURN c.uid, c.chunk_index, c.chunk_s3_key, "
|
"RETURN c.uid, c.chunk_index, c.chunk_s3_key, "
|
||||||
"i.uid, i.title, l.library_type LIMIT 1",
|
"i.uid, i.title, l.library_type LIMIT 1",
|
||||||
{"uid": chunk_uid},
|
{"uid": chunk_uid, "workspace_id": workspace_id},
|
||||||
)
|
)
|
||||||
if not rows:
|
if not rows:
|
||||||
raise ValueError(f"Chunk not found: {chunk_uid}")
|
raise ValueError(f"Chunk not found: {chunk_uid}")
|
||||||
|
|||||||
16
mnemosyne/mcp_server/urls.py
Normal file
16
mnemosyne/mcp_server/urls.py
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
"""URL routes for the MCP token self-service dashboard."""
|
||||||
|
|
||||||
|
from django.urls import path
|
||||||
|
|
||||||
|
from . import views
|
||||||
|
|
||||||
|
app_name = "mcp_server"
|
||||||
|
|
||||||
|
urlpatterns = [
|
||||||
|
path("profile/mcp-tokens/", views.mcp_token_list, name="mcp-token-list"),
|
||||||
|
path("profile/mcp-tokens/add/", views.mcp_token_create, name="mcp-token-create"),
|
||||||
|
path("profile/mcp-tokens/<int:pk>/", views.mcp_token_detail, name="mcp-token-detail"),
|
||||||
|
path("profile/mcp-tokens/<int:pk>/edit/", views.mcp_token_edit, name="mcp-token-edit"),
|
||||||
|
path("profile/mcp-tokens/<int:pk>/revoke/", views.mcp_token_revoke, name="mcp-token-revoke"),
|
||||||
|
path("profile/mcp-tokens/<int:pk>/delete/", views.mcp_token_delete, name="mcp-token-delete"),
|
||||||
|
]
|
||||||
95
mnemosyne/mcp_server/views.py
Normal file
95
mnemosyne/mcp_server/views.py
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
"""Self-service dashboard for MCP bearer tokens.
|
||||||
|
|
||||||
|
Mirrors the Themis API-keys flow visually but stores hashed tokens. Plaintext
|
||||||
|
is shown to the user exactly once (on the create-success page) and never
|
||||||
|
persisted.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from django.contrib import messages
|
||||||
|
from django.contrib.auth.decorators import login_required
|
||||||
|
from django.http import HttpRequest, HttpResponse
|
||||||
|
from django.shortcuts import get_object_or_404, redirect, render
|
||||||
|
from django.views.decorators.http import require_GET, require_http_methods, require_POST
|
||||||
|
|
||||||
|
from .forms import MCPTokenCreateForm, MCPTokenEditForm
|
||||||
|
from .models import MCPToken
|
||||||
|
|
||||||
|
|
||||||
|
@login_required
|
||||||
|
@require_GET
|
||||||
|
def mcp_token_list(request: HttpRequest) -> HttpResponse:
|
||||||
|
tokens = MCPToken.objects.filter(user=request.user).order_by("-created_at")
|
||||||
|
return render(request, "mcp_server/tokens/list.html", {"tokens": tokens})
|
||||||
|
|
||||||
|
|
||||||
|
@login_required
|
||||||
|
@require_http_methods(["GET", "POST"])
|
||||||
|
def mcp_token_create(request: HttpRequest) -> HttpResponse:
|
||||||
|
if request.method == "POST":
|
||||||
|
form = MCPTokenCreateForm(request.POST)
|
||||||
|
if form.is_valid():
|
||||||
|
token, plaintext = MCPToken.objects.create_token(
|
||||||
|
user=request.user,
|
||||||
|
name=form.cleaned_data["name"],
|
||||||
|
allowed_tools=form.cleaned_data.get("allowed_tools") or [],
|
||||||
|
expires_at=form.cleaned_data.get("expires_at") or None,
|
||||||
|
)
|
||||||
|
return render(
|
||||||
|
request,
|
||||||
|
"mcp_server/tokens/created.html",
|
||||||
|
{"token": token, "plaintext": plaintext},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
form = MCPTokenCreateForm()
|
||||||
|
|
||||||
|
return render(request, "mcp_server/tokens/create.html", {"form": form})
|
||||||
|
|
||||||
|
|
||||||
|
@login_required
|
||||||
|
@require_GET
|
||||||
|
def mcp_token_detail(request: HttpRequest, pk: int) -> HttpResponse:
|
||||||
|
token = get_object_or_404(MCPToken, pk=pk, user=request.user)
|
||||||
|
return render(request, "mcp_server/tokens/detail.html", {"token": token})
|
||||||
|
|
||||||
|
|
||||||
|
@login_required
|
||||||
|
@require_http_methods(["GET", "POST"])
|
||||||
|
def mcp_token_edit(request: HttpRequest, pk: int) -> HttpResponse:
|
||||||
|
token = get_object_or_404(MCPToken, pk=pk, user=request.user)
|
||||||
|
|
||||||
|
if request.method == "POST":
|
||||||
|
form = MCPTokenEditForm(request.POST, instance=token)
|
||||||
|
if form.is_valid():
|
||||||
|
instance = form.save(commit=False)
|
||||||
|
instance.allowed_tools = form.cleaned_data.get("allowed_tools") or []
|
||||||
|
instance.save()
|
||||||
|
messages.success(request, "MCP token updated.")
|
||||||
|
return redirect("mcp_server:mcp-token-detail", pk=token.pk)
|
||||||
|
else:
|
||||||
|
form = MCPTokenEditForm(instance=token)
|
||||||
|
|
||||||
|
return render(
|
||||||
|
request, "mcp_server/tokens/edit.html", {"form": form, "token": token}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@login_required
|
||||||
|
@require_POST
|
||||||
|
def mcp_token_revoke(request: HttpRequest, pk: int) -> HttpResponse:
|
||||||
|
token = get_object_or_404(MCPToken, pk=pk, user=request.user)
|
||||||
|
token.is_active = False
|
||||||
|
token.save(update_fields=["is_active", "updated_at"])
|
||||||
|
messages.success(request, f"Revoked “{token.name}”. The token can no longer be used.")
|
||||||
|
return redirect("mcp_server:mcp-token-detail", pk=token.pk)
|
||||||
|
|
||||||
|
|
||||||
|
@login_required
|
||||||
|
@require_POST
|
||||||
|
def mcp_token_delete(request: HttpRequest, pk: int) -> HttpResponse:
|
||||||
|
token = get_object_or_404(MCPToken, pk=pk, user=request.user)
|
||||||
|
name = token.name
|
||||||
|
token.delete()
|
||||||
|
messages.success(request, f"Deleted “{name}”.")
|
||||||
|
return redirect("mcp_server:mcp-token-list")
|
||||||
@@ -20,7 +20,7 @@ django.setup()
|
|||||||
|
|
||||||
from django.core.asgi import get_asgi_application # noqa: E402
|
from django.core.asgi import get_asgi_application # noqa: E402
|
||||||
from starlette.applications import Starlette # noqa: E402
|
from starlette.applications import Starlette # noqa: E402
|
||||||
from starlette.responses import JSONResponse # noqa: E402
|
from starlette.responses import JSONResponse, RedirectResponse # noqa: E402
|
||||||
from starlette.routing import Mount, Route # noqa: E402
|
from starlette.routing import Mount, Route # noqa: E402
|
||||||
|
|
||||||
from mcp_server.server import mcp # noqa: E402
|
from mcp_server.server import mcp # noqa: E402
|
||||||
@@ -35,6 +35,13 @@ async def health(request):
|
|||||||
return JSONResponse({"status": "ok"})
|
return JSONResponse({"status": "ok"})
|
||||||
|
|
||||||
|
|
||||||
|
async def mcp_redirect(request):
|
||||||
|
"""Bare /mcp → /mcp/ — MCP clients that omit the trailing slash hit
|
||||||
|
Starlette's Mount which only matches /mcp/* paths, falling through to
|
||||||
|
Django and returning 404. A 307 preserves the POST body."""
|
||||||
|
return RedirectResponse(url="/mcp/", status_code=307)
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(app):
|
async def lifespan(app):
|
||||||
async with mcp_http_app.lifespan(app), mcp_sse_app.lifespan(app):
|
async with mcp_http_app.lifespan(app), mcp_sse_app.lifespan(app):
|
||||||
@@ -44,6 +51,7 @@ async def lifespan(app):
|
|||||||
app = Starlette(
|
app = Starlette(
|
||||||
routes=[
|
routes=[
|
||||||
Route("/mcp/health", health),
|
Route("/mcp/health", health),
|
||||||
|
Route("/mcp", mcp_redirect, methods=["GET", "POST"]),
|
||||||
Mount("/mcp/sse", app=mcp_sse_app),
|
Mount("/mcp/sse", app=mcp_sse_app),
|
||||||
Mount("/mcp", app=mcp_http_app),
|
Mount("/mcp", app=mcp_http_app),
|
||||||
Mount("/", app=application),
|
Mount("/", app=application),
|
||||||
|
|||||||
@@ -181,6 +181,18 @@ else:
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# --- Daedalus S3 (cross-bucket reads for ingest) ---
|
||||||
|
# Mnemosyne ingests files written to Daedalus's S3 bucket. These vars
|
||||||
|
# configure read access; the file is copied into AWS_STORAGE_BUCKET_NAME
|
||||||
|
# (Mnemosyne's own bucket) by the Celery ingest task before processing.
|
||||||
|
DAEDALUS_S3_ENDPOINT_URL = env("DAEDALUS_S3_ENDPOINT_URL", default="")
|
||||||
|
DAEDALUS_S3_ACCESS_KEY_ID = env("DAEDALUS_S3_ACCESS_KEY_ID", default="")
|
||||||
|
DAEDALUS_S3_SECRET_ACCESS_KEY = env("DAEDALUS_S3_SECRET_ACCESS_KEY", default="")
|
||||||
|
DAEDALUS_S3_BUCKET_NAME = env("DAEDALUS_S3_BUCKET_NAME", default="daedalus")
|
||||||
|
DAEDALUS_S3_REGION_NAME = env("DAEDALUS_S3_REGION_NAME", default="us-east-1")
|
||||||
|
DAEDALUS_S3_USE_SSL = env.bool("DAEDALUS_S3_USE_SSL", default=False)
|
||||||
|
DAEDALUS_S3_VERIFY = env.bool("DAEDALUS_S3_VERIFY", default=True)
|
||||||
|
|
||||||
# --- Celery / RabbitMQ ---
|
# --- Celery / RabbitMQ ---
|
||||||
CELERY_BROKER_URL = env(
|
CELERY_BROKER_URL = env(
|
||||||
"CELERY_BROKER_URL",
|
"CELERY_BROKER_URL",
|
||||||
@@ -196,6 +208,7 @@ CELERY_TASK_ACKS_LATE = True
|
|||||||
CELERY_WORKER_PREFETCH_MULTIPLIER = 1
|
CELERY_WORKER_PREFETCH_MULTIPLIER = 1
|
||||||
CELERY_TASK_ROUTES = {
|
CELERY_TASK_ROUTES = {
|
||||||
"library.tasks.embed_*": {"queue": "embedding"},
|
"library.tasks.embed_*": {"queue": "embedding"},
|
||||||
|
"library.tasks.ingest_*": {"queue": "embedding"},
|
||||||
"library.tasks.batch_*": {"queue": "batch"},
|
"library.tasks.batch_*": {"queue": "batch"},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -25,4 +25,6 @@ urlpatterns = [
|
|||||||
path("library/", include("library.urls")),
|
path("library/", include("library.urls")),
|
||||||
# LLM Manager
|
# LLM Manager
|
||||||
path("llm/", include("llm_manager.urls")),
|
path("llm/", include("llm_manager.urls")),
|
||||||
|
# MCP server (token dashboard at /profile/mcp-tokens/)
|
||||||
|
path("", include("mcp_server.urls")),
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -52,15 +52,20 @@ class PostgreSQLTestRunner(DiscoverRunner):
|
|||||||
from django.db import connections
|
from django.db import connections
|
||||||
|
|
||||||
db_cfg = self.pg_manager.get_django_database_config()
|
db_cfg = self.pg_manager.get_django_database_config()
|
||||||
# Preserve Django's defaulted TEST sub-dict (CHARSET/MIRROR/MIGRATE…).
|
# Preserve Django's defaulted top-level keys (ATOMIC_REQUESTS,
|
||||||
existing_test = connections["default"].settings_dict.get("TEST", {})
|
# AUTOCOMMIT, OPTIONS, …) and the TEST sub-dict (CHARSET, MIRROR,
|
||||||
merged_test = {**existing_test, **db_cfg.get("TEST", {})}
|
# MIGRATE, …) — these are populated lazily by Django and absent from
|
||||||
db_cfg["TEST"] = merged_test
|
# the user's raw settings.DATABASES, so a naive overwrite breaks
|
||||||
settings.DATABASES["default"] = db_cfg
|
# request handling that consults them.
|
||||||
|
existing = connections["default"].settings_dict
|
||||||
|
existing_test = existing.get("TEST", {})
|
||||||
|
merged = {**existing, **db_cfg}
|
||||||
|
merged["TEST"] = {**existing_test, **db_cfg.get("TEST", {})}
|
||||||
|
settings.DATABASES["default"] = merged
|
||||||
# The default connection was instantiated at Django bootstrap; its
|
# The default connection was instantiated at Django bootstrap; its
|
||||||
# settings_dict is independent of settings.DATABASES. Sync it
|
# settings_dict is independent of settings.DATABASES. Sync it
|
||||||
# manually so test code talks to the container, not the dev DB.
|
# manually so test code talks to the container, not the dev DB.
|
||||||
connections["default"].settings_dict.update(db_cfg)
|
connections["default"].settings_dict.update(merged)
|
||||||
logger.info("PostgreSQL test DB ready on port %s", self.pg_manager.assigned_port)
|
logger.info("PostgreSQL test DB ready on port %s", self.pg_manager.assigned_port)
|
||||||
|
|
||||||
# ── Neo4j ──────────────────────────────────────────────────────
|
# ── Neo4j ──────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
# Generated by Django 5.2.13 on 2026-04-27 11:30
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('themis', '0003_alter_userprofile_current_timezone_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='userapikey',
|
||||||
|
name='key_type',
|
||||||
|
field=models.CharField(choices=[('api', 'API Key'), ('dav', 'DAV Credentials'), ('token', 'Access Token'), ('secret', 'Secret Key'), ('other', 'Other')], default='api', help_text='Type of credential', max_length=30),
|
||||||
|
),
|
||||||
|
]
|
||||||
@@ -228,7 +228,6 @@ class UserAPIKey(models.Model):
|
|||||||
|
|
||||||
KEY_TYPE_CHOICES = [
|
KEY_TYPE_CHOICES = [
|
||||||
("api", "API Key"),
|
("api", "API Key"),
|
||||||
("mcp", "MCP Server"),
|
|
||||||
("dav", "DAV Credentials"),
|
("dav", "DAV Credentials"),
|
||||||
("token", "Access Token"),
|
("token", "Access Token"),
|
||||||
("secret", "Secret Key"),
|
("secret", "Secret Key"),
|
||||||
|
|||||||
@@ -38,6 +38,16 @@
|
|||||||
API Keys
|
API Keys
|
||||||
</a>
|
</a>
|
||||||
</li>
|
</li>
|
||||||
|
<li>
|
||||||
|
<a href="{% url 'mcp_server:mcp-token-list' %}">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4" fill="none"
|
||||||
|
viewBox="0 0 24 24" stroke="currentColor">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||||
|
d="M9 12l2 2 4-4m5.618-4.016A11.955 11.955 0 0112 2.944a11.955 11.955 0 01-8.618 3.04A12.02 12.02 0 003 9c0 5.591 3.824 10.29 9 11.622 5.176-1.332 9-6.03 9-11.622 0-1.042-.133-2.052-.382-3.016z" />
|
||||||
|
</svg>
|
||||||
|
MCP Tokens
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
<div class="divider my-0"></div>
|
<div class="divider my-0"></div>
|
||||||
<li>
|
<li>
|
||||||
<form method="post" action="{% url 'logout' %}">
|
<form method="post" action="{% url 'logout' %}">
|
||||||
|
|||||||
@@ -248,7 +248,7 @@ class APIKeyEditFormTest(TestCase):
|
|||||||
form = APIKeyEditForm(
|
form = APIKeyEditForm(
|
||||||
data={
|
data={
|
||||||
"service_name": "Updated Service",
|
"service_name": "Updated Service",
|
||||||
"key_type": "mcp",
|
"key_type": "token",
|
||||||
"label": "Updated",
|
"label": "Updated",
|
||||||
"instructions": "",
|
"instructions": "",
|
||||||
"help_url": "",
|
"help_url": "",
|
||||||
@@ -260,6 +260,6 @@ class APIKeyEditFormTest(TestCase):
|
|||||||
form.save()
|
form.save()
|
||||||
self.key.refresh_from_db()
|
self.key.refresh_from_db()
|
||||||
self.assertEqual(self.key.service_name, "Updated Service")
|
self.assertEqual(self.key.service_name, "Updated Service")
|
||||||
self.assertEqual(self.key.key_type, "mcp")
|
self.assertEqual(self.key.key_type, "token")
|
||||||
self.assertFalse(self.key.is_active)
|
self.assertFalse(self.key.is_active)
|
||||||
self.assertEqual(self.key.encrypted_value, original_encrypted)
|
self.assertEqual(self.key.encrypted_value, original_encrypted)
|
||||||
|
|||||||
@@ -209,7 +209,6 @@ class UserAPIKeyModelTest(TestCase):
|
|||||||
"""KEY_TYPE_CHOICES contains expected types."""
|
"""KEY_TYPE_CHOICES contains expected types."""
|
||||||
type_keys = [t[0] for t in UserAPIKey.KEY_TYPE_CHOICES]
|
type_keys = [t[0] for t in UserAPIKey.KEY_TYPE_CHOICES]
|
||||||
self.assertIn("api", type_keys)
|
self.assertIn("api", type_keys)
|
||||||
self.assertIn("mcp", type_keys)
|
|
||||||
self.assertIn("dav", type_keys)
|
self.assertIn("dav", type_keys)
|
||||||
self.assertIn("token", type_keys)
|
self.assertIn("token", type_keys)
|
||||||
self.assertIn("secret", type_keys)
|
self.assertIn("secret", type_keys)
|
||||||
|
|||||||
86
nginx/mnemosyne.conf
Normal file
86
nginx/mnemosyne.conf
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
# Mnemosyne nginx — single virtual host that fronts the Django web app
|
||||||
|
# and the FastMCP server. HAProxy on Titania terminates TLS and routes by
|
||||||
|
# hostname; this nginx is plain HTTP on the internal network.
|
||||||
|
|
||||||
|
# Map of upstreams to give us readable proxy_pass targets and easy retries.
|
||||||
|
upstream mnemosyne_web {
|
||||||
|
server web:8000 max_fails=3 fail_timeout=30s;
|
||||||
|
}
|
||||||
|
|
||||||
|
upstream mnemosyne_mcp {
|
||||||
|
server mcp:22091 max_fails=3 fail_timeout=30s;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80 default_server;
|
||||||
|
server_name _;
|
||||||
|
|
||||||
|
# Reasonable limits — file uploads to the ingest endpoint can be big,
|
||||||
|
# but the bulk path is S3-direct from Daedalus. 64 MB covers admin
|
||||||
|
# uploads and direct REST POST /library/api/items/upload.
|
||||||
|
client_max_body_size 64m;
|
||||||
|
client_body_timeout 120s;
|
||||||
|
|
||||||
|
# Mnemosyne's REST API — Django REST Framework views + admin.
|
||||||
|
# Under /library/api/* per mnemosyne/urls.py and /admin/* per Django.
|
||||||
|
location /library/ {
|
||||||
|
proxy_pass http://mnemosyne_web;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_read_timeout 300s;
|
||||||
|
}
|
||||||
|
|
||||||
|
location /admin/ {
|
||||||
|
proxy_pass http://mnemosyne_web;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_read_timeout 300s;
|
||||||
|
}
|
||||||
|
|
||||||
|
# FastMCP Streamable HTTP at /mcp/ and SSE at /mcp/sse/.
|
||||||
|
# Long-running streams need disabled buffering and a generous timeout.
|
||||||
|
location /mcp/ {
|
||||||
|
proxy_pass http://mnemosyne_mcp;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_cache off;
|
||||||
|
proxy_read_timeout 600s;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Static files baked into the image at /app/staticfiles, mounted into
|
||||||
|
# this nginx via a named volume populated by the web service.
|
||||||
|
location /static/ {
|
||||||
|
alias /var/www/static/;
|
||||||
|
access_log off;
|
||||||
|
expires 30d;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Prometheus scrape endpoint — internal networks only.
|
||||||
|
# Allows: localhost + RFC1918 private ranges (10/8, 172.16/12, 192.168/16).
|
||||||
|
location /metrics {
|
||||||
|
allow 127.0.0.0/8;
|
||||||
|
allow 10.0.0.0/8;
|
||||||
|
allow 172.16.0.0/12;
|
||||||
|
allow 192.168.0.0/16;
|
||||||
|
deny all;
|
||||||
|
proxy_pass http://mnemosyne_web;
|
||||||
|
access_log off;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Liveness probe — proxies through to the MCP health endpoint.
|
||||||
|
location = /healthz {
|
||||||
|
proxy_pass http://mnemosyne_mcp/mcp/health;
|
||||||
|
access_log off;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -40,6 +40,17 @@ dev = [
|
|||||||
"django-debug-toolbar>=4.0,<5.0",
|
"django-debug-toolbar>=4.0,<5.0",
|
||||||
"docker>=7.0,<8.0",
|
"docker>=7.0,<8.0",
|
||||||
]
|
]
|
||||||
|
test = [
|
||||||
|
"pytest>=8.0,<9.0",
|
||||||
|
"pytest-django>=4.8,<5.0",
|
||||||
|
]
|
||||||
|
lint = [
|
||||||
|
"ruff>=0.6,<1.0",
|
||||||
|
]
|
||||||
|
docs = [
|
||||||
|
"mkdocs>=1.6,<2.0",
|
||||||
|
"mkdocs-material>=9.5,<10.0",
|
||||||
|
]
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
requires = ["setuptools>=68.0"]
|
requires = ["setuptools>=68.0"]
|
||||||
|
|||||||
6
validator/.gitignore
vendored
Normal file
6
validator/.gitignore
vendored
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
fastagent.secrets.yaml
|
||||||
|
fastagent.jsonl
|
||||||
|
.venv/
|
||||||
|
*.egg-info/
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
101
validator/README.md
Normal file
101
validator/README.md
Normal file
@@ -0,0 +1,101 @@
|
|||||||
|
# Mnemosyne Validator
|
||||||
|
|
||||||
|
A bare [FastAgent](https://github.com/evalstate/fast-agent) + [Pallas](https://git.helu.ca/r/pallas) project whose only purpose is to exercise Mnemosyne's MCP server end-to-end. Use it to confirm the transport works, every MCP tool registers, args/responses round-trip, and the local LLM can actually drive the tools.
|
||||||
|
|
||||||
|
This is **not** a production agent. It does not represent the long-term Daedalus integration — when Daedalus ships, it will inject `workspace_id` server-side. The validator never sets `workspace_id`, meaning all calls run against the global scope (libraries with `workspace_id IS NULL`).
|
||||||
|
|
||||||
|
## Layout
|
||||||
|
|
||||||
|
```
|
||||||
|
validator/
|
||||||
|
├── pyproject.toml # pallas-mcp + fast-agent-mcp deps
|
||||||
|
├── agents.yaml # one-agent Pallas topology
|
||||||
|
├── fastagent.config.yaml # default_model + mnemosyne MCP server
|
||||||
|
├── fastagent.secrets.yaml.example # schema for the (gitignored) secrets file
|
||||||
|
└── agents/
|
||||||
|
└── mnemosyne_validator.py # the FastAgent definition
|
||||||
|
```
|
||||||
|
|
||||||
|
All configuration lives in the FastAgent YAML files — no `.env` is used. The
|
||||||
|
real `fastagent.secrets.yaml` is gitignored; `fastagent.secrets.yaml.example`
|
||||||
|
documents its shape and is checked in.
|
||||||
|
|
||||||
|
## Setup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd validator/
|
||||||
|
python3.13 -m venv .venv
|
||||||
|
source .venv/bin/activate
|
||||||
|
pip install -e .
|
||||||
|
```
|
||||||
|
|
||||||
|
Copy the secrets template and fill in the bearer token (see next section):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp fastagent.secrets.yaml.example fastagent.secrets.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
## Provision an MCP bearer token
|
||||||
|
|
||||||
|
Mnemosyne requires a bearer token when `MCP_REQUIRE_AUTH=True` (the default). Generate one for your user:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd ../mnemosyne
|
||||||
|
python manage.py create_mcp_token --user <username> --name validator
|
||||||
|
```
|
||||||
|
|
||||||
|
The command prints the token **once** — paste it into `validator/fastagent.secrets.yaml` under `mcp.servers.mnemosyne.headers.Authorization` (keep the `Bearer ` prefix).
|
||||||
|
|
||||||
|
## Start Mnemosyne's MCP server
|
||||||
|
|
||||||
|
The validator hits the Mnemosyne ASGI endpoint, so Mnemosyne's MCP server must be running. From the Mnemosyne project:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd mnemosyne/
|
||||||
|
uvicorn mnemosyne.asgi:app --host 0.0.0.0 --port 22091 --workers 1
|
||||||
|
```
|
||||||
|
|
||||||
|
By default the validator points at `http://localhost:22091/mcp/` (note the trailing slash — Starlette's `Mount` only matches paths under the mount prefix, so `/mcp` without the slash falls through to Django). If your Mnemosyne is on another host, override `mcp.servers.mnemosyne.url` in `fastagent.secrets.yaml`.
|
||||||
|
|
||||||
|
## Run the validator
|
||||||
|
|
||||||
|
The `mnemosyne-validator` script is a thin alias for `pallas`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Start with the registry (Pallas mode):
|
||||||
|
mnemosyne-validator
|
||||||
|
|
||||||
|
# Or run the agent directly (no registry):
|
||||||
|
mnemosyne-validator --agent mnemosyne_validator
|
||||||
|
```
|
||||||
|
|
||||||
|
To chat with the agent directly without spinning up a Pallas registry, use the `fast-agent` CLI (provided by `fast-agent-mcp`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
fast-agent go --config-path fastagent.config.yaml --url http://localhost:24301/mcp mnemosyne_validator
|
||||||
|
```
|
||||||
|
|
||||||
|
## What to test
|
||||||
|
|
||||||
|
These prompts exercise every Mnemosyne MCP tool. After each, the agent should call the named tool and surface the result.
|
||||||
|
|
||||||
|
| Prompt | Tool | What to verify |
|
||||||
|
|--------|------|----------------|
|
||||||
|
| "Run a health check on Mnemosyne." | `get_health` | Returns `status: ok` if Neo4j + S3 + embedding model are all reachable. `degraded` if one is down. |
|
||||||
|
| "List all libraries." | `list_libraries` | Returns the libraries seeded by `load_library_types`, each with `library_type` set. |
|
||||||
|
| "List collections in library `<uid>`." | `list_collections` | Returns collections inside the named library. |
|
||||||
|
| "List items in collection `<uid>`." | `list_items` | Returns items with `chunk_count` and `embedding_status`. |
|
||||||
|
| "Search the technical libraries for `<query>`." | `search` | Returns ranked candidates with `chunk_uid`, `score`, `text_preview`, `library_type`. |
|
||||||
|
| "Fetch the full text of chunk `<chunk_uid>`." | `get_chunk` | Returns the full chunk text from S3. |
|
||||||
|
|
||||||
|
If a call errors, the agent surfaces it verbatim — that's the failure mode you want.
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
**"Invalid MCP token"** — token wasn't provisioned, was provisioned for a different user, or got mangled when pasted. Re-run `create_mcp_token` and paste again. Tokens are SHA-256 hashed at rest and can't be retrieved later.
|
||||||
|
|
||||||
|
**"Couldn't connect to Mnemosyne"** — the ASGI server isn't running, or it's bound to a different host/port than `mcp.servers.mnemosyne.url` says. Check `curl http://localhost:22091/mcp/health` returns `{"status":"ok"}`.
|
||||||
|
|
||||||
|
**"No system embedding model configured" in `get_health`** — `LLMModel.get_system_embedding_model()` returns nothing. Configure the embedding model via the Mnemosyne admin or `manage.py` before searches will work.
|
||||||
|
|
||||||
|
**Search returns zero candidates with no error** — Mnemosyne is reachable but has no embedded content yet. Upload an item and run `embed_item`, or use the Daedalus ingest endpoint, before re-testing search.
|
||||||
17
validator/agents.yaml
Normal file
17
validator/agents.yaml
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
# Mnemosyne Validator — Pallas deployment topology
|
||||||
|
#
|
||||||
|
# A single-agent Pallas project whose only purpose is to validate the
|
||||||
|
# Mnemosyne MCP server end-to-end. Not a production deployment.
|
||||||
|
|
||||||
|
name: mnemosyne-validator
|
||||||
|
version: "0.1.0"
|
||||||
|
host: localhost
|
||||||
|
namespace: ca.helu.mnemosyne-validator
|
||||||
|
registry_port: 24300
|
||||||
|
|
||||||
|
agents:
|
||||||
|
mnemosyne_validator:
|
||||||
|
module: agents.mnemosyne_validator
|
||||||
|
port: 24301
|
||||||
|
title: Mnemosyne Validator
|
||||||
|
description: "Exercises Mnemosyne's MCP tools: search, get_chunk, list_*, get_health"
|
||||||
0
validator/agents/__init__.py
Normal file
0
validator/agents/__init__.py
Normal file
56
validator/agents/mnemosyne_validator.py
Normal file
56
validator/agents/mnemosyne_validator.py
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
"""
|
||||||
|
Mnemosyne Validator Agent
|
||||||
|
|
||||||
|
A bare FastAgent that wraps the Mnemosyne MCP server. Exists solely to
|
||||||
|
exercise the Mnemosyne MCP transport, tool registration, and round-trip
|
||||||
|
serialization — no production role.
|
||||||
|
|
||||||
|
Drive it from the CLI to confirm:
|
||||||
|
- search works against the running Mnemosyne (vector + fulltext + graph)
|
||||||
|
- get_chunk fetches full chunk text from S3
|
||||||
|
- list_libraries / list_collections / list_items return the expected shape
|
||||||
|
- get_health returns ok/degraded with the right dependency breakdown
|
||||||
|
|
||||||
|
When the Daedalus integration ships, the workspace_id parameter will be
|
||||||
|
injected by Daedalus's chat path (force-overwritten before the call leaves
|
||||||
|
Daedalus). This validator never sets it — meaning all calls go to the
|
||||||
|
GLOBAL scope (libraries with workspace_id IS NULL).
|
||||||
|
"""
|
||||||
|
from fast_agent import FastAgent
|
||||||
|
|
||||||
|
fast = FastAgent("Mnemosyne Validator", parse_cli_args=False)
|
||||||
|
|
||||||
|
|
||||||
|
@fast.agent(
|
||||||
|
name="mnemosyne_validator",
|
||||||
|
instruction="""You are a validator for the Mnemosyne knowledge base. Your
|
||||||
|
job is to exercise its MCP tools when asked, report what you saw, and surface
|
||||||
|
errors clearly.
|
||||||
|
|
||||||
|
You have direct access to Mnemosyne via these tools:
|
||||||
|
|
||||||
|
- search(query, library_uid?, library_type?, collection_uid?, limit?, rerank?, include_images?, search_types?)
|
||||||
|
Hybrid retrieval. Returns ranked chunks with text_preview (~500 chars),
|
||||||
|
chunk_uid, item_uid, item_title, library_type, score, source.
|
||||||
|
- get_chunk(chunk_uid)
|
||||||
|
Fetch the full text of a chunk by uid (typically obtained from search).
|
||||||
|
- list_libraries(limit?, offset?)
|
||||||
|
List libraries (uid, name, library_type, description).
|
||||||
|
- list_collections(library_uid?, limit?, offset?)
|
||||||
|
List collections, optionally filtered by parent library.
|
||||||
|
- list_items(collection_uid?, library_uid?, limit?, offset?)
|
||||||
|
List items (documents) with chunk_count, embedding_status, etc.
|
||||||
|
- get_health()
|
||||||
|
Health check: {status: ok|degraded|error, checks: {neo4j, s3, embedding}}.
|
||||||
|
|
||||||
|
When the user asks "what libraries exist", call list_libraries and report.
|
||||||
|
When they ask a research question, call search and surface chunk_uid + score
|
||||||
|
+ item_title for each candidate. If they want full text, call get_chunk.
|
||||||
|
Show raw structured output, not flowery prose — this is a validation tool,
|
||||||
|
not a chat assistant.
|
||||||
|
|
||||||
|
If a tool errors, paste the error message verbatim.""",
|
||||||
|
servers=["mnemosyne"],
|
||||||
|
)
|
||||||
|
async def mnemosyne_validator():
|
||||||
|
pass
|
||||||
32
validator/fastagent.config.yaml
Normal file
32
validator/fastagent.config.yaml
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
# Mnemosyne Validator — FastAgent + MCP configuration
|
||||||
|
#
|
||||||
|
# Secrets (api_key, MCP bearer tokens) live in fastagent.secrets.yaml
|
||||||
|
# (gitignored) and merge with this file at runtime.
|
||||||
|
|
||||||
|
# Local llama.cpp on Nyx (OpenAI-compatible). Override via
|
||||||
|
# fastagent.secrets.yaml if you want to point at a different model server.
|
||||||
|
default_model: openai.Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf
|
||||||
|
|
||||||
|
# Capabilities for the model — Pallas registers it with fast-agent's
|
||||||
|
# ModelDatabase using these values. vision: true so we can validate image
|
||||||
|
# round-trip later (search returns image candidates by default).
|
||||||
|
model_capabilities:
|
||||||
|
vision: true
|
||||||
|
context_window: 192000
|
||||||
|
max_output_tokens: 16384
|
||||||
|
|
||||||
|
# ── LLM Providers ───────────────────────────────────────────────────────────
|
||||||
|
openai:
|
||||||
|
base_url: "http://nyx.helu.ca:22079/v1"
|
||||||
|
|
||||||
|
# ── MCP Servers ─────────────────────────────────────────────────────────────
|
||||||
|
mcp:
|
||||||
|
servers:
|
||||||
|
# Mnemosyne MCP server — Streamable HTTP at /mcp.
|
||||||
|
# Default assumes the validator runs on the same host as Mnemosyne;
|
||||||
|
# override the URL in fastagent.secrets.yaml or via Ansible if remote.
|
||||||
|
mnemosyne:
|
||||||
|
transport: http
|
||||||
|
url: "http://localhost:22091/mcp/"
|
||||||
|
# Bearer token in fastagent.secrets.yaml (provisioned via
|
||||||
|
# `python manage.py create_mcp_token <user>`).
|
||||||
22
validator/fastagent.secrets.yaml.example
Normal file
22
validator/fastagent.secrets.yaml.example
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
# Mnemosyne Validator — secrets template
|
||||||
|
#
|
||||||
|
# Copy to fastagent.secrets.yaml and fill in real values. The .yaml is
|
||||||
|
# gitignored; the .yaml.example is committed.
|
||||||
|
|
||||||
|
# ── LLM provider keys ───────────────────────────────────────────────────────
|
||||||
|
# Local llama.cpp doesn't authenticate, but fast-agent requires the key field
|
||||||
|
# to be present. "0000" or any non-empty string is fine.
|
||||||
|
openai:
|
||||||
|
api_key: "0000"
|
||||||
|
|
||||||
|
# ── MCP server bearer tokens ────────────────────────────────────────────────
|
||||||
|
mcp:
|
||||||
|
servers:
|
||||||
|
mnemosyne:
|
||||||
|
headers:
|
||||||
|
# Mnemosyne MCP server requires a bearer token when MCP_REQUIRE_AUTH=True.
|
||||||
|
# Provision one with:
|
||||||
|
# cd ../mnemosyne
|
||||||
|
# python manage.py create_mcp_token --user <username> --name validator
|
||||||
|
# then paste the printed token here (it is shown once and not retrievable).
|
||||||
|
Authorization: "Bearer paste-mcp-token-here"
|
||||||
23
validator/pyproject.toml
Normal file
23
validator/pyproject.toml
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
[project]
|
||||||
|
name = "mnemosyne-validator"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "FastAgent + Pallas validator that talks to Mnemosyne's MCP server end-to-end"
|
||||||
|
requires-python = ">=3.13"
|
||||||
|
dependencies = [
|
||||||
|
"pallas-mcp @ git+ssh://git@git.helu.ca:22022/r/pallas.git",
|
||||||
|
"fast-agent-mcp>=0.6.10",
|
||||||
|
"pyyaml>=6.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.scripts]
|
||||||
|
mnemosyne-validator = "pallas.server:main"
|
||||||
|
|
||||||
|
[build-system]
|
||||||
|
requires = ["hatchling"]
|
||||||
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
|
[tool.hatch.metadata]
|
||||||
|
allow-direct-references = true
|
||||||
|
|
||||||
|
[tool.hatch.build.targets.wheel]
|
||||||
|
packages = ["agents"]
|
||||||
Reference in New Issue
Block a user