diff --git a/docker-compose.yaml b/docker-compose.yaml index 66eabef..39cc0f0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -24,10 +24,17 @@ # # Run: # docker compose up -d -# docker compose run --rm app migrate # one-shot DB migrate -# docker compose run --rm app setup # Neo4j indexes + library types +# +# The `init` sidecar (below) runs Postgres migrations, Neo4j index setup, +# and library-type seeding on every `up`. Long-running services wait for +# it via `depends_on: init: service_completed_successfully` — so a failure +# there (missing embedding model, dimension mismatch, unreachable DB) +# blocks the stack rather than letting it serve silent zero-result +# searches. The standalone `migrate` / `setup` entrypoint commands remain +# available for ad-hoc ops work. # ============================================================================= + services: # ── Static-file seeder: copies /app/staticfiles into the shared volume on # every `up`. Runs once and exits. Without this, the named volume is only @@ -41,6 +48,41 @@ services: - mnemosyne-static:/shared-static restart: "no" + # ── Init sidecar: one-shot Postgres migrate + Neo4j index setup + library + # type seed. Runs on every `up` and exits. Long-running services below + # depend on `service_completed_successfully`, so a failure here (no system + # embedding model configured, dimension mismatch, unreachable DB) blocks + # `app`/`mcp`/`worker` from starting — which is the whole point. All three + # commands are idempotent: re-running is a no-op unless state actually + # needs to change. + # + # This sidecar only needs Postgres, Neo4j, and logging env — no S3, no + # Celery, no LLM encryption key. Keep it that way. + init: + image: git.helu.ca/r/mnemosyne:latest + pull_policy: always + command: ["init"] + environment: + # Django core (settings import) + - DJANGO_SETTINGS_MODULE=mnemosyne.settings + - SECRET_KEY=${SECRET_KEY} + - DEBUG=${DEBUG} + - TIME_ZONE=${TIME_ZONE} + - LANGUAGE_CODE=${LANGUAGE_CODE} + # Postgres (migrate) + - APP_DB_NAME=${APP_DB_NAME} + - APP_DB_USER=${APP_DB_USER} + - APP_DB_PASSWORD=${APP_DB_PASSWORD} + - DB_HOST=${DB_HOST} + - DB_PORT=${DB_PORT} + # Neo4j (setup_neo4j_indexes + load_library_types) + - NEOMODEL_NEO4J_BOLT_URL=${NEOMODEL_NEO4J_BOLT_URL} + # Logging + - LOGGING_LEVEL=${LOGGING_LEVEL} + - DJANGO_LOGGING_LEVEL=${DJANGO_LOGGING_LEVEL} + restart: "no" + + # ── App: Django REST API + admin ────────────────────────────────────────── # Serves /library/api/*, /admin/, /live/, /ready/, /metrics. Enqueues # Celery tasks (hence CELERY_BROKER_URL is required here too — Django is @@ -103,6 +145,8 @@ services: depends_on: static-init: condition: service_completed_successfully + init: + condition: service_completed_successfully volumes: - mnemosyne-media:/app/media healthcheck: @@ -112,6 +156,7 @@ services: retries: 3 start_period: 30s + # ── MCP server: FastMCP Streamable HTTP at /mcp/ ─────────────────────────── # Read-only LLM-facing surface. Intentionally excluded: # CELERY_BROKER_URL — MCP must not enqueue tasks @@ -171,6 +216,9 @@ services: - LOGGING_LEVEL=${LOGGING_LEVEL} - DJANGO_LOGGING_LEVEL=${DJANGO_LOGGING_LEVEL} restart: unless-stopped + depends_on: + init: + condition: service_completed_successfully volumes: - mnemosyne-media:/app/media healthcheck: @@ -180,6 +228,7 @@ services: retries: 3 start_period: 30s + # ── Celery worker: embedding + ingest + batch queues ─────────────────────── # Consumer side of the queue. Needs the full S3 block (reads Daedalus's # bucket, writes to Mnemosyne's), the LLM API encryption key (ingest calls diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index eca079a..2af9b0d 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -55,6 +55,19 @@ case "$1" in python manage.py load_library_types ;; + init) + # Bundled one-shot init run by the `init` sidecar on every + # `docker compose up`. Idempotent: re-runs are no-ops unless migrations + # or indexes need to change. A non-zero exit here blocks `app`, `mcp`, + # and `worker` from starting, which is the point — we'd rather fail + # loudly than serve silent zero-result searches. + set -e + python manage.py migrate --noinput + python manage.py setup_neo4j_indexes + python manage.py load_library_types + ;; + + shell) # Drop into the management shell for ad-hoc work. exec python manage.py shell diff --git a/docs/PHASE_3_SEARCH_AND_RERANKING.md b/docs/PHASE_3_SEARCH_AND_RERANKING.md index e51608d..71c1a22 100644 --- a/docs/PHASE_3_SEARCH_AND_RERANKING.md +++ b/docs/PHASE_3_SEARCH_AND_RERANKING.md @@ -61,6 +61,22 @@ POST http://pan.helu.ca:8400/v1/rerank } ``` +> **`LLMApi.base_url` convention.** Every Mnemosyne service client +> (`EmbeddingClient`, `RerankerClient`, `vision.py`, `concepts.py`) +> treats `base_url` as the **OpenAI-style `/v1` root** and appends a +> path-only segment: `/embeddings`, `/rerank`, `/chat/completions`. +> So a single `LLMApi` row with `base_url=http://pan.helu.ca:8400/v1` +> serves both the embedding and the reranker endpoints — no per-purpose +> duplication needed. +> +> Get this wrong (e.g. set `base_url=http://pan.helu.ca:8400` with no +> `/v1`, or have a client prepend `/v1` locally) and you get a +> double-prefixed URL like `…/v1/v1/rerank` that 404s silently — +> `SearchService._rerank` catches the exception, the UI shows +> "Re-rank: Skipped", and the search falls back to raw RRF order. +> Check `results.reranker_skip_reason` on the search page for the +> specific error. + ## Deliverables ### 1. Search Service (`library/services/search.py`) diff --git a/docs/mnemosyne.html b/docs/mnemosyne.html index e4c990b..34c7aee 100644 --- a/docs/mnemosyne.html +++ b/docs/mnemosyne.html @@ -294,31 +294,37 @@ graph LR
// Chunk text+image embeddings (4096 dimensions, no pgvector limits!)
-CREATE VECTOR INDEX chunk_embedding FOR (c:Chunk)
+ Neo4j Indexes (managed by setup_neo4j_indexes)
+ Created by the init sidecar on every docker compose up. Vector dimensions come from the system embedding model's vector_dimensions field — the command fails if no model is configured. Current production model: Pan Synesis · qwen3-vl-embedding-2b · 2048d.
+ // Chunk text+image embeddings (dimensions read from system embedding model)
+CREATE VECTOR INDEX chunk_embedding_index FOR (c:Chunk)
ON (c.embedding) OPTIONS {indexConfig: {
- `vector.dimensions`: 4096,
+ `vector.dimensions`: 2048,
`vector.similarity_function`: 'cosine'
}}
// Concept embeddings for semantic concept search
-CREATE VECTOR INDEX concept_embedding FOR (con:Concept)
+CREATE VECTOR INDEX concept_embedding_index FOR (con:Concept)
ON (con.embedding) OPTIONS {indexConfig: {
- `vector.dimensions`: 4096,
+ `vector.dimensions`: 2048,
`vector.similarity_function`: 'cosine'
}}
// Image multimodal embeddings
-CREATE VECTOR INDEX image_embedding FOR (ie:ImageEmbedding)
+CREATE VECTOR INDEX image_embedding_index FOR (ie:ImageEmbedding)
ON (ie.embedding) OPTIONS {indexConfig: {
- `vector.dimensions`: 4096,
+ `vector.dimensions`: 2048,
`vector.similarity_function`: 'cosine'
}}
-// Full-text index for keyword/BM25-style search
-CREATE FULLTEXT INDEX chunk_fulltext FOR (c:Chunk) ON EACH [c.text_preview]
+// Full-text indexes (BM25-style keyword search)
+CREATE FULLTEXT INDEX chunk_text_fulltext FOR (c:Chunk) ON EACH [c.text_preview]
+CREATE FULLTEXT INDEX concept_name_fulltext FOR (c:Concept) ON EACH [c.name]
+CREATE FULLTEXT INDEX item_title_fulltext FOR (i:Item) ON EACH [i.title]
+CREATE FULLTEXT INDEX library_name_fulltext FOR (l:Library) ON EACH [l.name]
+ Changing the embedding model or dimensions is a re-embedding event. Drop + recreate the vector indexes (setup_neo4j_indexes --drop) and re-queue all content for embedding. Old vectors at the previous dimension remain on the nodes until overwritten but are no longer indexed.
Cosine similarity via Neo4j vector index on Chunk and ImageEmbedding nodes.
CALL db.index.vector.queryNodes(
- 'chunk_embedding', 30,
+ 'chunk_embedding_index', 30,
$query_vector
) YIELD node, score
WHERE score > $threshold
+
Neo4j native full-text index for keyword matching (BM25-equivalent).
CALL db.index.fulltext.queryNodes(
- 'chunk_fulltext',
+ 'chunk_text_fulltext',
$query_text
) YIELD node, score
+