Robert Helewka
c485a8560c
Adds DAEDALUS_S3_* settings (read-only credentials for the Daedalus bucket)
and a small `daedalus_s3.py` helper that fetches a file from Daedalus's
bucket and writes it into Mnemosyne's bucket via default_storage.
Adds the Celery task `library.tasks.ingest_from_daedalus`. Given an
IngestJob row, it:
1. Resolves the target Library (by library_uid).
2. Supersedes a prior Item with the same source_ref but different
content_hash by deleting the old Item + chunks first.
3. Fetches from Daedalus S3, copies into items/{item_uid}/original.{ext}.
4. Creates the Item node, links it to a default Collection.
5. Runs the existing EmbeddingPipeline.process_item.
6. Marks the job completed with chunks/concepts counts.
Failures retry up to 3× with exponential backoff; final failure marks
the job failed with the exception text. Routed to the embedding queue
so single-worker setups must consume it.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
85 lines
2.6 KiB
Plaintext
85 lines
2.6 KiB
Plaintext
# =============================================================================
|
|
# Mnemosyne Django Environment Variables
|
|
# =============================================================================
|
|
# Copy this file to .env and configure for your environment
|
|
# This file contains all variables read by Django settings.py
|
|
|
|
# --- Security ---
|
|
SECRET_KEY=change-me-to-a-real-secret-key
|
|
DEBUG=True
|
|
ALLOWED_HOSTS=localhost,127.0.0.1,mnemosyne.ouranos.helu.ca
|
|
CSRF_TRUSTED_ORIGINS=http://localhost:8000,https://mnemosyne.ouranos.helu.ca
|
|
|
|
# --- PostgreSQL Database ---
|
|
APP_DB_NAME=mnemosyne
|
|
APP_DB_USER=mnemosyne
|
|
APP_DB_PASSWORD=password
|
|
DB_HOST=portia.incus
|
|
DB_PORT=5432
|
|
|
|
# --- Neo4j Graph Database ---
|
|
NEOMODEL_NEO4J_BOLT_URL=bolt://neo4j:password@ariel.incus:25554
|
|
|
|
# --- Memcached ---
|
|
KVDB_LOCATION=127.0.0.1:11211
|
|
KVDB_PREFIX=mnemosyne
|
|
|
|
# --- Celery / RabbitMQ ---
|
|
CELERY_BROKER_URL=amqp://mnemosyne:password@oberon.incus:5672/mnemosyne
|
|
CELERY_RESULT_BACKEND=rpc://
|
|
CELERY_TASK_ALWAYS_EAGER=False
|
|
|
|
# --- S3 Storage (Incus bucket, MinIO-backed) ---
|
|
AWS_ACCESS_KEY_ID=
|
|
AWS_SECRET_ACCESS_KEY=
|
|
AWS_STORAGE_BUCKET_NAME=mnemosyne-content
|
|
AWS_S3_ENDPOINT_URL=
|
|
AWS_S3_USE_SSL=False
|
|
AWS_S3_VERIFY=False
|
|
AWS_S3_REGION_NAME=us-east-1
|
|
# Set to True to use local FileSystemStorage instead of S3 (dev/test)
|
|
USE_LOCAL_STORAGE=True
|
|
|
|
# --- Daedalus S3 (cross-bucket reads for ingest) ---
|
|
# Mnemosyne ingests files from the Daedalus S3 bucket. These vars
|
|
# configure read access; the file is copied into AWS_STORAGE_BUCKET_NAME
|
|
# (Mnemosyne's own bucket) by the ingest Celery task.
|
|
DAEDALUS_S3_ENDPOINT_URL=
|
|
DAEDALUS_S3_ACCESS_KEY_ID=
|
|
DAEDALUS_S3_SECRET_ACCESS_KEY=
|
|
DAEDALUS_S3_BUCKET_NAME=daedalus
|
|
DAEDALUS_S3_REGION_NAME=us-east-1
|
|
DAEDALUS_S3_USE_SSL=False
|
|
DAEDALUS_S3_VERIFY=True
|
|
|
|
# --- MCP Server ---
|
|
# Set to False for internal-only deployments where the MCP transport is
|
|
# already on a trusted network (10.10.0.0/24).
|
|
MCP_REQUIRE_AUTH=True
|
|
|
|
# --- Email (smtp4dev on Oberon) ---
|
|
EMAIL_HOST=oberon.incus
|
|
EMAIL_PORT=22025
|
|
EMAIL_USE_TLS=False
|
|
|
|
# --- LLM API Encryption ---
|
|
# Encryption key for LLM API keys stored in the database
|
|
# Generate with: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
|
|
LLM_API_SECRETS_ENCRYPTION_KEY=
|
|
|
|
# --- Embedding Pipeline (Phase 2) ---
|
|
# Batch size for embedding API calls (smaller for local GPU, larger for cloud)
|
|
EMBEDDING_BATCH_SIZE=8
|
|
# Timeout in seconds for embedding API requests
|
|
EMBEDDING_TIMEOUT=120
|
|
|
|
# --- Logging ---
|
|
# Valid levels: DEBUG, INFO, WARNING, ERROR, CRITICAL
|
|
LOGGING_LEVEL=INFO
|
|
CELERY_LOGGING_LEVEL=INFO
|
|
DJANGO_LOGGING_LEVEL=WARNING
|
|
|
|
# --- Localization ---
|
|
TIME_ZONE=UTC
|
|
LANGUAGE_CODE=en-us
|