Compare commits
12 Commits
feat/admin
...
feat/kairo
| Author | SHA1 | Date | |
|---|---|---|---|
| d01afd6203 | |||
| 840b9435a3 | |||
| 6120e9cd1f | |||
| 31a98b4f3a | |||
| 3394726ca1 | |||
| 03e3155bd6 | |||
| 929a3c8c3c | |||
| 2af72d6e82 | |||
| 70b1fc510b | |||
| 46ca2a934d | |||
| dd06f923cd | |||
| 539d9b6c34 |
13
.env.example
13
.env.example
@@ -91,6 +91,19 @@ SPELUNKER_S3_REGION_NAME=us-east-1
|
|||||||
SPELUNKER_S3_USE_SSL=True
|
SPELUNKER_S3_USE_SSL=True
|
||||||
SPELUNKER_S3_VERIFY=True
|
SPELUNKER_S3_VERIFY=True
|
||||||
|
|
||||||
|
# --- Kairos S3 (cross-bucket reads for ingest, source="kairos-mail") ---
|
||||||
|
# Consumed by: worker only
|
||||||
|
# Kairos renders synced mail to text documents in its own bucket and posts
|
||||||
|
# ingest requests with source="kairos-mail". These creds should be scoped
|
||||||
|
# read-only to the Kairos bucket in your secret manager.
|
||||||
|
KAIROS_S3_ENDPOINT_URL=https://nyx.helu.ca:8555
|
||||||
|
KAIROS_S3_ACCESS_KEY_ID=
|
||||||
|
KAIROS_S3_SECRET_ACCESS_KEY=
|
||||||
|
KAIROS_S3_BUCKET_NAME=kairos
|
||||||
|
KAIROS_S3_REGION_NAME=us-east-1
|
||||||
|
KAIROS_S3_USE_SSL=True
|
||||||
|
KAIROS_S3_VERIFY=True
|
||||||
|
|
||||||
# --- Celery / RabbitMQ (Oberon) ---------------------------------------------
|
# --- Celery / RabbitMQ (Oberon) ---------------------------------------------
|
||||||
# Consumed by: app (producer), worker (consumer). NOT mcp.
|
# Consumed by: app (producer), worker (consumer). NOT mcp.
|
||||||
# Remember to percent-encode any password characters that have meaning in a
|
# Remember to percent-encode any password characters that have meaning in a
|
||||||
|
|||||||
@@ -3,8 +3,6 @@ name: CVE Scan & Docker Build
|
|||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
pull_request:
|
|
||||||
branches: [main]
|
|
||||||
|
|
||||||
env:
|
env:
|
||||||
REGISTRY: git.helu.ca
|
REGISTRY: git.helu.ca
|
||||||
@@ -75,8 +73,6 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||||
tags: |
|
tags: |
|
||||||
type=ref,event=branch
|
|
||||||
type=ref,event=pr
|
|
||||||
type=sha,prefix=
|
type=sha,prefix=
|
||||||
type=raw,value=latest,enable=${{ gitea.ref == 'refs/heads/main' }}
|
type=raw,value=latest,enable=${{ gitea.ref == 'refs/heads/main' }}
|
||||||
|
|
||||||
|
|||||||
@@ -346,6 +346,13 @@ services:
|
|||||||
- SPELUNKER_S3_REGION_NAME=${SPELUNKER_S3_REGION_NAME}
|
- SPELUNKER_S3_REGION_NAME=${SPELUNKER_S3_REGION_NAME}
|
||||||
- SPELUNKER_S3_USE_SSL=${SPELUNKER_S3_USE_SSL}
|
- SPELUNKER_S3_USE_SSL=${SPELUNKER_S3_USE_SSL}
|
||||||
- SPELUNKER_S3_VERIFY=${SPELUNKER_S3_VERIFY}
|
- SPELUNKER_S3_VERIFY=${SPELUNKER_S3_VERIFY}
|
||||||
|
- KAIROS_S3_ENDPOINT_URL=${KAIROS_S3_ENDPOINT_URL}
|
||||||
|
- KAIROS_S3_ACCESS_KEY_ID=${KAIROS_S3_ACCESS_KEY_ID}
|
||||||
|
- KAIROS_S3_SECRET_ACCESS_KEY=${KAIROS_S3_SECRET_ACCESS_KEY}
|
||||||
|
- KAIROS_S3_BUCKET_NAME=${KAIROS_S3_BUCKET_NAME}
|
||||||
|
- KAIROS_S3_REGION_NAME=${KAIROS_S3_REGION_NAME}
|
||||||
|
- KAIROS_S3_USE_SSL=${KAIROS_S3_USE_SSL}
|
||||||
|
- KAIROS_S3_VERIFY=${KAIROS_S3_VERIFY}
|
||||||
# Celery / RabbitMQ
|
# Celery / RabbitMQ
|
||||||
- CELERY_BROKER_URL=${CELERY_BROKER_URL}
|
- CELERY_BROKER_URL=${CELERY_BROKER_URL}
|
||||||
- CELERY_RESULT_BACKEND=${CELERY_RESULT_BACKEND}
|
- CELERY_RESULT_BACKEND=${CELERY_RESULT_BACKEND}
|
||||||
@@ -371,7 +378,12 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- media:/mnt/media
|
- media:/mnt/media
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "celery", "-A", "mnemosyne", "inspect", "ping", "-d", "celery@$$HOSTNAME"]
|
# No -d destination: exec-form CMD has no shell, so $$HOSTNAME never
|
||||||
|
# expanded and the literal "celery@$HOSTNAME" matched no node → every
|
||||||
|
# check failed. There's one worker per container, so an unfiltered ping
|
||||||
|
# (any node replies = healthy) is correct. -t gives the reply room to
|
||||||
|
# round-trip through the broker on Oberon (~450ms observed) under jitter.
|
||||||
|
test: ["CMD", "celery", "-A", "mnemosyne", "inspect", "ping", "-t", "8"]
|
||||||
interval: 60s
|
interval: 60s
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
retries: 3
|
retries: 3
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ across users.
|
|||||||
import logging
|
import logging
|
||||||
|
|
||||||
from neomodel import db
|
from neomodel import db
|
||||||
|
from neomodel.exceptions import UniqueProperty
|
||||||
from rest_framework import status
|
from rest_framework import status
|
||||||
from rest_framework.decorators import api_view, permission_classes
|
from rest_framework.decorators import api_view, permission_classes
|
||||||
from rest_framework.permissions import IsAuthenticated
|
from rest_framework.permissions import IsAuthenticated
|
||||||
@@ -85,7 +86,10 @@ def workspace_create(request):
|
|||||||
data["workspace_id"], request.user.username,
|
data["workspace_id"], request.user.username,
|
||||||
)
|
)
|
||||||
return Response(
|
return Response(
|
||||||
{"detail": "Workspace id is already in use."},
|
{
|
||||||
|
"detail": "Workspace id is already in use.",
|
||||||
|
"code": "owner_conflict",
|
||||||
|
},
|
||||||
status=status.HTTP_409_CONFLICT,
|
status=status.HTTP_409_CONFLICT,
|
||||||
)
|
)
|
||||||
if existing.library_type != data["library_type"]:
|
if existing.library_type != data["library_type"]:
|
||||||
@@ -95,7 +99,8 @@ def workspace_create(request):
|
|||||||
"library_type is immutable for an existing workspace "
|
"library_type is immutable for an existing workspace "
|
||||||
f"(have '{existing.library_type}', "
|
f"(have '{existing.library_type}', "
|
||||||
f"got '{data['library_type']}')."
|
f"got '{data['library_type']}')."
|
||||||
)
|
),
|
||||||
|
"code": "library_type_immutable",
|
||||||
},
|
},
|
||||||
status=status.HTTP_409_CONFLICT,
|
status=status.HTTP_409_CONFLICT,
|
||||||
)
|
)
|
||||||
@@ -120,7 +125,29 @@ def workspace_create(request):
|
|||||||
reranker_instruction=defaults["reranker_instruction"],
|
reranker_instruction=defaults["reranker_instruction"],
|
||||||
llm_context_prompt=defaults["llm_context_prompt"],
|
llm_context_prompt=defaults["llm_context_prompt"],
|
||||||
)
|
)
|
||||||
lib.save()
|
try:
|
||||||
|
lib.save()
|
||||||
|
except UniqueProperty:
|
||||||
|
# Library.name is globally unique. A name collision here almost always
|
||||||
|
# means an orphaned Library survived a failed Daedalus workspace delete
|
||||||
|
# (the old node kept the name), and the recreate under a new
|
||||||
|
# workspace_id now clashes. Surface a clean 409 instead of a 500 so
|
||||||
|
# Daedalus can record + report it; the operator clears the orphan
|
||||||
|
# (admin delete) or renames the workspace.
|
||||||
|
logger.warning(
|
||||||
|
"workspace_create name_conflict workspace_id=%s name=%s",
|
||||||
|
data["workspace_id"], data["name"],
|
||||||
|
)
|
||||||
|
return Response(
|
||||||
|
{
|
||||||
|
"detail": (
|
||||||
|
f"A library named '{data['name']}' already exists in "
|
||||||
|
"Mnemosyne."
|
||||||
|
),
|
||||||
|
"code": "name_conflict",
|
||||||
|
},
|
||||||
|
status=status.HTTP_409_CONFLICT,
|
||||||
|
)
|
||||||
logger.info(
|
logger.info(
|
||||||
"Workspace created workspace_id=%s library_uid=%s library_type=%s",
|
"Workspace created workspace_id=%s library_uid=%s library_type=%s",
|
||||||
data["workspace_id"], lib.uid, lib.library_type,
|
data["workspace_id"], lib.uid, lib.library_type,
|
||||||
|
|||||||
@@ -25,14 +25,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="flex gap-2">
|
<div class="flex gap-2">
|
||||||
<a href="{% url 'library:library-edit' uid=library.uid %}" class="btn btn-sm btn-outline">Edit</a>
|
<a href="{% url 'library:library-edit' uid=library.uid %}" class="btn btn-sm btn-outline">Edit</a>
|
||||||
{% if library.workspace_id %}
|
|
||||||
<button type="button" class="btn btn-sm btn-error btn-outline" disabled
|
|
||||||
title="This library is managed by Daedalus. Delete it from the Daedalus workspace, not here.">
|
|
||||||
Delete
|
|
||||||
</button>
|
|
||||||
{% else %}
|
|
||||||
<a href="{% url 'library:library-delete' uid=library.uid %}" class="btn btn-sm btn-error btn-outline">Delete</a>
|
<a href="{% url 'library:library-delete' uid=library.uid %}" class="btn btn-sm btn-error btn-outline">Delete</a>
|
||||||
{% endif %}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -43,8 +36,11 @@
|
|||||||
<div class="text-sm opacity-80">
|
<div class="text-sm opacity-80">
|
||||||
This library was created for Daedalus workspace
|
This library was created for Daedalus workspace
|
||||||
<code class="font-mono">{{ library.workspace_id }}</code>.
|
<code class="font-mono">{{ library.workspace_id }}</code>.
|
||||||
Items here are owned by the workspace; deleting the workspace in
|
Normally you manage it from Daedalus. Deleting it here removes its
|
||||||
Daedalus will remove this library. Do not delete it manually.
|
embedded content from Mnemosyne, but the source files still live in
|
||||||
|
Daedalus — it will be recreated and re-embedded on the next sync.
|
||||||
|
Use Delete to clear an orphaned library that is blocking workspace
|
||||||
|
re-registration.
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -15,6 +15,18 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<form method="get" class="mb-4 flex flex-wrap gap-3 items-end">
|
||||||
|
<div class="form-control">
|
||||||
|
<label class="label"><span class="label-text">Scope</span></label>
|
||||||
|
<select name="scope" class="select select-bordered select-sm">
|
||||||
|
<option value="all" {% if scope == "all" %}selected{% endif %}>All libraries</option>
|
||||||
|
<option value="global" {% if scope == "global" %}selected{% endif %}>Global only</option>
|
||||||
|
<option value="daedalus" {% if scope == "daedalus" %}selected{% endif %}>Daedalus workspaces only</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<button type="submit" class="btn btn-sm btn-outline">Filter</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
{% if error %}
|
{% if error %}
|
||||||
<div class="alert alert-warning mb-4">
|
<div class="alert alert-warning mb-4">
|
||||||
<span>{{ error }}</span>
|
<span>{{ error }}</span>
|
||||||
@@ -53,8 +65,12 @@
|
|||||||
{% else %}
|
{% else %}
|
||||||
{% if not error %}
|
{% if not error %}
|
||||||
<div class="text-center py-12 opacity-60">
|
<div class="text-center py-12 opacity-60">
|
||||||
|
{% if scope == "all" %}
|
||||||
<p class="text-lg">No libraries yet.</p>
|
<p class="text-lg">No libraries yet.</p>
|
||||||
<p class="mt-2">Create your first library to get started.</p>
|
<p class="mt-2">Create your first library to get started.</p>
|
||||||
|
{% else %}
|
||||||
|
<p class="text-lg">No libraries match this filter.</p>
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|||||||
@@ -48,30 +48,3 @@ class ConceptExtractionParsingTests(TestCase):
|
|||||||
result = self.extractor._parse_concept_response(response)
|
result = self.extractor._parse_concept_response(response)
|
||||||
self.assertEqual(len(result), 1)
|
self.assertEqual(len(result), 1)
|
||||||
self.assertEqual(result[0]["name"], "valid")
|
self.assertEqual(result[0]["name"], "valid")
|
||||||
|
|
||||||
|
|
||||||
class SampleIndexSelectionTests(TestCase):
|
|
||||||
"""Tests for sample index selection."""
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
self.extractor = ConceptExtractor(MagicMock())
|
|
||||||
|
|
||||||
def test_small_total_returns_all(self):
|
|
||||||
indices = self.extractor._select_sample_indices(5, max_samples=10)
|
|
||||||
self.assertEqual(indices, [0, 1, 2, 3, 4])
|
|
||||||
|
|
||||||
def test_equal_total_returns_all(self):
|
|
||||||
indices = self.extractor._select_sample_indices(10, max_samples=10)
|
|
||||||
self.assertEqual(indices, list(range(10)))
|
|
||||||
|
|
||||||
def test_large_total_returns_max_samples(self):
|
|
||||||
indices = self.extractor._select_sample_indices(100, max_samples=10)
|
|
||||||
self.assertEqual(len(indices), 10)
|
|
||||||
# Should be evenly spaced
|
|
||||||
self.assertEqual(indices[0], 0)
|
|
||||||
self.assertEqual(indices[-1], 90)
|
|
||||||
|
|
||||||
def test_returns_integers(self):
|
|
||||||
indices = self.extractor._select_sample_indices(50, max_samples=7)
|
|
||||||
for idx in indices:
|
|
||||||
self.assertIsInstance(idx, int)
|
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ class EmbeddingPipelineInitTests(TestCase):
|
|||||||
class PipelineItemNotFoundTests(TestCase):
|
class PipelineItemNotFoundTests(TestCase):
|
||||||
"""Tests for handling missing items."""
|
"""Tests for handling missing items."""
|
||||||
|
|
||||||
@patch("library.services.pipeline.Item")
|
@patch("library.models.Item")
|
||||||
def test_process_nonexistent_item_raises(self, mock_item_cls):
|
def test_process_nonexistent_item_raises(self, mock_item_cls):
|
||||||
mock_item_cls.nodes.get.side_effect = Exception("Not found")
|
mock_item_cls.nodes.get.side_effect = Exception("Not found")
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ class PipelineItemNotFoundTests(TestCase):
|
|||||||
pipeline.process_item("nonexistent-uid")
|
pipeline.process_item("nonexistent-uid")
|
||||||
self.assertIn("Item not found", str(ctx.exception))
|
self.assertIn("Item not found", str(ctx.exception))
|
||||||
|
|
||||||
@patch("library.services.pipeline.Item")
|
@patch("library.models.Item")
|
||||||
def test_reprocess_nonexistent_item_raises(self, mock_item_cls):
|
def test_reprocess_nonexistent_item_raises(self, mock_item_cls):
|
||||||
mock_item_cls.nodes.get.side_effect = Exception("Not found")
|
mock_item_cls.nodes.get.side_effect = Exception("Not found")
|
||||||
|
|
||||||
@@ -69,9 +69,9 @@ class PipelineItemNotFoundTests(TestCase):
|
|||||||
class PipelineNoEmbeddingModelTests(TestCase):
|
class PipelineNoEmbeddingModelTests(TestCase):
|
||||||
"""Tests for handling missing system embedding model."""
|
"""Tests for handling missing system embedding model."""
|
||||||
|
|
||||||
@patch("library.services.pipeline.LLMModel")
|
@patch("llm_manager.models.LLMModel")
|
||||||
@patch("library.services.pipeline.default_storage")
|
@patch("library.services.pipeline.default_storage")
|
||||||
@patch("library.services.pipeline.DocumentParser")
|
@patch("library.services.parsers.DocumentParser")
|
||||||
def test_no_embedding_model_raises(self, mock_parser, mock_storage, mock_llm):
|
def test_no_embedding_model_raises(self, mock_parser, mock_storage, mock_llm):
|
||||||
"""Pipeline raises ValueError if no system embedding model is configured."""
|
"""Pipeline raises ValueError if no system embedding model is configured."""
|
||||||
mock_llm.get_system_embedding_model.return_value = None
|
mock_llm.get_system_embedding_model.return_value = None
|
||||||
@@ -86,7 +86,7 @@ class PipelineNoEmbeddingModelTests(TestCase):
|
|||||||
mock_item.chunks.all.return_value = []
|
mock_item.chunks.all.return_value = []
|
||||||
mock_item.images.all.return_value = []
|
mock_item.images.all.return_value = []
|
||||||
|
|
||||||
with patch("library.services.pipeline.Item") as mock_item_cls:
|
with patch("library.models.Item") as mock_item_cls:
|
||||||
mock_item_cls.nodes.get.return_value = mock_item
|
mock_item_cls.nodes.get.return_value = mock_item
|
||||||
|
|
||||||
# Mock S3 read
|
# Mock S3 read
|
||||||
@@ -166,11 +166,11 @@ class PipelineVisionStageTests(TestCase):
|
|||||||
item.images.all.return_value = []
|
item.images.all.return_value = []
|
||||||
return item
|
return item
|
||||||
|
|
||||||
@patch("library.services.pipeline.ConceptExtractor")
|
@patch("library.services.concepts.ConceptExtractor")
|
||||||
@patch("library.services.pipeline.EmbeddingClient")
|
@patch("library.services.embedding_client.EmbeddingClient")
|
||||||
@patch("library.services.pipeline.ContentTypeChunker")
|
@patch("library.services.chunker.ContentTypeChunker")
|
||||||
@patch("library.services.pipeline.DocumentParser")
|
@patch("library.services.parsers.DocumentParser")
|
||||||
@patch("library.services.pipeline.LLMModel")
|
@patch("llm_manager.models.LLMModel")
|
||||||
@patch("library.services.pipeline.default_storage")
|
@patch("library.services.pipeline.default_storage")
|
||||||
def test_no_vision_model_marks_images_skipped(
|
def test_no_vision_model_marks_images_skipped(
|
||||||
self, mock_storage, mock_llm, mock_parser_cls,
|
self, mock_storage, mock_llm, mock_parser_cls,
|
||||||
@@ -227,12 +227,12 @@ class PipelineVisionStageTests(TestCase):
|
|||||||
img_node.save.assert_called()
|
img_node.save.assert_called()
|
||||||
self.assertEqual(result["images_analyzed"], 0)
|
self.assertEqual(result["images_analyzed"], 0)
|
||||||
|
|
||||||
@patch("library.services.pipeline.VisionAnalyzer")
|
@patch("library.services.vision.VisionAnalyzer")
|
||||||
@patch("library.services.pipeline.ConceptExtractor")
|
@patch("library.services.concepts.ConceptExtractor")
|
||||||
@patch("library.services.pipeline.EmbeddingClient")
|
@patch("library.services.embedding_client.EmbeddingClient")
|
||||||
@patch("library.services.pipeline.ContentTypeChunker")
|
@patch("library.services.chunker.ContentTypeChunker")
|
||||||
@patch("library.services.pipeline.DocumentParser")
|
@patch("library.services.parsers.DocumentParser")
|
||||||
@patch("library.services.pipeline.LLMModel")
|
@patch("llm_manager.models.LLMModel")
|
||||||
@patch("library.services.pipeline.default_storage")
|
@patch("library.services.pipeline.default_storage")
|
||||||
def test_vision_model_triggers_analysis(
|
def test_vision_model_triggers_analysis(
|
||||||
self, mock_storage, mock_llm, mock_parser_cls,
|
self, mock_storage, mock_llm, mock_parser_cls,
|
||||||
@@ -287,7 +287,7 @@ class PipelineVisionStageTests(TestCase):
|
|||||||
mock_vision_cls.assert_called_once_with(mock_vision_model, user=None)
|
mock_vision_cls.assert_called_once_with(mock_vision_model, user=None)
|
||||||
mock_analyzer.analyze_images.assert_called_once()
|
mock_analyzer.analyze_images.assert_called_once()
|
||||||
|
|
||||||
@patch("library.services.pipeline.LLMModel")
|
@patch("llm_manager.models.LLMModel")
|
||||||
def test_no_images_skips_vision_entirely(self, mock_llm):
|
def test_no_images_skips_vision_entirely(self, mock_llm):
|
||||||
"""When there are no images, vision stage is a no-op regardless of model."""
|
"""When there are no images, vision stage is a no-op regardless of model."""
|
||||||
mock_vision_model = MagicMock()
|
mock_vision_model = MagicMock()
|
||||||
@@ -309,10 +309,10 @@ class PipelineVisionStageTests(TestCase):
|
|||||||
patch.object(pipeline, "_store_chunks", return_value=[]), \
|
patch.object(pipeline, "_store_chunks", return_value=[]), \
|
||||||
patch.object(pipeline, "_store_images", return_value=[]), \
|
patch.object(pipeline, "_store_images", return_value=[]), \
|
||||||
patch.object(pipeline, "_associate_images_with_chunks"), \
|
patch.object(pipeline, "_associate_images_with_chunks"), \
|
||||||
patch("library.services.pipeline.DocumentParser") as mock_parser_cls, \
|
patch("library.services.parsers.DocumentParser") as mock_parser_cls, \
|
||||||
patch("library.services.pipeline.ContentTypeChunker") as mock_chunker_cls, \
|
patch("library.services.chunker.ContentTypeChunker") as mock_chunker_cls, \
|
||||||
patch("library.services.pipeline.EmbeddingClient"), \
|
patch("library.services.embedding_client.EmbeddingClient"), \
|
||||||
patch("library.services.pipeline.VisionAnalyzer") as mock_vision_cls:
|
patch("library.services.vision.VisionAnalyzer") as mock_vision_cls:
|
||||||
|
|
||||||
mock_parser = MagicMock()
|
mock_parser = MagicMock()
|
||||||
mock_parser.parse_bytes.return_value = MagicMock(images=[], text_blocks=[])
|
mock_parser.parse_bytes.return_value = MagicMock(images=[], text_blocks=[])
|
||||||
|
|||||||
@@ -100,7 +100,7 @@ class SearchAPIResponseTest(TestCase):
|
|||||||
self.client = APIClient()
|
self.client = APIClient()
|
||||||
self.client.force_authenticate(user=self.user)
|
self.client.force_authenticate(user=self.user)
|
||||||
|
|
||||||
@patch("library.api.views.SearchService")
|
@patch("library.services.search.SearchService")
|
||||||
def test_successful_search_response_format(self, MockService):
|
def test_successful_search_response_format(self, MockService):
|
||||||
"""Successful search returns expected JSON structure."""
|
"""Successful search returns expected JSON structure."""
|
||||||
mock_response = SearchResponse(
|
mock_response = SearchResponse(
|
||||||
@@ -159,7 +159,7 @@ class SearchAPIResponseTest(TestCase):
|
|||||||
self.assertEqual(image["image_uid"], "img1")
|
self.assertEqual(image["image_uid"], "img1")
|
||||||
self.assertEqual(image["image_type"], "diagram")
|
self.assertEqual(image["image_type"], "diagram")
|
||||||
|
|
||||||
@patch("library.api.views.SearchService")
|
@patch("library.services.search.SearchService")
|
||||||
def test_vector_only_endpoint(self, MockService):
|
def test_vector_only_endpoint(self, MockService):
|
||||||
"""Vector-only endpoint sets correct search types."""
|
"""Vector-only endpoint sets correct search types."""
|
||||||
mock_response = SearchResponse(
|
mock_response = SearchResponse(
|
||||||
@@ -184,7 +184,7 @@ class SearchAPIResponseTest(TestCase):
|
|||||||
self.assertEqual(call_args.search_types, ["vector"])
|
self.assertEqual(call_args.search_types, ["vector"])
|
||||||
self.assertFalse(call_args.rerank)
|
self.assertFalse(call_args.rerank)
|
||||||
|
|
||||||
@patch("library.api.views.SearchService")
|
@patch("library.services.search.SearchService")
|
||||||
def test_fulltext_only_endpoint(self, MockService):
|
def test_fulltext_only_endpoint(self, MockService):
|
||||||
"""Fulltext-only endpoint sets correct search types."""
|
"""Fulltext-only endpoint sets correct search types."""
|
||||||
mock_response = SearchResponse(
|
mock_response = SearchResponse(
|
||||||
@@ -208,7 +208,7 @@ class SearchAPIResponseTest(TestCase):
|
|||||||
self.assertEqual(call_args.search_types, ["fulltext"])
|
self.assertEqual(call_args.search_types, ["fulltext"])
|
||||||
self.assertFalse(call_args.rerank)
|
self.assertFalse(call_args.rerank)
|
||||||
|
|
||||||
@patch("library.api.views.SearchService")
|
@patch("library.services.search.SearchService")
|
||||||
def test_reranker_skip_reason_surfaced_in_json(self, MockService):
|
def test_reranker_skip_reason_surfaced_in_json(self, MockService):
|
||||||
"""``reranker_skip_reason`` propagates through the JSON API."""
|
"""``reranker_skip_reason`` propagates through the JSON API."""
|
||||||
mock_response = SearchResponse(
|
mock_response = SearchResponse(
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ class AllLibraryUidsHelperTests(TestCase):
|
|||||||
|
|
||||||
def test_returns_empty_when_neo4j_unavailable(self):
|
def test_returns_empty_when_neo4j_unavailable(self):
|
||||||
"""Helper must not touch ``Library.nodes`` if Neo4j is down."""
|
"""Helper must not touch ``Library.nodes`` if Neo4j is down."""
|
||||||
with patch("library.views.neo4j_available", return_value=False):
|
with patch("library.utils.neo4j_available", return_value=False):
|
||||||
self.assertEqual(views._all_library_uids(), [])
|
self.assertEqual(views._all_library_uids(), [])
|
||||||
|
|
||||||
def test_returns_every_library_uid(self):
|
def test_returns_every_library_uid(self):
|
||||||
@@ -62,7 +62,7 @@ class AllLibraryUidsHelperTests(TestCase):
|
|||||||
fake_nodes.all.return_value = fake_libs
|
fake_nodes.all.return_value = fake_libs
|
||||||
fake_library_cls = SimpleNamespace(nodes=fake_nodes)
|
fake_library_cls = SimpleNamespace(nodes=fake_nodes)
|
||||||
|
|
||||||
with patch("library.views.neo4j_available", return_value=True), \
|
with patch("library.utils.neo4j_available", return_value=True), \
|
||||||
patch.dict("sys.modules", {"library.models": SimpleNamespace(Library=fake_library_cls)}):
|
patch.dict("sys.modules", {"library.models": SimpleNamespace(Library=fake_library_cls)}):
|
||||||
result = views._all_library_uids()
|
result = views._all_library_uids()
|
||||||
|
|
||||||
@@ -83,7 +83,7 @@ class AllLibraryUidsHelperTests(TestCase):
|
|||||||
fake_nodes.all.return_value = fake_libs
|
fake_nodes.all.return_value = fake_libs
|
||||||
fake_library_cls = SimpleNamespace(nodes=fake_nodes)
|
fake_library_cls = SimpleNamespace(nodes=fake_nodes)
|
||||||
|
|
||||||
with patch("library.views.neo4j_available", return_value=True), \
|
with patch("library.utils.neo4j_available", return_value=True), \
|
||||||
patch.dict("sys.modules", {"library.models": SimpleNamespace(Library=fake_library_cls)}):
|
patch.dict("sys.modules", {"library.models": SimpleNamespace(Library=fake_library_cls)}):
|
||||||
result = views._all_library_uids()
|
result = views._all_library_uids()
|
||||||
|
|
||||||
@@ -95,7 +95,7 @@ class AllLibraryUidsHelperTests(TestCase):
|
|||||||
fake_nodes.all.side_effect = RuntimeError("neo4j blew up")
|
fake_nodes.all.side_effect = RuntimeError("neo4j blew up")
|
||||||
fake_library_cls = SimpleNamespace(nodes=fake_nodes)
|
fake_library_cls = SimpleNamespace(nodes=fake_nodes)
|
||||||
|
|
||||||
with patch("library.views.neo4j_available", return_value=True), \
|
with patch("library.utils.neo4j_available", return_value=True), \
|
||||||
patch.dict("sys.modules", {"library.models": SimpleNamespace(Library=fake_library_cls)}):
|
patch.dict("sys.modules", {"library.models": SimpleNamespace(Library=fake_library_cls)}):
|
||||||
self.assertEqual(views._all_library_uids(), [])
|
self.assertEqual(views._all_library_uids(), [])
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from django.test import TestCase, override_settings
|
|||||||
class EmbedItemTaskTests(TestCase):
|
class EmbedItemTaskTests(TestCase):
|
||||||
"""Tests for the embed_item task."""
|
"""Tests for the embed_item task."""
|
||||||
|
|
||||||
@patch("library.tasks.EmbeddingPipeline")
|
@patch("library.services.pipeline.EmbeddingPipeline")
|
||||||
def test_embed_item_success(self, mock_pipeline_cls):
|
def test_embed_item_success(self, mock_pipeline_cls):
|
||||||
from library.tasks import embed_item
|
from library.tasks import embed_item
|
||||||
|
|
||||||
@@ -31,7 +31,7 @@ class EmbedItemTaskTests(TestCase):
|
|||||||
self.assertEqual(result["item_uid"], "test-uid-123")
|
self.assertEqual(result["item_uid"], "test-uid-123")
|
||||||
mock_pipeline.process_item.assert_called_once()
|
mock_pipeline.process_item.assert_called_once()
|
||||||
|
|
||||||
@patch("library.tasks.EmbeddingPipeline")
|
@patch("library.services.pipeline.EmbeddingPipeline")
|
||||||
def test_embed_item_failure(self, mock_pipeline_cls):
|
def test_embed_item_failure(self, mock_pipeline_cls):
|
||||||
from library.tasks import embed_item
|
from library.tasks import embed_item
|
||||||
|
|
||||||
@@ -49,7 +49,7 @@ class EmbedItemTaskTests(TestCase):
|
|||||||
class ReembedItemTaskTests(TestCase):
|
class ReembedItemTaskTests(TestCase):
|
||||||
"""Tests for the reembed_item task."""
|
"""Tests for the reembed_item task."""
|
||||||
|
|
||||||
@patch("library.tasks.EmbeddingPipeline")
|
@patch("library.services.pipeline.EmbeddingPipeline")
|
||||||
def test_reembed_item_success(self, mock_pipeline_cls):
|
def test_reembed_item_success(self, mock_pipeline_cls):
|
||||||
from library.tasks import reembed_item
|
from library.tasks import reembed_item
|
||||||
|
|
||||||
|
|||||||
92
mnemosyne/library/tests/test_views.py
Normal file
92
mnemosyne/library/tests/test_views.py
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
"""Tests for the library CRUD HTML views.
|
||||||
|
|
||||||
|
Currently covers ``library_list``'s Daedalus-workspace scope filter. The
|
||||||
|
view loads every ``Library`` node from Neo4j and narrows it by a ``scope``
|
||||||
|
GET param (``all`` / ``global`` / ``daedalus``). These tests stub out
|
||||||
|
Neo4j entirely — patching ``neo4j_available`` and injecting a fake
|
||||||
|
``Library`` class via ``sys.modules`` — so they assert on the queryset
|
||||||
|
``.filter(...)`` call the view makes and the context it renders, not on
|
||||||
|
real graph behaviour. Mirrors the mocking style in
|
||||||
|
``test_search_views_admin_scope.py``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.test import TestCase
|
||||||
|
from django.urls import reverse
|
||||||
|
|
||||||
|
User = get_user_model()
|
||||||
|
|
||||||
|
|
||||||
|
class LibraryListScopeFilterTests(TestCase):
|
||||||
|
"""Cover the ``scope`` filter branches of ``library_list``."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(
|
||||||
|
username="op", email="op@example.com", password="pw"
|
||||||
|
)
|
||||||
|
self.client.force_login(self.user)
|
||||||
|
self.url = reverse("library:library-list")
|
||||||
|
|
||||||
|
def _fake_library_cls(self):
|
||||||
|
"""Return (Library stub, nodes mock) where ``nodes`` chains fluently.
|
||||||
|
|
||||||
|
``Library.nodes`` → ``.filter(...)`` → ``.order_by(...)`` all return
|
||||||
|
the same MagicMock so the view's queryset building works regardless
|
||||||
|
of which branch it takes, and ``.filter`` records its kwargs.
|
||||||
|
"""
|
||||||
|
fake_nodes = MagicMock()
|
||||||
|
fake_nodes.filter.return_value = fake_nodes
|
||||||
|
fake_nodes.order_by.return_value = []
|
||||||
|
return SimpleNamespace(nodes=fake_nodes), fake_nodes
|
||||||
|
|
||||||
|
def _get(self, fake_library_cls, **params):
|
||||||
|
with patch("library.views.neo4j_available", return_value=True), \
|
||||||
|
patch.dict(
|
||||||
|
"sys.modules",
|
||||||
|
{"library.models": SimpleNamespace(Library=fake_library_cls)},
|
||||||
|
):
|
||||||
|
return self.client.get(self.url, params)
|
||||||
|
|
||||||
|
def test_default_scope_is_all_and_does_not_filter(self):
|
||||||
|
fake_cls, fake_nodes = self._fake_library_cls()
|
||||||
|
response = self._get(fake_cls)
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertEqual(response.context["scope"], "all")
|
||||||
|
fake_nodes.filter.assert_not_called()
|
||||||
|
fake_nodes.order_by.assert_called_once_with("name")
|
||||||
|
|
||||||
|
def test_global_scope_filters_workspace_isnull_true(self):
|
||||||
|
fake_cls, fake_nodes = self._fake_library_cls()
|
||||||
|
response = self._get(fake_cls, scope="global")
|
||||||
|
|
||||||
|
self.assertEqual(response.context["scope"], "global")
|
||||||
|
fake_nodes.filter.assert_called_once_with(workspace_id__isnull=True)
|
||||||
|
|
||||||
|
def test_daedalus_scope_filters_workspace_isnull_false(self):
|
||||||
|
fake_cls, fake_nodes = self._fake_library_cls()
|
||||||
|
response = self._get(fake_cls, scope="daedalus")
|
||||||
|
|
||||||
|
self.assertEqual(response.context["scope"], "daedalus")
|
||||||
|
fake_nodes.filter.assert_called_once_with(workspace_id__isnull=False)
|
||||||
|
|
||||||
|
def test_unknown_scope_does_not_filter(self):
|
||||||
|
"""An unexpected scope value degrades to the unfiltered list."""
|
||||||
|
fake_cls, fake_nodes = self._fake_library_cls()
|
||||||
|
response = self._get(fake_cls, scope="bogus")
|
||||||
|
|
||||||
|
self.assertEqual(response.context["scope"], "bogus")
|
||||||
|
fake_nodes.filter.assert_not_called()
|
||||||
|
|
||||||
|
def test_neo4j_unavailable_sets_error_and_empty_list(self):
|
||||||
|
with patch("library.views.neo4j_available", return_value=False):
|
||||||
|
response = self.client.get(self.url)
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertEqual(list(response.context["libraries"]), [])
|
||||||
|
self.assertEqual(response.context["error"], "Neo4j is not available.")
|
||||||
@@ -31,14 +31,20 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
@login_required
|
@login_required
|
||||||
def library_list(request):
|
def library_list(request):
|
||||||
"""List all libraries."""
|
"""List libraries, optionally filtered by Daedalus-workspace scope."""
|
||||||
|
scope = request.GET.get("scope", "all")
|
||||||
libraries = []
|
libraries = []
|
||||||
error = None
|
error = None
|
||||||
if neo4j_available():
|
if neo4j_available():
|
||||||
try:
|
try:
|
||||||
from .models import Library
|
from .models import Library
|
||||||
|
|
||||||
libraries = Library.nodes.order_by("name")
|
qs = Library.nodes
|
||||||
|
if scope == "daedalus":
|
||||||
|
qs = qs.filter(workspace_id__isnull=False)
|
||||||
|
elif scope == "global":
|
||||||
|
qs = qs.filter(workspace_id__isnull=True)
|
||||||
|
libraries = qs.order_by("name")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
error = f"Could not connect to Neo4j: {e}"
|
error = f"Could not connect to Neo4j: {e}"
|
||||||
logger.error(error)
|
logger.error(error)
|
||||||
@@ -47,7 +53,7 @@ def library_list(request):
|
|||||||
return render(
|
return render(
|
||||||
request,
|
request,
|
||||||
"library/library_list.html",
|
"library/library_list.html",
|
||||||
{"libraries": libraries, "error": error},
|
{"libraries": libraries, "error": error, "scope": scope},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -23,8 +23,9 @@ env = environ.Env(
|
|||||||
DEBUG=(bool, True),
|
DEBUG=(bool, True),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Read .env file if it exists
|
# Read .env file if it exists. Lives at the repo root (one level above the
|
||||||
environ.Env.read_env(BASE_DIR / ".env")
|
# Django project package), shared with the docker-compose stack.
|
||||||
|
environ.Env.read_env(BASE_DIR.parent / ".env")
|
||||||
|
|
||||||
# --- Security ---
|
# --- Security ---
|
||||||
SECRET_KEY = env("SECRET_KEY", default="django-insecure-change-me-in-production")
|
SECRET_KEY = env("SECRET_KEY", default="django-insecure-change-me-in-production")
|
||||||
@@ -267,6 +268,15 @@ SPELUNKER_S3_REGION_NAME = env("SPELUNKER_S3_REGION_NAME", default="us-east-1")
|
|||||||
SPELUNKER_S3_USE_SSL = env.bool("SPELUNKER_S3_USE_SSL", default=False)
|
SPELUNKER_S3_USE_SSL = env.bool("SPELUNKER_S3_USE_SSL", default=False)
|
||||||
SPELUNKER_S3_VERIFY = env.bool("SPELUNKER_S3_VERIFY", default=True)
|
SPELUNKER_S3_VERIFY = env.bool("SPELUNKER_S3_VERIFY", default=True)
|
||||||
|
|
||||||
|
# Kairos renders synced mail to text documents in its own bucket.
|
||||||
|
KAIROS_S3_ENDPOINT_URL = env("KAIROS_S3_ENDPOINT_URL", default="")
|
||||||
|
KAIROS_S3_ACCESS_KEY_ID = env("KAIROS_S3_ACCESS_KEY_ID", default="")
|
||||||
|
KAIROS_S3_SECRET_ACCESS_KEY = env("KAIROS_S3_SECRET_ACCESS_KEY", default="")
|
||||||
|
KAIROS_S3_BUCKET_NAME = env("KAIROS_S3_BUCKET_NAME", default="kairos")
|
||||||
|
KAIROS_S3_REGION_NAME = env("KAIROS_S3_REGION_NAME", default="us-east-1")
|
||||||
|
KAIROS_S3_USE_SSL = env.bool("KAIROS_S3_USE_SSL", default=False)
|
||||||
|
KAIROS_S3_VERIFY = env.bool("KAIROS_S3_VERIFY", default=True)
|
||||||
|
|
||||||
# Registry keyed by the ingest `source` field. Unknown/blank sources fall
|
# Registry keyed by the ingest `source` field. Unknown/blank sources fall
|
||||||
# back to "daedalus" for backwards compatibility.
|
# back to "daedalus" for backwards compatibility.
|
||||||
SOURCE_S3_BUCKETS = {
|
SOURCE_S3_BUCKETS = {
|
||||||
@@ -288,6 +298,15 @@ SOURCE_S3_BUCKETS = {
|
|||||||
"use_ssl": SPELUNKER_S3_USE_SSL,
|
"use_ssl": SPELUNKER_S3_USE_SSL,
|
||||||
"verify": SPELUNKER_S3_VERIFY,
|
"verify": SPELUNKER_S3_VERIFY,
|
||||||
},
|
},
|
||||||
|
"kairos-mail": {
|
||||||
|
"endpoint_url": KAIROS_S3_ENDPOINT_URL,
|
||||||
|
"access_key_id": KAIROS_S3_ACCESS_KEY_ID,
|
||||||
|
"secret_access_key": KAIROS_S3_SECRET_ACCESS_KEY,
|
||||||
|
"bucket_name": KAIROS_S3_BUCKET_NAME,
|
||||||
|
"region_name": KAIROS_S3_REGION_NAME,
|
||||||
|
"use_ssl": KAIROS_S3_USE_SSL,
|
||||||
|
"verify": KAIROS_S3_VERIFY,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
# --- Celery / RabbitMQ ---
|
# --- Celery / RabbitMQ ---
|
||||||
|
|||||||
Reference in New Issue
Block a user