Robert Helewka
31a98b4f3a
Add a `scope` GET parameter to `library_list` allowing users to filter libraries by all, global-only, or Daedalus workspace-only. Includes a filter form in the template, an updated empty state message, and tests covering each scope branch with mocked Neo4j dependencies.
981 lines
34 KiB
Python
981 lines
34 KiB
Python
"""
|
|
Custom admin views for Library, Collection, and Item CRUD.
|
|
|
|
Since neomodel StructuredNodes cannot use Django's standard ModelAdmin,
|
|
these FBVs provide CRUD operations rendered within Themis's template structure.
|
|
All views require login.
|
|
"""
|
|
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
|
|
from django.contrib import messages
|
|
from django.contrib.auth.decorators import login_required
|
|
from django.core.files.base import ContentFile
|
|
from django.core.files.storage import default_storage
|
|
from django.http import Http404
|
|
from django.shortcuts import redirect, render
|
|
|
|
from .content_types import get_library_type_config
|
|
from .forms import CollectionForm, ItemForm, LibraryForm
|
|
from .utils import neo4j_available
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Library views
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@login_required
|
|
def library_list(request):
|
|
"""List libraries, optionally filtered by Daedalus-workspace scope."""
|
|
scope = request.GET.get("scope", "all")
|
|
libraries = []
|
|
error = None
|
|
if neo4j_available():
|
|
try:
|
|
from .models import Library
|
|
|
|
qs = Library.nodes
|
|
if scope == "daedalus":
|
|
qs = qs.filter(workspace_id__isnull=False)
|
|
elif scope == "global":
|
|
qs = qs.filter(workspace_id__isnull=True)
|
|
libraries = qs.order_by("name")
|
|
except Exception as e:
|
|
error = f"Could not connect to Neo4j: {e}"
|
|
logger.error(error)
|
|
else:
|
|
error = "Neo4j is not available."
|
|
return render(
|
|
request,
|
|
"library/library_list.html",
|
|
{"libraries": libraries, "error": error, "scope": scope},
|
|
)
|
|
|
|
|
|
@login_required
|
|
def library_create(request):
|
|
"""Create a new library."""
|
|
if request.method == "POST":
|
|
form = LibraryForm(request.POST)
|
|
if form.is_valid():
|
|
try:
|
|
from .models import Library
|
|
|
|
# If content-type fields are empty, populate from defaults
|
|
library_type = form.cleaned_data["library_type"]
|
|
defaults = get_library_type_config(library_type)
|
|
|
|
lib = Library(
|
|
name=form.cleaned_data["name"],
|
|
library_type=library_type,
|
|
description=form.cleaned_data.get("description", ""),
|
|
chunking_config=defaults["chunking_config"],
|
|
embedding_instruction=(
|
|
form.cleaned_data.get("embedding_instruction")
|
|
or defaults["embedding_instruction"]
|
|
),
|
|
reranker_instruction=(
|
|
form.cleaned_data.get("reranker_instruction")
|
|
or defaults["reranker_instruction"]
|
|
),
|
|
llm_context_prompt=(
|
|
form.cleaned_data.get("llm_context_prompt")
|
|
or defaults["llm_context_prompt"]
|
|
),
|
|
)
|
|
lib.save()
|
|
messages.success(request, f'Library "{lib.name}" created.')
|
|
return redirect("library:library-detail", uid=lib.uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Error creating library: {e}")
|
|
else:
|
|
form = LibraryForm()
|
|
return render(request, "library/library_form.html", {"form": form, "editing": False})
|
|
|
|
|
|
def _library_detail_context(library):
|
|
"""
|
|
Build the base context for the library detail page.
|
|
|
|
Shared between ``library_detail`` and ``library_search`` so the search
|
|
POST handler renders the same page chrome plus its results layered on
|
|
top.
|
|
"""
|
|
from llm_manager.models import LLMModel
|
|
|
|
embedding_model = LLMModel.get_system_embedding_model()
|
|
multimodal_available = bool(embedding_model and embedding_model.supports_multimodal)
|
|
|
|
return {
|
|
"library": library,
|
|
"collections": library.collections.all(),
|
|
"multimodal_available": multimodal_available,
|
|
"search_query": "",
|
|
"search_used_image": False,
|
|
"results_baseline": None,
|
|
"results_reranked": None,
|
|
"search_error": None,
|
|
}
|
|
|
|
|
|
@login_required
|
|
def library_detail(request, uid):
|
|
"""View library details and its collections."""
|
|
try:
|
|
from .models import Library
|
|
|
|
lib = Library.nodes.get(uid=uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Library not found: {e}")
|
|
return redirect("library:library-list")
|
|
return render(
|
|
request,
|
|
"library/library_detail.html",
|
|
_library_detail_context(lib),
|
|
)
|
|
|
|
|
|
# Cap query-image uploads at 8 MB. Multimodal embedders happily accept
|
|
# larger payloads but they're slow and almost never come from a real
|
|
# in-browser screenshot/photo.
|
|
_MAX_QUERY_IMAGE_BYTES = 8 * 1024 * 1024
|
|
|
|
|
|
def _all_library_uids() -> list[str]:
|
|
"""Legacy alias for :func:`library.utils.all_library_uids`.
|
|
|
|
Kept here so existing tests that patch
|
|
``library.views._all_library_uids`` continue to work during the
|
|
Phase-2 refactor. New code should import ``all_library_uids``
|
|
directly from ``library.utils``.
|
|
"""
|
|
from .utils import all_library_uids
|
|
|
|
return all_library_uids()
|
|
|
|
|
|
@login_required
|
|
def library_search(request, uid):
|
|
"""
|
|
Run an A/B search (with and without re-ranker) scoped to a single
|
|
library, and re-render ``library_detail.html`` with both result sets.
|
|
"""
|
|
try:
|
|
from .models import Library
|
|
|
|
lib = Library.nodes.get(uid=uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Library not found: {e}")
|
|
return redirect("library:library-list")
|
|
|
|
context = _library_detail_context(lib)
|
|
|
|
if request.method != "POST":
|
|
return redirect("library:library-detail", uid=uid)
|
|
|
|
query = (request.POST.get("query") or "").strip()
|
|
context["search_query"] = query
|
|
|
|
image_bytes = None
|
|
image_ext = "png"
|
|
uploaded = request.FILES.get("query_image")
|
|
if uploaded and context["multimodal_available"]:
|
|
if uploaded.size > _MAX_QUERY_IMAGE_BYTES:
|
|
context["search_error"] = (
|
|
f"Image too large ({uploaded.size} bytes). "
|
|
f"Max is {_MAX_QUERY_IMAGE_BYTES} bytes."
|
|
)
|
|
return render(request, "library/library_detail.html", context)
|
|
image_bytes = uploaded.read()
|
|
# Derive extension from the filename; default to png. The embedder
|
|
# only uses this to set the MIME type for the multimodal request.
|
|
_, ext = os.path.splitext(uploaded.name or "")
|
|
if ext.startswith("."):
|
|
ext = ext[1:].lower()
|
|
if ext:
|
|
image_ext = ext
|
|
context["search_used_image"] = True
|
|
|
|
if not query and not image_bytes:
|
|
context["search_error"] = "Enter a query (text or image) before searching."
|
|
return render(request, "library/library_detail.html", context)
|
|
|
|
try:
|
|
from django.conf import settings as django_settings
|
|
|
|
from .services.search import SearchRequest, SearchService
|
|
|
|
# Admin UI searches every library the operator can see — including
|
|
# Daedalus workspace-scoped ones. See ``_all_library_uids`` for
|
|
# why this is needed (the default scope clause branch hides any
|
|
# library with a non-null ``workspace_id``).
|
|
allowed = _all_library_uids()
|
|
|
|
def _make_request(rerank: bool) -> "SearchRequest":
|
|
return SearchRequest(
|
|
query=query,
|
|
query_image=image_bytes,
|
|
query_image_ext=image_ext,
|
|
library_uid=uid,
|
|
resolved_libraries=allowed,
|
|
limit=getattr(django_settings, "SEARCH_DEFAULT_LIMIT", 20),
|
|
vector_top_k=getattr(django_settings, "SEARCH_VECTOR_TOP_K", 50),
|
|
fulltext_top_k=getattr(django_settings, "SEARCH_FULLTEXT_TOP_K", 30),
|
|
rerank=rerank,
|
|
include_images=True,
|
|
)
|
|
|
|
service = SearchService(user=request.user)
|
|
baseline = service.search(_make_request(rerank=False))
|
|
reranked = service.search(_make_request(rerank=True))
|
|
|
|
# Annotate the reranked candidates with a rank-delta label so the
|
|
# template can render a badge without doing arithmetic. ``new`` =
|
|
# the reranker pulled this in from outside the baseline top-N.
|
|
baseline_pos = {c.chunk_uid: i for i, c in enumerate(baseline.candidates)}
|
|
for new_index, cand in enumerate(reranked.candidates):
|
|
old_index = baseline_pos.get(cand.chunk_uid)
|
|
if old_index is None:
|
|
cand.rank_delta_label = "new"
|
|
cand.rank_delta_kind = "new"
|
|
else:
|
|
delta = old_index - new_index # +N == moved up
|
|
if delta > 0:
|
|
cand.rank_delta_label = f"↑{delta}"
|
|
cand.rank_delta_kind = "up"
|
|
elif delta < 0:
|
|
cand.rank_delta_label = f"↓{-delta}"
|
|
cand.rank_delta_kind = "down"
|
|
else:
|
|
cand.rank_delta_label = "="
|
|
cand.rank_delta_kind = "same"
|
|
|
|
context["results_baseline"] = baseline
|
|
context["results_reranked"] = reranked
|
|
except Exception as exc:
|
|
logger.error("Library search failed: %s", exc, exc_info=True)
|
|
context["search_error"] = str(exc)
|
|
|
|
return render(request, "library/library_detail.html", context)
|
|
|
|
|
|
@login_required
|
|
def library_edit(request, uid):
|
|
"""Edit an existing library."""
|
|
try:
|
|
from .models import Library
|
|
|
|
lib = Library.nodes.get(uid=uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Library not found: {e}")
|
|
return redirect("library:library-list")
|
|
|
|
if request.method == "POST":
|
|
form = LibraryForm(request.POST)
|
|
if form.is_valid():
|
|
try:
|
|
lib.name = form.cleaned_data["name"]
|
|
lib.library_type = form.cleaned_data["library_type"]
|
|
lib.description = form.cleaned_data.get("description", "")
|
|
lib.embedding_instruction = form.cleaned_data.get(
|
|
"embedding_instruction", ""
|
|
)
|
|
lib.reranker_instruction = form.cleaned_data.get(
|
|
"reranker_instruction", ""
|
|
)
|
|
lib.llm_context_prompt = form.cleaned_data.get(
|
|
"llm_context_prompt", ""
|
|
)
|
|
lib.save()
|
|
messages.success(request, f'Library "{lib.name}" updated.')
|
|
return redirect("library:library-detail", uid=lib.uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Error updating library: {e}")
|
|
else:
|
|
form = LibraryForm(
|
|
initial={
|
|
"name": lib.name,
|
|
"library_type": lib.library_type,
|
|
"description": lib.description,
|
|
"embedding_instruction": lib.embedding_instruction,
|
|
"reranker_instruction": lib.reranker_instruction,
|
|
"llm_context_prompt": lib.llm_context_prompt,
|
|
}
|
|
)
|
|
return render(
|
|
request,
|
|
"library/library_form.html",
|
|
{"form": form, "editing": True, "library": lib},
|
|
)
|
|
|
|
|
|
@login_required
|
|
def library_delete(request, uid):
|
|
"""Delete a library (and confirm)."""
|
|
try:
|
|
from .models import Library
|
|
|
|
lib = Library.nodes.get(uid=uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Library not found: {e}")
|
|
return redirect("library:library-list")
|
|
|
|
# Daedalus owns the lifecycle of workspace-scoped libraries. Deleting one
|
|
# here is allowed but discouraged: the confirm page warns that Daedalus
|
|
# still holds the source content and will recreate + re-embed it on the
|
|
# next sync. The risk is low (no data loss — only re-embedding cost), and
|
|
# this is the supported escape hatch for clearing an orphaned Library that
|
|
# blocks workspace re-registration.
|
|
if request.method == "POST":
|
|
name = lib.name
|
|
# Use the shared cascade so child nodes (Collections/Items/Chunks/
|
|
# Images) and orphan Concepts are removed too — a bare lib.delete()
|
|
# would leak them.
|
|
from .services.library_delete import delete_library_cascade
|
|
|
|
delete_library_cascade(lib)
|
|
messages.success(request, f'Library "{name}" deleted.')
|
|
return redirect("library:library-list")
|
|
return render(request, "library/library_confirm_delete.html", {"library": lib})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Collection views
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@login_required
|
|
def collection_create(request, library_uid):
|
|
"""Create a new collection within a library."""
|
|
try:
|
|
from .models import Library
|
|
|
|
lib = Library.nodes.get(uid=library_uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Library not found: {e}")
|
|
return redirect("library:library-list")
|
|
|
|
if request.method == "POST":
|
|
form = CollectionForm(request.POST)
|
|
if form.is_valid():
|
|
try:
|
|
from .models import Collection
|
|
|
|
col = Collection(
|
|
name=form.cleaned_data["name"],
|
|
description=form.cleaned_data.get("description", ""),
|
|
)
|
|
col.save()
|
|
lib.collections.connect(col)
|
|
col.library.connect(lib)
|
|
messages.success(request, f'Collection "{col.name}" created.')
|
|
return redirect("library:collection-detail", uid=col.uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Error creating collection: {e}")
|
|
else:
|
|
form = CollectionForm()
|
|
return render(
|
|
request,
|
|
"library/collection_form.html",
|
|
{"form": form, "library": lib, "editing": False},
|
|
)
|
|
|
|
|
|
@login_required
|
|
def collection_detail(request, uid):
|
|
"""View collection details and its items."""
|
|
try:
|
|
from .models import Collection
|
|
|
|
col = Collection.nodes.get(uid=uid)
|
|
items = col.items.all()
|
|
libraries = col.library.all()
|
|
library = libraries[0] if libraries else None
|
|
except Exception as e:
|
|
messages.error(request, f"Collection not found: {e}")
|
|
return redirect("library:library-list")
|
|
return render(
|
|
request,
|
|
"library/collection_detail.html",
|
|
{"collection": col, "items": items, "library": library},
|
|
)
|
|
|
|
|
|
@login_required
|
|
def collection_edit(request, uid):
|
|
"""Edit an existing collection."""
|
|
try:
|
|
from .models import Collection
|
|
|
|
col = Collection.nodes.get(uid=uid)
|
|
libraries = col.library.all()
|
|
library = libraries[0] if libraries else None
|
|
except Exception as e:
|
|
messages.error(request, f"Collection not found: {e}")
|
|
return redirect("library:library-list")
|
|
|
|
if request.method == "POST":
|
|
form = CollectionForm(request.POST)
|
|
if form.is_valid():
|
|
try:
|
|
col.name = form.cleaned_data["name"]
|
|
col.description = form.cleaned_data.get("description", "")
|
|
col.save()
|
|
messages.success(request, f'Collection "{col.name}" updated.')
|
|
return redirect("library:collection-detail", uid=col.uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Error updating collection: {e}")
|
|
else:
|
|
form = CollectionForm(
|
|
initial={
|
|
"name": col.name,
|
|
"description": col.description,
|
|
}
|
|
)
|
|
return render(
|
|
request,
|
|
"library/collection_form.html",
|
|
{"form": form, "collection": col, "library": library, "editing": True},
|
|
)
|
|
|
|
|
|
@login_required
|
|
def collection_delete(request, uid):
|
|
"""Delete a collection."""
|
|
try:
|
|
from .models import Collection
|
|
|
|
col = Collection.nodes.get(uid=uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Collection not found: {e}")
|
|
return redirect("library:library-list")
|
|
|
|
if request.method == "POST":
|
|
name = col.name
|
|
col.delete()
|
|
messages.success(request, f'Collection "{name}" deleted.')
|
|
return redirect("library:library-list")
|
|
return render(
|
|
request,
|
|
"library/collection_confirm_delete.html",
|
|
{"collection": col},
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Item views
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@login_required
|
|
def item_create(request, collection_uid):
|
|
"""Create a new item within a collection, with optional file upload."""
|
|
try:
|
|
from .models import Collection
|
|
|
|
col = Collection.nodes.get(uid=collection_uid)
|
|
libraries = col.library.all()
|
|
library = libraries[0] if libraries else None
|
|
except Exception as e:
|
|
messages.error(request, f"Collection not found: {e}")
|
|
return redirect("library:library-list")
|
|
|
|
if request.method == "POST":
|
|
form = ItemForm(request.POST, request.FILES)
|
|
if form.is_valid():
|
|
try:
|
|
from .models import Item
|
|
|
|
uploaded_file = request.FILES.get("file")
|
|
file_type = form.cleaned_data.get("file_type", "")
|
|
|
|
# Infer file_type from upload if not explicitly set
|
|
if uploaded_file and not file_type:
|
|
_, ext = os.path.splitext(uploaded_file.name)
|
|
file_type = ext.lstrip(".").lower()
|
|
|
|
item = Item(
|
|
title=form.cleaned_data["title"],
|
|
item_type=form.cleaned_data.get("item_type", ""),
|
|
file_type=file_type,
|
|
embedding_status="pending",
|
|
)
|
|
|
|
# Handle file upload
|
|
if uploaded_file:
|
|
file_data = uploaded_file.read()
|
|
item.file_size = len(file_data)
|
|
item.content_hash = hashlib.sha256(file_data).hexdigest()
|
|
item.save()
|
|
|
|
# Store in S3
|
|
s3_key = f"items/{item.uid}/original.{file_type}"
|
|
default_storage.save(s3_key, ContentFile(file_data))
|
|
item.s3_key = s3_key
|
|
item.save()
|
|
else:
|
|
item.save()
|
|
|
|
col.items.connect(item)
|
|
|
|
# Auto-trigger embedding if file uploaded and checkbox set
|
|
auto_embed = form.cleaned_data.get("auto_embed", True)
|
|
if uploaded_file and auto_embed:
|
|
try:
|
|
from .tasks import embed_item
|
|
|
|
task = embed_item.delay(item.uid, request.user.id)
|
|
messages.info(
|
|
request,
|
|
f"Embedding queued (task: {task.id})",
|
|
)
|
|
except Exception as exc:
|
|
logger.warning("Failed to queue embedding: %s", exc)
|
|
|
|
messages.success(request, f'Item "{item.title}" created.')
|
|
return redirect("library:item-detail", uid=item.uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Error creating item: {e}")
|
|
else:
|
|
form = ItemForm(initial={"auto_embed": True})
|
|
return render(
|
|
request,
|
|
"library/item_form.html",
|
|
{"form": form, "collection": col, "library": library, "editing": False},
|
|
)
|
|
|
|
|
|
@login_required
|
|
def item_detail(request, uid):
|
|
"""View item details."""
|
|
try:
|
|
from .models import Item
|
|
|
|
item = Item.nodes.get(uid=uid)
|
|
chunks = item.chunks.all()
|
|
images = item.images.all()
|
|
concepts = item.concepts.all()
|
|
except Exception as e:
|
|
messages.error(request, f"Item not found: {e}")
|
|
return redirect("library:library-list")
|
|
return render(
|
|
request,
|
|
"library/item_detail.html",
|
|
{"item": item, "chunks": chunks, "images": images, "concepts": concepts},
|
|
)
|
|
|
|
|
|
@login_required
|
|
def item_edit(request, uid):
|
|
"""Edit an existing item."""
|
|
try:
|
|
from .models import Item
|
|
|
|
item = Item.nodes.get(uid=uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Item not found: {e}")
|
|
return redirect("library:library-list")
|
|
|
|
if request.method == "POST":
|
|
form = ItemForm(request.POST)
|
|
if form.is_valid():
|
|
try:
|
|
item.title = form.cleaned_data["title"]
|
|
item.item_type = form.cleaned_data.get("item_type", "")
|
|
item.file_type = form.cleaned_data.get("file_type", "")
|
|
item.save()
|
|
messages.success(request, f'Item "{item.title}" updated.')
|
|
return redirect("library:item-detail", uid=item.uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Error updating item: {e}")
|
|
else:
|
|
form = ItemForm(
|
|
initial={
|
|
"title": item.title,
|
|
"item_type": item.item_type,
|
|
"file_type": item.file_type,
|
|
}
|
|
)
|
|
return render(
|
|
request,
|
|
"library/item_form.html",
|
|
{"form": form, "item": item, "editing": True},
|
|
)
|
|
|
|
|
|
@login_required
|
|
def item_reembed(request, uid):
|
|
"""Trigger re-embedding for an item."""
|
|
try:
|
|
from .models import Item
|
|
|
|
item = Item.nodes.get(uid=uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Item not found: {e}")
|
|
return redirect("library:library-list")
|
|
|
|
if request.method == "POST":
|
|
try:
|
|
from .tasks import reembed_item
|
|
|
|
task = reembed_item.delay(uid, request.user.id)
|
|
messages.info(request, f"Re-embedding queued for \"{item.title}\" (task: {task.id})")
|
|
except Exception as exc:
|
|
messages.error(request, f"Failed to queue re-embedding: {exc}")
|
|
return redirect("library:item-detail", uid=uid)
|
|
|
|
return redirect("library:item-detail", uid=uid)
|
|
|
|
|
|
@login_required
|
|
def item_delete(request, uid):
|
|
"""Delete an item."""
|
|
try:
|
|
from .models import Item
|
|
|
|
item = Item.nodes.get(uid=uid)
|
|
except Exception as e:
|
|
messages.error(request, f"Item not found: {e}")
|
|
return redirect("library:library-list")
|
|
|
|
if request.method == "POST":
|
|
title = item.title
|
|
item.delete()
|
|
messages.success(request, f'Item "{title}" deleted.')
|
|
return redirect("library:library-list")
|
|
return render(request, "library/item_confirm_delete.html", {"item": item})
|
|
|
|
|
|
@login_required
|
|
def item_download(request, uid):
|
|
"""Redirect to a presigned/storage URL for the item's original file."""
|
|
try:
|
|
from .models import Item
|
|
|
|
item = Item.nodes.get(uid=uid)
|
|
except Exception:
|
|
raise Http404("Item not found.")
|
|
|
|
if not item.s3_key:
|
|
raise Http404("No file available for this item.")
|
|
|
|
try:
|
|
url = default_storage.url(item.s3_key)
|
|
except Exception as e:
|
|
logger.error("Failed to generate download URL for %s: %s", item.s3_key, e)
|
|
raise Http404("File not accessible.")
|
|
|
|
return redirect(url)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Image views
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@login_required
|
|
def image_serve(request, uid):
|
|
"""Redirect to a presigned/storage URL for an image file."""
|
|
try:
|
|
from .models import Image
|
|
|
|
img = Image.nodes.get(uid=uid)
|
|
except Exception:
|
|
raise Http404("Image not found.")
|
|
|
|
if not img.s3_key:
|
|
raise Http404("No file available for this image.")
|
|
|
|
try:
|
|
url = default_storage.url(img.s3_key)
|
|
except Exception as e:
|
|
logger.error("Failed to generate image URL for %s: %s", img.s3_key, e)
|
|
raise Http404("Image not accessible.")
|
|
|
|
return redirect(url)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Embedding Pipeline Dashboard
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@login_required
|
|
def embedding_dashboard(request):
|
|
"""
|
|
Embedding pipeline dashboard — system model status, item embedding
|
|
progress, knowledge graph node counts, and batch actions.
|
|
"""
|
|
context = {
|
|
"system_embedding_model": None,
|
|
"system_chat_model": None,
|
|
"system_reranker_model": None,
|
|
"system_vision_model": None,
|
|
"status_counts": {},
|
|
"node_counts": {},
|
|
"total_items": 0,
|
|
"embedded_chunks": 0,
|
|
"total_chunks": 0,
|
|
"neo4j_available": False,
|
|
}
|
|
|
|
# Get system models from LLM Manager
|
|
try:
|
|
from llm_manager.models import LLMModel
|
|
|
|
context["system_embedding_model"] = LLMModel.get_system_embedding_model()
|
|
context["system_chat_model"] = LLMModel.get_system_chat_model()
|
|
context["system_reranker_model"] = LLMModel.get_system_reranker_model()
|
|
context["system_vision_model"] = LLMModel.get_system_vision_model()
|
|
except Exception as exc:
|
|
logger.warning("Could not load system models: %s", exc)
|
|
|
|
# Reachability of the system-default models (keyed by role for the
|
|
# template). A probe failure must never 500 the dashboard.
|
|
context["model_health"] = {}
|
|
try:
|
|
from library.services.model_health import probe_system_models
|
|
|
|
context["model_health"] = {r["role"]: r for r in probe_system_models()}
|
|
except Exception as exc:
|
|
logger.warning("Could not probe system model health: %s", exc)
|
|
|
|
# Get item status counts and node counts from Neo4j
|
|
if neo4j_available():
|
|
context["neo4j_available"] = True
|
|
try:
|
|
from neomodel import db
|
|
|
|
for status in ["pending", "processing", "completed", "failed"]:
|
|
results, _ = db.cypher_query(
|
|
"MATCH (i:Item {embedding_status: $status}) RETURN count(i)",
|
|
{"status": status},
|
|
)
|
|
context["status_counts"][status] = results[0][0] if results else 0
|
|
|
|
results, _ = db.cypher_query("MATCH (i:Item) RETURN count(i)")
|
|
context["total_items"] = results[0][0] if results else 0
|
|
|
|
for label in ["Library", "Collection", "Item", "Chunk", "Concept", "Image", "ImageEmbedding"]:
|
|
results, _ = db.cypher_query(f"MATCH (n:{label}) RETURN count(n)")
|
|
context["node_counts"][label] = results[0][0] if results else 0
|
|
|
|
results, _ = db.cypher_query(
|
|
"MATCH (c:Chunk) WHERE c.embedding IS NOT NULL RETURN count(c)"
|
|
)
|
|
context["embedded_chunks"] = results[0][0] if results else 0
|
|
context["total_chunks"] = context["node_counts"].get("Chunk", 0)
|
|
|
|
except Exception as exc:
|
|
logger.warning("Could not query Neo4j for dashboard: %s", exc)
|
|
messages.warning(request, f"Neo4j query error: {exc}")
|
|
|
|
return render(request, "library/embedding_dashboard.html", context)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Search views (Phase 3)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@login_required
|
|
def search_page(request):
|
|
"""Search page — query input with filters and results display."""
|
|
from .utils import neo4j_available
|
|
|
|
context = {
|
|
"query": "",
|
|
"results": None,
|
|
"libraries": [],
|
|
"error": None,
|
|
}
|
|
|
|
# Load libraries for filter dropdown
|
|
if neo4j_available():
|
|
try:
|
|
from .models import Library
|
|
|
|
context["libraries"] = Library.nodes.order_by("name")
|
|
except Exception:
|
|
pass
|
|
|
|
if request.method == "POST" or request.GET.get("q"):
|
|
query = request.POST.get("query", "") or request.GET.get("q", "")
|
|
library_uid = request.POST.get("library_uid", "") or request.GET.get("library_uid", "")
|
|
library_type = request.POST.get("library_type", "") or request.GET.get("library_type", "")
|
|
rerank = request.POST.get("rerank", "on") == "on"
|
|
|
|
context["query"] = query
|
|
|
|
if query.strip():
|
|
try:
|
|
from django.conf import settings as django_settings
|
|
|
|
from .services.search import SearchRequest, SearchService
|
|
|
|
search_request = SearchRequest(
|
|
query=query,
|
|
library_uid=library_uid or None,
|
|
library_type=library_type or None,
|
|
# Admin UI is session-authenticated and sees every
|
|
# library, Daedalus-workspace-scoped or global.
|
|
# ``library.utils.all_library_uids`` materializes the
|
|
# full Library UID set as the request's
|
|
# ``resolved_libraries`` — see the unified auth model
|
|
# in ``docs/DAEDALUS_PALLAS_INTEGRATION_v1.md`` §3.3.
|
|
resolved_libraries=_all_library_uids(),
|
|
limit=getattr(django_settings, "SEARCH_DEFAULT_LIMIT", 20),
|
|
vector_top_k=getattr(django_settings, "SEARCH_VECTOR_TOP_K", 50),
|
|
fulltext_top_k=getattr(django_settings, "SEARCH_FULLTEXT_TOP_K", 30),
|
|
rerank=rerank,
|
|
include_images=True,
|
|
)
|
|
|
|
service = SearchService(user=request.user)
|
|
context["results"] = service.search(search_request)
|
|
except Exception as exc:
|
|
logger.error("Search failed: %s", exc, exc_info=True)
|
|
context["error"] = str(exc)
|
|
|
|
return render(request, "library/search.html", context)
|
|
|
|
|
|
@login_required
|
|
def concept_list_page(request):
|
|
"""Browse concepts with optional search."""
|
|
context = {
|
|
"concepts": [],
|
|
"query": "",
|
|
"error": None,
|
|
}
|
|
|
|
query = request.GET.get("q", "")
|
|
context["query"] = query
|
|
|
|
try:
|
|
if query:
|
|
from neomodel import db
|
|
|
|
results, _ = db.cypher_query(
|
|
"CALL db.index.fulltext.queryNodes('concept_name_fulltext', $query) "
|
|
"YIELD node, score "
|
|
"RETURN node.uid AS uid, node.name AS name, "
|
|
" node.concept_type AS concept_type, score "
|
|
"ORDER BY score DESC LIMIT 50",
|
|
{"query": query},
|
|
)
|
|
context["concepts"] = [
|
|
{"uid": r[0], "name": r[1], "concept_type": r[2] or "", "score": r[3]}
|
|
for r in results
|
|
]
|
|
else:
|
|
from .models import Concept
|
|
|
|
concepts = Concept.nodes.order_by("name")[:100]
|
|
context["concepts"] = [
|
|
{"uid": c.uid, "name": c.name, "concept_type": c.concept_type or ""}
|
|
for c in concepts
|
|
]
|
|
except Exception as exc:
|
|
logger.error("Concept list failed: %s", exc)
|
|
context["error"] = str(exc)
|
|
|
|
return render(request, "library/concept_list.html", context)
|
|
|
|
|
|
@login_required
|
|
def concept_detail_page(request, uid):
|
|
"""View a concept and its graph connections."""
|
|
context = {
|
|
"concept": None,
|
|
"items": [],
|
|
"related_concepts": [],
|
|
"chunk_count": 0,
|
|
"image_count": 0,
|
|
"error": None,
|
|
}
|
|
|
|
try:
|
|
from neomodel import db
|
|
|
|
results, _ = db.cypher_query(
|
|
"MATCH (c:Concept {uid: $uid}) "
|
|
"OPTIONAL MATCH (c)<-[:MENTIONS]-(chunk:Chunk)<-[:HAS_CHUNK]-(item:Item) "
|
|
"OPTIONAL MATCH (c)<-[:DEPICTS]-(img:Image)<-[:HAS_IMAGE]-(img_item:Item) "
|
|
"OPTIONAL MATCH (c)-[:RELATED_TO]-(related:Concept) "
|
|
"RETURN c.uid AS uid, c.name AS name, c.concept_type AS concept_type, "
|
|
" collect(DISTINCT {uid: item.uid, title: item.title})[..20] AS items, "
|
|
" collect(DISTINCT {uid: related.uid, name: related.name, "
|
|
" concept_type: related.concept_type}) AS related_concepts, "
|
|
" count(DISTINCT chunk) AS chunk_count, "
|
|
" count(DISTINCT img) AS image_count",
|
|
{"uid": uid},
|
|
)
|
|
|
|
if not results or not results[0][0]:
|
|
messages.error(request, "Concept not found.")
|
|
return redirect("library:concept-list")
|
|
|
|
row = results[0]
|
|
context["concept"] = {
|
|
"uid": row[0],
|
|
"name": row[1],
|
|
"concept_type": row[2] or "",
|
|
}
|
|
context["items"] = [i for i in (row[3] or []) if i.get("uid")]
|
|
context["related_concepts"] = [r for r in (row[4] or []) if r.get("uid")]
|
|
context["chunk_count"] = row[5] or 0
|
|
context["image_count"] = row[6] or 0
|
|
|
|
except Exception as exc:
|
|
logger.error("Concept detail failed: %s", exc)
|
|
context["error"] = str(exc)
|
|
|
|
return render(request, "library/concept_detail.html", context)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Batch Embedding
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@login_required
|
|
def embed_all_pending(request):
|
|
"""
|
|
Trigger embedding for all pending items with uploaded files.
|
|
POST-only action, redirects back to dashboard.
|
|
"""
|
|
if request.method != "POST":
|
|
return redirect("library:embedding-dashboard")
|
|
|
|
try:
|
|
from neomodel import db
|
|
|
|
results, _ = db.cypher_query(
|
|
"MATCH (i:Item {embedding_status: 'pending'}) "
|
|
"WHERE i.s3_key IS NOT NULL AND i.s3_key <> '' "
|
|
"RETURN i.uid"
|
|
)
|
|
item_uids = [row[0] for row in results]
|
|
|
|
if not item_uids:
|
|
messages.info(request, "No pending items with files to embed.")
|
|
else:
|
|
from .tasks import batch_embed_items
|
|
|
|
task = batch_embed_items.delay(item_uids, request.user.id)
|
|
messages.success(
|
|
request,
|
|
f"Queued embedding for {len(item_uids)} items (task: {task.id})",
|
|
)
|
|
except Exception as exc:
|
|
logger.error("Failed to trigger batch embedding: %s", exc, exc_info=True)
|
|
messages.error(request, f"Failed to trigger embedding: {exc}")
|
|
|
|
return redirect("library:embedding-dashboard")
|