endoreg-db 0.8.8.9__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

endoreg_db/urls/media.py

@@ -1,36 +1,35 @@
 from django.urls import path
 
 from endoreg_db.views import VideoStreamView
-from endoreg_db.views.ai import label_list
 from endoreg_db.views.media import (
-    PdfMediaView,  # Alias to avoid conflict with legacy pdf.PDFMediaView
-    VideoMediaView,
-    get_sensitive_metadata_pk,
-    pdf_sensitive_metadata,
-    pdf_sensitive_metadata_list,
-    pdf_sensitive_metadata_verify,
-    sensitive_metadata_list,
-    video_sensitive_metadata,
-    video_sensitive_metadata_verify,
+                                    PdfMediaView,  # Alias to avoid conflict with legacy pdf.reportMediaView
+                                    VideoMediaView,
+                                    get_sensitive_metadata_pk,
+                                    pdf_sensitive_metadata,
+                                    pdf_sensitive_metadata_list,
+                                    pdf_sensitive_metadata_verify,
+                                    sensitive_metadata_list,
+                                    video_sensitive_metadata,
+                                    video_sensitive_metadata_verify,
 )
 from endoreg_db.views.report.reimport import ReportReimportView
 from endoreg_db.views.report.report_stream import ReportStreamView
 from endoreg_db.views.video import (
-    VideoReimportView,
-    video_segment_detail,
-    video_segment_validate,
-    video_segments_by_video,
-    video_segments_collection,
-    video_segments_stats,
-    video_segments_validate_bulk,
-    video_segments_validation_status,
+                                    VideoReimportView,
+                                    video_segment_detail,
+                                    video_segment_validate,
+                                    video_segments_by_video,
+                                    video_segments_collection,
+                                    video_segments_stats,
+                                    video_segments_validate_bulk,
+                                    video_segments_validation_status,
 )
+from endoreg_db.views.video.ai import label_list
 from endoreg_db.views.video.correction import (
-    VideoApplyMaskView,
-    VideoCorrectionView,
-    VideoMetadataStatsView,
-    VideoProcessingHistoryView,
-    VideoRemoveFramesView,
+                                    VideoApplyMaskView,
+                                    VideoCorrectionView,
+                                    VideoMetadataStatsView,
+                                    VideoRemoveFramesView,
 )
 
 # ---------------------------------------------------------------------------------------
@@ -95,14 +94,6 @@ urlpatterns = [
         VideoMetadataStatsView.as_view(),
         name="video-metadata",
     ),
-    # Video Processing History API
-    # GET /api/media/videos/<int:pk>/processing-history/
-    # Returns history of all processing operations (masking, frame removal, analysis)
-    path(
-        "media/videos/<int:pk>/processing-history/",
-        VideoProcessingHistoryView.as_view(),
-        name="video-processing-history",
-    ),
     # Video Analysis API
     # POST /api/media/videos/<int:pk>/analyze/
     # Analyzes video for sensitive frames using MiniCPM-o 2.6 or OCR+LLM
@@ -234,7 +225,7 @@ urlpatterns = [
     ),
     # List Endpoints (Collection-Level)
     # GET /api/media/sensitive-metadata/
-    # List all sensitive metadata (combined PDFs and Videos)
+    # List all sensitive metadata (combined reports and Videos)
     # Supports filtering: ?content_type=pdf|video&verified=true&search=name
     path(
         "media/sensitive-metadata/",
@@ -242,7 +233,7 @@ urlpatterns = [
         name="sensitive-metadata-list",
     ),
     # GET /api/media/pdfs/sensitive-metadata/
-    # List sensitive metadata for PDFs only
+    # List sensitive metadata for reports only
     # Replaces legacy /api/pdf/sensitivemeta/list/
     path(
         "media/pdfs/sensitive-metadata/",
@@ -259,7 +250,9 @@ urlpatterns = [
     # POST /api/media/pdfs/<int:pk>/reimport/
     # Re-imports a report file to regenerate metadata when OCR failed or data is incomplete
     path(
-        "media/pdfs/<int:pk>/reimport/", ReportReimportView.as_view(), name="report-reimport"
+        "media/pdfs/<int:pk>/reimport/",
+        ReportReimportView.as_view(),
+        name="report-reimport",
     ),
 ]
 # ---------------------------------------------------------------------------------------

endoreg_db/urls/patient.py

@@ -2,18 +2,22 @@ from endoreg_db.views import (
     GenderViewSet,
     CenterViewSet,
     PatientViewSet,
-    PatientFindingViewSet
+    PatientFindingViewSet,
 )
 from rest_framework.routers import DefaultRouter
 from django.urls import path, include
 
 router = DefaultRouter()
-router.register(r'patients', PatientViewSet)
-router.register(r'centers', CenterViewSet)
-router.register(r'genders', GenderViewSet)
-router.register(r'patient-findings', PatientFindingViewSet)
+router.register(r"patients", PatientViewSet)
+router.register(r"centers", CenterViewSet)
+router.register(r"genders", GenderViewSet)
+router.register(r"patient-findings", PatientFindingViewSet)
 
 urlpatterns = [
-    path('', include(router.urls)),
-    path('check_pe_exist/<int:pk>/', PatientViewSet.as_view({'get': 'check_pe_exist'}), name='check_pe_exist'),
+    path("", include(router.urls)),
+    path(
+        "check_pe_exist/<int:pk>/",
+        PatientViewSet.as_view({"get": "check_pe_exist"}),
+        name="check_pe_exist",
+    ),
 ]

endoreg_db/urls/requirements.py

@@ -9,5 +9,7 @@ router.register(r"lookup", LookupViewSet, basename="lookup/")
 
 urlpatterns = [
     path("", include(router.urls)),
-    path("evaluate-requirements/", evaluate_requirements, name="evaluate-requirements/"),
+    path(
+        "evaluate-requirements/", evaluate_requirements, name="evaluate-requirements/"
+    ),
 ]

endoreg_db/urls/root_urls.py

@@ -5,20 +5,19 @@ from django.http import HttpResponse
 from django.conf import settings
 from django.conf.urls.static import static
 
+
 def public_home(_request):
     return HttpResponse("Public home – no login required.")
 
+
 urlpatterns = [
     # Public landing page
     path("", public_home, name="public_home"),
-
     # Django admin (optional)
     path("admin/", admin.site.urls),
-
     # Mount ALL API routes under /api/
     # This pulls the urlpatterns exported by endoreg_db/urls/__init__.py
     path("api/", include("endoreg_db.urls")),
-
     # Keycloak OIDC (mozilla-django-oidc provides /oidc/authenticate/ and /oidc/callback/)
     path("oidc/", include("mozilla_django_oidc.urls")),
 ]

endoreg_db/urls/stats.py

@@ -13,34 +13,42 @@ url_patterns = [
     #
     # Diese Endpunkte stellen Dashboard-Statistiken bereit für das Frontend
     # ---------------------------------------------------------------------------------------
-
     # Examination Statistics API
     # GET /api/examinations/stats/
     # Liefert Statistiken über Examinations und PatientExaminations
-    path('examinations/stats/', ExaminationStatsView.as_view(), name='examination_stats'),
-
-    # Video Segment Statistics API  
+    path(
+        "examinations/stats/", ExaminationStatsView.as_view(), name="examination_stats"
+    ),
+    # Video Segment Statistics API
     # GET /api/video-segment/stats/  (Note: singular 'segment' to match frontend)
     # Liefert Statistiken über Video-Segmente und Label-Verteilung
-    path('video-segment/stats/', VideoSegmentStatsView.as_view(), name='video_segment_stats'),
-
+    path(
+        "video-segment/stats/",
+        VideoSegmentStatsView.as_view(),
+        name="video_segment_stats",
+    ),
     # Alternative Video Segments Statistics API (plural version for compatibility)
     # GET /api/video-segments/stats/
-    path('video-segments/stats/', VideoSegmentStatsView.as_view(), name='video_segments_stats'),
-
+    path(
+        "video-segments/stats/",
+        VideoSegmentStatsView.as_view(),
+        name="video_segments_stats",
+    ),
     # Sensitive Meta Statistics API
     # GET /api/video/sensitivemeta/stats/
     # Liefert Statistiken über SensitiveMeta-Einträge und Verifikationsstatus
-    path('video/sensitivemeta/stats/', SensitiveMetaStatsView.as_view(), name='sensitive_meta_stats'),
-
+    path(
+        "video/sensitivemeta/stats/",
+        SensitiveMetaStatsView.as_view(),
+        name="sensitive_meta_stats",
+    ),
     # General Dashboard Statistics API
     # GET /api/stats/
     # Liefert allgemeine Übersichtsstatistiken für das Dashboard
-    path('stats/', GeneralStatsView.as_view(), name='general_stats'),
-
+    path("stats/", GeneralStatsView.as_view(), name="general_stats"),
     path(
-    'video-segments/stats/', 
-    VideoSegmentStatsView.as_view(), 
-    name='video_segments_stats'
+        "video-segments/stats/",
+        VideoSegmentStatsView.as_view(),
+        name="video_segments_stats",
     ),
-]
+]

endoreg_db/urls/upload.py

@@ -7,14 +7,6 @@ from endoreg_db.views import (
 
 urlpatterns = [
     # Upload endpoints
-    path(
-        'upload/', 
-        UploadFileView.as_view(), 
-        name='video_upload'
-    ),
-    path(
-        'upload/<uuid:id>/status', 
-        UploadStatusView.as_view(), 
-        name='upload_status'
-    ),
-]
+    path("upload/", UploadFileView.as_view(), name="video_upload"),
+    path("upload/<uuid:id>/status", UploadStatusView.as_view(), name="upload_status"),
+]

endoreg_db/utils/__init__.py

@@ -3,7 +3,12 @@
 # --- Imports from submodules ---
 
 # dataloader
-from endoreg_db.utils.video.ffmpeg_wrapper import assemble_video_from_frames, get_stream_info, transcode_video, transcode_videofile_if_required
+from endoreg_db.utils.video.ffmpeg_wrapper import (
+    assemble_video_from_frames,
+    get_stream_info,
+    transcode_video,
+    transcode_videofile_if_required,
+)
 
 from .dataloader import load_model_data_from_yaml
 
@@ -14,7 +19,10 @@ from .dates import ensure_aware_datetime, random_day_by_month_year, random_day_b
 from .env import DEBUG, DJANGO_SETTINGS_MODULE, get_env_var
 
 # file_operations
-from .file_operations import copy_with_progress, get_uuid_filename, rename_file_uuid
+from .file_operations import (
+    copy_with_progress,
+    get_content_hash_filename,
+)
 
 # hashs
 from .hashs import (
@@ -41,23 +49,15 @@ from .paths import data_paths
 
 # pydantic_models
 from .pydantic_models import DbConfig
-from .storage import (
-    delete_field_file,
-    ensure_local_file,
-    save_local_file,
-)
-from .storage import (
-    file_exists as storage_file_exists,
-)
+from .storage import delete_field_file, ensure_local_file, save_local_file
+from .storage import file_exists as storage_file_exists
 
 # validate_endo_roi
 from .validate_endo_roi import validate_endo_roi
 from .video import split_video
 
 # ffmpeg_wrapper
-from .video.ffmpeg_wrapper import (
-    extract_frames,
-)
+from .video.ffmpeg_wrapper import extract_frames
 
 # --- Exports ---
 
@@ -77,13 +77,12 @@ __all__ = [
     "get_hash_string",
     "get_patient_examination_hash",
     "get_pdf_hash",
-    "get_uuid_filename",
+    "get_content_hash_filename",
     "get_video_hash",
     "guess_name_gender",
     "load_model_data_from_yaml",
     "random_day_by_month_year",
     "random_day_by_year",
-    "rename_file_uuid",
     "validate_endo_roi",
     "assemble_video_from_frames",  # Updated name
     "get_stream_info",

endoreg_db/utils/ai/__init__.py

@@ -6,4 +6,4 @@ __all__ = [
     "InferenceDataset",
     "MultiLabelClassificationNet",
     "Classifier",
-]
+]

endoreg_db/utils/ai/data_loader_for_model_input.py

@@ -0,0 +1,262 @@
+# endoreg_db/utils/ai/data_loader_for_model_training.py
+
+from __future__ import annotations
+
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, List, Optional, TypedDict
+
+from django.db import models
+
+from endoreg_db.models import (
+    AIDataSet,
+    Frame,
+    ImageClassificationAnnotation,
+    Label,
+    LabelSet,
+)
+
+
+class ImageMultilabelDataset(TypedDict):
+    """
+    In-memory representation of an image multi-label training dataset.
+
+    All lists are aligned by index:
+
+        image_paths[i]   -> path to image file for sample i
+        label_vectors[i] -> list[int|None] of length == len(labels)
+        label_masks[i]   -> list[int]       of length == len(labels)
+
+    Where:
+        - label_vectors[i][j] is:
+            1     -> positive annotation (value=True)
+            0     -> negative annotation (value=False)
+            None  -> UNKNOWN (no annotation for that (frame, label))
+
+        - label_masks[i][j] is:
+            1 -> this entry participates in the loss (0 or 1 is known)
+            0 -> IGNORE in the loss (value was None)
+    """
+
+    # type description of the returned dict.
+
+    image_paths: List[str]
+    label_vectors: List[List[Optional[int]]]
+    label_masks: List[List[int]]
+    labels: List[Label]
+    labelset: LabelSet
+
+    # New: keep track of which DB rows were used, and their legacy exam ids
+    frame_ids: List[int]  # Frame.pk for each sample
+    old_examination_ids: List[Optional[int]]  # may be None if not set
+
+
+def _infer_labelset_from_annotations(
+    annotations_qs: models.QuerySet[ImageClassificationAnnotation],
+) -> LabelSet:
+    """
+    Try to infer a unique LabelSet from the labels used in the annotations.
+
+    Strategy:
+        1. Collect all distinct label_ids from the annotations.
+        2. Fetch all Label objects + their label_sets.
+        3. Compute the intersection of all label_sets across all labels.
+        4. If there is exactly ONE common LabelSet, return it.
+           Otherwise, raise NotImplementedError for now.
+    """
+    label_ids = list(annotations_qs.values_list("label_id", flat=True).distinct())
+    if not label_ids:
+        raise ValueError("Cannot infer LabelSet: annotations queryset has no labels.")
+
+    labels_qs = Label.objects.filter(id__in=label_ids).prefetch_related("label_sets")
+    labelsets_for_each_label = []
+
+    for lbl in labels_qs:
+        # lbl.label_sets is the reverse of LabelSet.labels M2M
+        ls_ids = list(lbl.label_sets.values_list("id", flat=True))
+        if not ls_ids:
+            # This label is not part of any LabelSet -> ambiguous
+            raise NotImplementedError(
+                f"Label id={lbl.id}, name='{lbl.name}' is not part of any LabelSet. "
+                "Explicit LabelSet selection is required."
+            )
+        labelsets_for_each_label.append(set(ls_ids))
+
+    # Intersection of all labelset id sets
+    common_ids = set.intersection(*labelsets_for_each_label)
+    if not common_ids:
+        raise NotImplementedError(
+            "No common LabelSet across all labels in this AIDataSet. "
+            "Please specify a LabelSet explicitly."
+        )
+    if len(common_ids) > 1:
+        raise NotImplementedError(
+            "More than one common LabelSet found for the labels in this AIDataSet. "
+            "Please specify a LabelSet explicitly to disambiguate."
+        )
+
+    ls_id = next(iter(common_ids))
+    return LabelSet.objects.get(id=ls_id)
+
+
+def build_image_multilabel_dataset_from_db(
+    dataset: AIDataSet,
+    labelset: Optional[LabelSet] = None,
+) -> ImageMultilabelDataset:
+    """
+    Build an in-memory multilabel dataset for an IMAGE-based AIDataSet.
+
+    Steps:
+        1. Take all ImageClassificationAnnotation rows linked to this AIDataSet
+           (via dataset.image_annotations M2M).
+        2. Determine the LabelSet (either explicitly given or inferred).
+        3. For each used Frame, build:
+            - an image path
+            - a label vector (1, 0, or None for each label in LabelSet)
+            - a mask vector (1 where known, 0 where unknown)
+        4. Return a dict that can be wrapped in a torch/tf Dataset.
+
+    NOTE:
+        - This function does NOT write anything to the DB.
+        - It only reads DB rows and returns Python structures.
+    """
+    if dataset.dataset_type != AIDataSet.DATASET_TYPE_IMAGE:
+        raise ValueError(
+            f"build_image_multilabel_dataset_from_db expected dataset_type='image', "
+            f"got '{dataset.dataset_type}' for AIDataSet id={dataset.id}."
+        )
+
+    # Get the annotation relation dynamically (for future video/text types)
+    annotations_qs = dataset.get_annotations_queryset().select_related("frame", "label")
+
+    if annotations_qs.count() == 0:
+        raise ValueError(
+            f"AIDataSet id={dataset.id} has no annotations attached. "
+            "Make sure your import script populated image_annotations."
+        )
+
+    # Decide which LabelSet to use
+    if labelset is None:
+        labelset = _infer_labelset_from_annotations(annotations_qs)
+
+    # Fixed label order (= fixed column order for the label vectors)
+    labels_in_order: List[Label] = labelset.get_labels_in_order()
+    if not labels_in_order:
+        raise ValueError(
+            f"LabelSet id={labelset.id}, name='{labelset.name}' has no labels."
+        )
+
+    num_labels = len(labels_in_order)
+    label_index: Dict[int, int] = {
+        lbl.id: idx for idx, lbl in enumerate(labels_in_order)
+    }
+
+    # Group annotations by frame
+    anns_by_frame: Dict[int, List[ImageClassificationAnnotation]] = defaultdict(list)
+    frames_order: List[int] = []
+
+    for ann in annotations_qs:
+        frame_id = ann.frame_id
+        if frame_id not in anns_by_frame:
+            frames_order.append(frame_id)
+        anns_by_frame[frame_id].append(ann)
+
+    # Build vectors
+    image_paths: List[str] = []
+    label_vectors: List[List[Optional[int]]] = []
+    label_masks: List[List[int]] = []
+
+    # New: id tracking for splitting / logging
+    frame_ids: List[int] = []
+    old_examination_ids: List[Optional[int]] = []
+
+    # Cache frames to avoid repeated DB hits
+    frame_obj_by_id: Dict[int, Frame] = {}
+
+    for frame_id in frames_order:
+        frame_annotations = anns_by_frame[frame_id]
+
+        # Resolve frame object (from first annotation of this frame)
+        frame = frame_obj_by_id.get(frame_id)
+        if frame is None:
+            frame = frame_annotations[0].frame
+            frame_obj_by_id[frame_id] = frame
+
+            # New: remember DB ids for this sample
+        frame_ids.append(frame_id)
+        old_examination_ids.append(getattr(frame, "old_examination_id", None))
+
+        # Start with unknown for all labels
+        vec: List[Optional[int]] = [None] * num_labels
+
+        # Fill with 1/0 where we have annotations
+        for ann in frame_annotations:
+            idx = label_index.get(ann.label_id)
+            if idx is None:
+                # Label not part of this LabelSet: ignore
+                continue
+            vec[idx] = 1 if ann.value else 0
+
+        # Build mask: 1 where vec is known, 0 where unknown
+        mask: List[int] = [0 if v is None else 1 for v in vec]
+
+        # Resolve absolute image path from the Frame model
+        file_path: Path = frame.file_path
+        image_paths.append(str(file_path))
+        label_vectors.append(vec)
+        label_masks.append(mask)
+
+    return ImageMultilabelDataset(
+        image_paths=image_paths,
+        label_vectors=label_vectors,
+        label_masks=label_masks,
+        labels=labels_in_order,
+        labelset=labelset,
+        frame_ids=frame_ids,
+        old_examination_ids=old_examination_ids,
+    )
+
+
+def build_dataset_for_training(
+    dataset: AIDataSet,
+    labelset: Optional[LabelSet] = None,
+):
+    """
+    High-level entry point to build a training dataset from an AIDataSet row.
+
+    It inspects:
+        - dataset.dataset_type
+        - dataset.ai_model_type
+
+    and dispatches to the appropriate builder.
+
+    For now, we support:
+        - dataset_type = "image"
+        - ai_model_type = "image_multilabel_classification"
+
+    Later, you can extend this to:
+        - video segmentation
+        - text classification
+        etc.
+    """
+    # IMAGE MULTILABEL CASE
+    if (
+        dataset.dataset_type == AIDataSet.DATASET_TYPE_IMAGE
+        and dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_IMAGE_MULTILABEL
+    ):
+        return build_image_multilabel_dataset_from_db(dataset, labelset=labelset)
+
+    # FUTURE EXTENSIONS (example structure, not yet implemented):
+    # if dataset.dataset_type == AIDataSet.DATASET_TYPE_VIDEO and \
+    #    dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_VIDEO_SEGMENTATION:
+    #     return build_video_segmentation_dataset_from_db(dataset, labelset=labelset)
+    #
+    # if dataset.dataset_type == AIDataSet.DATASET_TYPE_TEXT and \
+    #    dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_TEXT_CLASSIFICATION:
+    #     return build_text_classification_dataset_from_db(dataset, labelset=labelset)
+
+    raise NotImplementedError(
+        f"No dataset builder implemented for "
+        f"dataset_type='{dataset.dataset_type}', "
+        f"ai_model_type='{dataset.ai_model_type}'."
+    )