endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/management/commands/summarize_db_content.py

@@ -0,0 +1,280 @@
+from django.core.management.base import BaseCommand
+from django.apps import apps
+from django.db.models import Min, Max, Count, fields
+from django.utils.timezone import is_aware, make_naive
+import datetime
+import os
+from openpyxl import Workbook
+import csv
+
+
+class Command(BaseCommand):
+    help = "Generates a structured report summarizing the database content of custom endoreg_db models and saves it to Excel and CSV files, excluding models with zero records from the files."  # Updated help text
+
+    def handle(self, *args, **options):
+        """
+        Generates a summary report of all models in the 'endoreg_db' app and exports the results to Excel and CSV files.
+
+        For each model, counts the total number of records and includes only models with at least one record in the output files. For models with records, attempts to display the range of values for common date or datetime fields and provides value counts for up to three categorical or ForeignKey fields, using heuristics based on field type and name. Handles missing app configuration, file saving errors, and aggregation exceptions gracefully, outputting progress and warnings to the console. Creates a 'data' directory within the app if it does not exist and saves the reports there.
+        """
+        self.stdout.write(
+            self.style.SUCCESS(
+                "Starting database content summarization for endoreg_db models..."
+            )
+        )
+
+        try:
+            app_config = apps.get_app_config("endoreg_db")
+        except LookupError:
+            self.stdout.write(
+                self.style.ERROR(
+                    "Could not find the 'endoreg_db' app. Make sure it's correctly installed and configured."
+                )
+            )
+            return
+
+        data_dir = os.path.join(app_config.path, "data")
+        if not os.path.exists(data_dir):
+            os.makedirs(data_dir)
+            self.stdout.write(self.style.SUCCESS(f"Created directory: {data_dir}"))
+
+        excel_file_path = os.path.join(data_dir, "db_summary.xlsx")
+        csv_file_path = os.path.join(data_dir, "db_summary.csv")
+
+        # --- Excel Setup ---
+        wb = Workbook()
+        ws = wb.active
+        ws.title = "DB Summary"
+        excel_headers = ["Model Name", "Total Records"]
+        ws.append(excel_headers)
+        # --- End Excel Setup ---
+
+        # --- CSV Setup ---
+        csv_data_for_file = [
+            excel_headers
+        ]  # Initialize with headers for CSV, will only store rows with count > 0
+        # --- End CSV Setup ---
+
+        models = app_config.get_models()
+
+        if not models:
+            self.stdout.write(
+                self.style.WARNING("No models found in the 'endoreg_db' app.")
+            )
+            try:
+                wb.save(excel_file_path)  # wb will only have headers
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"Empty summary report saved to {excel_file_path}"
+                    )
+                )
+                with open(csv_file_path, "w", newline="") as f:
+                    writer = csv.writer(f)
+                    writer.writerows(
+                        csv_data_for_file
+                    )  # csv_data_for_file will only contain headers
+                self.stdout.write(
+                    self.style.SUCCESS(f"Empty summary report saved to {csv_file_path}")
+                )
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"Error saving files: {e}"))
+            return
+
+        for model in models:
+            model_name = model.__name__
+            self.stdout.write(self.style.HTTP_INFO(f"\n--- Model: {model_name} ---"))
+
+            try:
+                count = model.objects.count()
+                self.stdout.write(f"  Total records: {count}")
+
+                if count == 0:
+                    self.stdout.write(
+                        self.style.NOTICE(
+                            "  No records found for this model. Skipping from file output."
+                        )
+                    )
+                    continue  # Skip further processing for this model if count is 0 for file output
+
+                # --- Add to Excel (only if count > 0) ---
+                ws.append([model_name, count])
+                # --- End Add to Excel ---
+
+                # --- Add to CSV data list (only if count > 0) ---
+                csv_data_for_file.append([model_name, count])
+                # --- End Add to CSV data list ---
+
+                # Attempt to get date ranges for common date/datetime fields
+                date_fields_to_check = [
+                    "created_at",
+                    "updated_at",
+                    "timestamp",
+                    "date",
+                    "start_time",
+                    "end_time",
+                    "examination_date",
+                    "birth_date",
+                    "record_date",
+                ]
+                processed_date_field = False
+                for field_obj in model._meta.get_fields():
+                    if field_obj.name in date_fields_to_check and isinstance(
+                        field_obj, (fields.DateField, fields.DateTimeField)
+                    ):
+                        try:
+                            aggregation = model.objects.aggregate(
+                                min_date=Min(field_obj.name),
+                                max_date=Max(field_obj.name),
+                            )
+                            min_val = aggregation.get("min_date")
+                            max_val = aggregation.get("max_date")
+
+                            if min_val is not None and max_val is not None:
+                                if isinstance(min_val, datetime.datetime) and is_aware(
+                                    min_val
+                                ):
+                                    min_val = make_naive(min_val).strftime(
+                                        "%Y-%m-%d %H:%M:%S"
+                                    )
+                                elif isinstance(min_val, datetime.date):
+                                    min_val = min_val.strftime("%Y-%m-%d")
+
+                                if isinstance(max_val, datetime.datetime) and is_aware(
+                                    max_val
+                                ):
+                                    max_val = make_naive(max_val).strftime(
+                                        "%Y-%m-%d %H:%M:%S"
+                                    )
+                                elif isinstance(max_val, datetime.date):
+                                    max_val = max_val.strftime("%Y-%m-%d")
+
+                                self.stdout.write(
+                                    f"  {field_obj.verbose_name.capitalize()} range: {min_val} to {max_val}"
+                                )
+                                processed_date_field = True
+                                break  # Process only the first found relevant date field for brevity
+                        except Exception as e:
+                            self.stdout.write(
+                                self.style.WARNING(
+                                    f"  Could not aggregate date range for '{field_obj.name}': {e}"
+                                )
+                            )
+
+                # Attempt to get value counts for potential categorical/ForeignKey fields
+                categorical_fields_found = 0
+                MAX_CATEGORICAL_FIELDS_TO_ANALYZE = 3
+                MAX_DISTINCT_VALUES_TO_SHOW = 5
+
+                for field in model._meta.get_fields():
+                    if categorical_fields_found >= MAX_CATEGORICAL_FIELDS_TO_ANALYZE:
+                        break
+
+                    is_potential_categorical = False
+                    field_name_lower = field.name.lower()
+
+                    if (
+                        field.many_to_one or field.one_to_one
+                    ):  # ForeignKey or OneToOneField
+                        if field.related_model:
+                            is_potential_categorical = True
+                    elif isinstance(
+                        field,
+                        (
+                            fields.CharField,
+                            fields.IntegerField,
+                            fields.BooleanField,
+                            fields.TextField,
+                        ),
+                    ):
+                        # Heuristic for common categorical fields by name or type
+                        if (
+                            "type" in field_name_lower
+                            or "status" in field_name_lower
+                            or "gender" in field_name_lower
+                            or "category" in field_name_lower
+                            or isinstance(field, fields.BooleanField)
+                        ):
+                            is_potential_categorical = True
+                        # For CharField/TextField, also consider if it has few distinct values (expensive to check upfront for all)
+                        # We'll rely on the naming heuristic primarily or if it's a boolean.
+
+                    if is_potential_categorical:
+                        try:
+                            self.stdout.write(
+                                f"  Value counts for '{field.verbose_name.capitalize() if hasattr(field, 'verbose_name') else field.name}':"
+                            )
+
+                            # QuerySet for distinct values and their counts
+                            values_qs = (
+                                model.objects.values(field.name)
+                                .annotate(count=Count(field.name))
+                                .order_by("-count")
+                            )
+
+                            distinct_values_count = values_qs.count()
+
+                            if distinct_values_count == 0:
+                                self.stdout.write(
+                                    "    No distinct values found or field is often NULL."
+                                )
+                                continue
+
+                            for i, item in enumerate(values_qs):
+                                if i >= MAX_DISTINCT_VALUES_TO_SHOW:
+                                    self.stdout.write(
+                                        f"    ... and {distinct_values_count - MAX_DISTINCT_VALUES_TO_SHOW} more distinct values."
+                                    )
+                                    break
+                                val = item[field.name]
+                                val_display = (
+                                    str(val) if val is not None else "None/NULL"
+                                )
+                                # Truncate long string values for display
+                                if isinstance(val, str) and len(val_display) > 50:
+                                    val_display = val_display[:47] + "..."
+                                self.stdout.write(
+                                    f"    - {val_display}: {item['count']}"
+                                )
+
+                            categorical_fields_found += 1
+
+                        except Exception as e:
+                            # Some fields (like generic relations or complex custom fields) might not support this directly
+                            self.stdout.write(
+                                self.style.WARNING(
+                                    f"  Could not get value counts for '{field.name}': {e}"
+                                )
+                            )
+
+            except Exception as e:
+                self.stdout.write(
+                    self.style.ERROR(f"  Error processing model {model_name}: {e}")
+                )
+
+        # --- Save Excel File ---
+        try:
+            wb.save(excel_file_path)
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"\nDatabase summary report saved to {excel_file_path}"
+                )
+            )
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"\nError saving Excel file: {e}"))
+        # --- End Save Excel File ---
+
+        # --- Save CSV File ---
+        try:
+            with open(csv_file_path, "w", newline="") as f:
+                writer = csv.writer(f)
+                writer.writerows(csv_data_for_file)
+            self.stdout.write(
+                self.style.SUCCESS(f"Database summary report saved to {csv_file_path}")
+            )
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"Error saving CSV file: {e}"))
+        # --- End Save CSV File ---
+
+        self.stdout.write(
+            self.style.SUCCESS("\nDatabase content summarization finished.")
+        )

endoreg_db/management/commands/train_image_multilabel_model.py

@@ -0,0 +1,144 @@
+# endoreg_db/management/commands/train_image_multilabel_model.py
+
+from __future__ import annotations
+
+from django.core.management.base import BaseCommand, CommandError
+
+from endoreg_db.models import AIDataSet
+from endoreg_db.utils.ai.data_loader_for_model_training import (
+    build_dataset_for_training,
+)
+
+
+class Command(BaseCommand):
+    help = (
+        "Train / fine-tune the image multi-label model on a given AIDataSet.\n"
+        "\n"
+        "This command is fully dynamic:\n"
+        "- It uses the AIDataSet row (id) to decide which annotations to use.\n"
+        "- It *infers* the LabelSet from the labels in those annotations.\n"
+        "- It builds image_paths, label_vectors, and label_masks for training.\n"
+        "- It prints detailed debug information about the chosen LabelSet and samples.\n"
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--dataset-id",
+            type=int,
+            required=True,
+            help="Primary key of the AIDataSet to use for training.",
+        )
+
+    def handle(self, *args, **options):
+        dataset_id = options["dataset_id"]
+
+        try:
+            dataset = AIDataSet.objects.get(id=dataset_id)
+        except AIDataSet.DoesNotExist:
+            raise CommandError(f"AIDataSet with id={dataset_id} does not exist.")
+
+        # ------------------------------------------------------------------
+        # Basic dataset info (AIDataSet row)
+        # ------------------------------------------------------------------
+        self.stdout.write(
+            self.style.NOTICE(
+                f"Using AIDataSet id={dataset.id}, "
+                f"name={dataset.name!r}, "
+                f"dataset_type={dataset.dataset_type!r}, "
+                f"ai_model_type={dataset.ai_model_type!r}"
+            )
+        )
+
+        # Build in-memory dataset (completely dynamic, labelset inferred automatically)
+        data = build_dataset_for_training(dataset)
+
+        image_paths = data["image_paths"]
+        label_vectors = data["label_vectors"]
+        label_masks = data["label_masks"]
+        labels = data["labels"]
+        labelset = data["labelset"]
+
+        # Optional: additional meta info, if present
+        frame_ids = data.get("frame_ids", [])
+        old_examination_ids = data.get("old_examination_ids", [])
+
+        # ------------------------------------------------------------------
+        # Debug: show which LabelSet was picked and its labels
+        # ------------------------------------------------------------------
+        self.stdout.write(self.style.NOTICE("Inferred LabelSet for this AIDataSet:"))
+        self.stdout.write(
+            f"  LabelSet id={labelset.id}, "
+            f"name={labelset.name!r}, "
+            f"version={labelset.version}"
+        )
+        self.stdout.write("  Labels (index, id, name):")
+        for idx, lbl in enumerate(labels):
+            self.stdout.write(f"    [{idx}] id={lbl.id}, name={lbl.name!r}")
+
+        # ------------------------------------------------------------------
+        # Summary of constructed dataset
+        # ------------------------------------------------------------------
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"\nBuilt training dataset from AIDataSet id={dataset.id}:\n"
+                f"- #samples: {len(image_paths)}\n"
+                f"- #labels:  {len(labels)}"
+            )
+        )
+
+        # ------------------------------------------------------------------
+        # Debug: print each frame's path, label vector and mask
+        # NOTE: If your dataset is very large, this will spam the console.
+        #       For now you want full transparency, so we print all.
+        # ------------------------------------------------------------------
+        MAX_PRINT = 50
+        self.stdout.write(self.style.NOTICE("\nPer-sample debug output:"))
+        for i, (path, vec, mask) in enumerate(
+            zip(image_paths, label_vectors, label_masks)
+        ):
+            if i >= MAX_PRINT:
+                self.stdout.write(
+                    self.style.WARNING(
+                        f"... ({len(image_paths) - MAX_PRINT} more samples not shown)"
+                    )
+                )
+                break
+
+            frame_id = frame_ids[i] if i < len(frame_ids) else None
+            old_exam = old_examination_ids[i] if i < len(old_examination_ids) else None
+
+            self.stdout.write(
+                f"  Sample {i}:"
+                f"\n    path = {path!r}"
+                f"\n    frame_id = {frame_id}"
+                f"\n    old_examination_id = {old_exam}"
+                f"\n    vector (1/0/None) = {vec}"
+                f"\n    mask (1=use, 0=ignore) = {mask}"
+            )
+
+        # ------------------------------------------------------------------
+        # TODO: Insert your actual training loop here
+        # ------------------------------------------------------------------
+        # Example (pseudo-code):
+        #
+        #   import torch
+        #   from torch.utils.data import Dataset, DataLoader
+        #
+        #   class EndoDataset(Dataset):
+        #       def __init__(self, image_paths, label_vectors, label_masks):
+        #           ...
+        #
+        #   ds = EndoDataset(image_paths, label_vectors, label_masks)
+        #   loader = DataLoader(ds, batch_size=32, shuffle=True)
+        #
+        #   for batch in loader:
+        #       # forward, compute loss using mask, backward, step, ...
+        #
+        # ------------------------------------------------------------------
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                "\nDataset construction finished. "
+                "You can now plug this into your model training code."
+            )
+        )

endoreg_db/management/commands/validate_video_files.py

@@ -0,0 +1,189 @@
+"""
+Django management command to validate video file existence and accessibility.
+"""
+
+import logging
+from pathlib import Path
+
+from django.core.management.base import BaseCommand
+
+from endoreg_db.models import VideoFile
+
+logger = logging.getLogger(__name__)
+
+# TODO Review if this is still used. Delete if not.
+
+
+class Command(BaseCommand):
+    help = "Validate video file existence and accessibility"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--video-id",
+            type=int,
+            help="Check specific video ID",
+        )
+        parser.add_argument(
+            "--verbose",
+            action="store_true",
+            help="Enable verbose output",
+        )
+
+    def handle(self, *args, **options):
+        """Validate video files and their accessibility."""
+        verbose = options["verbose"]
+        video_id = options.get("video_id")
+        fix_missing = options["fix_missing", False]
+
+        if verbose:
+            self.stdout.write(self.style.SUCCESS("Starting video validation..."))
+
+        # Query videos
+        if video_id:
+            try:
+                videos = [VideoFile.objects.get(pk=video_id)]
+                self.stdout.write(f"Checking specific video ID: {video_id}")
+            except VideoFile.DoesNotExist:
+                self.stdout.write(
+                    self.style.ERROR(f"Video with ID {video_id} not found")
+                )
+                return
+        else:
+            videos = VideoFile.objects.all()
+            self.stdout.write(f"Checking {videos.count()} videos...")
+
+        missing_files = []
+        accessible_files = []
+        corrupted_files = []
+
+        for video in videos:
+            video_status = self.check_video_file(video, verbose)
+
+            if video_status["status"] == "missing":
+                missing_files.append(video_status)
+            elif video_status["status"] == "corrupted":
+                corrupted_files.append(video_status)
+            else:
+                accessible_files.append(video_status)
+
+        # Report results
+        self.stdout.write("\n" + "=" * 60)
+        self.stdout.write(self.style.SUCCESS("VALIDATION COMPLETE"))
+        self.stdout.write("=" * 60)
+
+        self.stdout.write(f"✅ Accessible videos: {len(accessible_files)}")
+        self.stdout.write(f"❌ Missing files: {len(missing_files)}")
+        self.stdout.write(f"⚠️  Potentially corrupted: {len(corrupted_files)}")
+
+        if missing_files:
+            self.stdout.write(self.style.WARNING("\nMISSING FILES:"))
+            for file_info in missing_files:
+                self.stdout.write(
+                    f"  - Video ID {file_info['video_id']}: {file_info['error']}"
+                )
+                if fix_missing:
+                    self.stdout.write("    → Marking as inactive (if applicable)")
+
+        if corrupted_files:
+            self.stdout.write(self.style.WARNING("\nPOTENTIALLY CORRUPTED FILES:"))
+            for file_info in corrupted_files:
+                self.stdout.write(
+                    f"  - Video ID {file_info['video_id']}: {file_info['error']}"
+                )
+
+        if verbose and accessible_files:
+            self.stdout.write(self.style.SUCCESS("\nACCESSIBLE FILES:"))
+            for file_info in accessible_files[:10]:  # Show first 10
+                self.stdout.write(
+                    f"  ✅ Video ID {file_info['video_id']}: {file_info['path']} ({file_info['size_mb']:.1f} MB)"
+                )
+
+            if len(accessible_files) > 10:
+                self.stdout.write(f"  ... and {len(accessible_files) - 10} more")
+
+    def check_video_file(self, video, verbose=False):
+        """
+        Check a single video file for existence and basic accessibility.
+
+        Returns:
+            dict: Status information about the video file
+        """
+        video_info = {
+            "video_id": video.id,
+            "video_uuid": str(video.video_hash) if hasattr(video, "uuid") else "N/A",
+            "status": "unknown",
+            "path": None,
+            "size_mb": 0,
+            "error": None,
+        }
+
+        # Helper to check a file attribute
+        def _check_file_attr(obj, attr, path_getter=None, label=None):
+            if not hasattr(obj, attr):
+                return None
+            file_field = getattr(obj, attr)
+            if not file_field:
+                return None
+            try:
+                file_path = Path(path_getter(file_field) if path_getter else file_field)
+                info = video_info.copy()
+                info["path"] = str(file_path)
+                if not file_path.exists():
+                    info["status"] = "missing"
+                    info["error"] = (
+                        f"{label or attr.replace('_', ' ').title()} does not exist: {file_path}"
+                    )
+                    return info
+                file_size = file_path.stat().st_size
+                info["size_mb"] = file_size / (1024 * 1024)
+                if file_size == 0:
+                    info["status"] = "corrupted"
+                    info["error"] = (
+                        f"{label or attr.replace('_', ' ').title()} exists but has zero size"
+                    )
+                else:
+                    info["status"] = "accessible"
+                return info
+            except (ValueError, OSError) as e:
+                info = video_info.copy()
+                info["path"] = str(getattr(file_field, "path", file_field))
+                info["status"] = "corrupted"
+                info["error"] = (
+                    f"Cannot access {label or attr.replace('_', ' ').title()}: {e}"
+                )
+                return info
+
+        # Try each file attribute in order of preference
+        result = None
+        # active_file_path: direct path string
+        result = _check_file_attr(video, "active_file_path", label="Active file path")
+        if result:
+            return result
+        # active_file: Django FileField
+        result = _check_file_attr(
+            video, "active_file", path_getter=lambda f: f.path, label="Active file"
+        )
+        if result:
+            return result
+        # raw_file: Django FileField
+        result = _check_file_attr(
+            video, "raw_file", path_getter=lambda f: f.path, label="Raw file"
+        )
+        if result:
+            return result
+        # processed_file: Django FileField
+        result = _check_file_attr(
+            video,
+            "processed_file",
+            path_getter=lambda f: f.path,
+            label="Processed file",
+        )
+        if result:
+            return result
+
+        # If none found
+        video_info["status"] = "missing"
+        video_info["error"] = (
+            "No video file paths found (no active_file, raw_file, or processed_file)"
+        )
+        return video_info

endoreg_db/management/commands/video_validation.py

@@ -0,0 +1,20 @@
+from typing import TYPE_CHECKING
+from django.core.management.base import BaseCommand
+
+"""
+This command handles the video validation as seen in tests _pipe_1
+"""
+
+if TYPE_CHECKING:
+    pass
+
+
+class Command(BaseCommand):
+    help = "Data extraction and validation of video files in the database and updating their states accordingly."
+
+    def handle(self, *args, **options):
+        """
+        Validates video files stored in the database and updates their states based on validation results.
+
+        This method is intended to be executed as a Django management command to ensure the integrity and correct status of video file records.
+        """

endoreg_db/mermaid/Overall_flow_patient_finding_intervention.md

@@ -0,0 +1,10 @@
+```mermaid
+graph TD;
+
+    %% Main Patient Workflow
+    A[Patient: John] --> B[Examination: Colonoscopy]
+    B --> C[Finding: Colon Polyp]
+    C --> D[Finding Location: Colonoscopy Default - Right Flexure]
+    D --> E[Finding Morphology: Colon Lesion Planarity Default - Colon Lesion Planarity Excavated]
+    E --> F[Intervention: Colon Lesion Polypectomy - Hot Snare]
+    F --> G[Report Generation]

endoreg_db/mermaid/anonymized_image_annotation.md

@@ -0,0 +1,20 @@
+```mermaid
+graph TD;
+
+    %% File Handling Process
+    A[**Uploaded File**] -->|Original File Uploaded| B[**Anonymized File**]
+    B -->|File Anonymized & Stored| C[**Anonymous Image Annotation**]
+
+    %% Image Annotation & Processing
+    C -->|Label Assigned| D[**Anonymized Image Label**]
+    C -->|Detected Personal Info Removed| E[**Dropped Name**]
+
+    %% Relationships & Processing
+    E -->|Stored for Record-Keeping| C;
+    D -->|Predefined Labels Used| C;
+
+    %% Example Data (Corrected & Meaningful)
+    X1(["**Example: Uploaded colonoscopy.jpg**"]) -.-> A;
+    X2(["**Example: anonymized_colonoscopy.jpg**"]) -.-> B;
+    X3(["**Example: Label - 'Polyp Detected'**"]) -.-> D;
+    X4(["**Example: Name 'John Doe' removed**"]) -.-> E;