endoreg-db 0.8.8.9__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

endoreg_db/management/commands/load_legacy_data.py

@@ -0,0 +1,303 @@
+# endoreg_db/management/commands/load_legacy_data.py
+
+import json
+from pathlib import Path
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+
+from endoreg_db.models import (
+    AIDataSet,
+    Frame,
+    ImageClassificationAnnotation,
+    Label,
+    LabelSet,
+    VideoFile,
+)
+
+DEFAULT_LABELSET_NAME = (
+    "multilabel_classification_colonoscopy_default"  # must be present in the DB
+)
+DEFAULT_LABELSET_VERSION = 1
+
+
+class Command(BaseCommand):
+    help = (
+        "Import legacy multilabel image data from JSONL + images into the database.\n"
+        "- Creates Frames linked to a given VideoFile\n"
+        "- Creates ImageClassificationAnnotations (value=True) for each listed label\n"
+        "- Reuses/extends an existing LabelSet\n"
+        "- Fills an AIDataSet (image dataset) with all annotations via image_annotations"
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--jsonl-path",
+            type=str,
+            default=str(
+                Path(settings.BASE_DIR)
+                / "data"
+                / "legacy_data"
+                / "legacy_img_dicts.jsonl"
+            ),
+            help="Path to legacy_img_dicts.jsonl",
+        )
+        parser.add_argument(
+            "--images-root",
+            type=str,
+            default=str(Path(settings.BASE_DIR) / "data" / "legacy_data" / "images"),
+            help="Root directory containing legacy images.",
+        )
+        # All imported frames need to belong to some VideoFile.
+        parser.add_argument(
+            "--video-id",
+            type=int,
+            required=True,
+            help="ID of an existing VideoFile to attach all legacy Frames to.",
+        )
+        parser.add_argument(
+            "--dataset-name",
+            type=str,
+            default="legacy_multilabel_dataset_v1",  # later change this if needed
+            help="Name for the created/reused AIDataSet.",
+        )
+        parser.add_argument(
+            "--dataset-description",
+            type=str,
+            default="Legacy multilabel colonoscopy dataset imported from JSONL.",
+            help="Description for the created AIDataSet.",
+        )
+        parser.add_argument(
+            "--labelset-name",
+            type=str,
+            default=DEFAULT_LABELSET_NAME,
+            help="LabelSet name to use (must exist).",
+        )
+        parser.add_argument(
+            "--labelset-version",
+            type=int,
+            default=DEFAULT_LABELSET_VERSION,
+            help="LabelSet version to use (must exist).",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Parse and validate, but do not write anything to the database.",
+        )
+
+    def handle(self, *args, **options):
+        jsonl_path = Path(options["jsonl_path"])
+        images_root = Path(options["images_root"])
+        video_id = options["video_id"]
+        dataset_name = options["dataset_name"]
+        dataset_description = options["dataset_description"]
+        labelset_name = options["labelset_name"]
+        labelset_version = options["labelset_version"]
+        dry_run = options["dry_run"]
+
+        # --- Basic checks ---
+        if not jsonl_path.exists():
+            raise CommandError(f"JSONL file not found: {jsonl_path}")
+
+        if not images_root.exists():
+            raise CommandError(f"Images root directory not found: {images_root}")
+
+        try:
+            video = VideoFile.objects.get(id=video_id)
+        except VideoFile.DoesNotExist:
+            raise CommandError(f"VideoFile with id={video_id} does not exist.")
+
+        self.stdout.write(
+            self.style.NOTICE(f"Using VideoFile id={video.id} for all Frames.")
+        )
+
+        # Ensure this VideoFile uses the legacy images folder as its frame_dir
+        # IMPORTANT: we only set this if frame_dir is empty, so we don't break other videos.
+        if not video.frame_dir:
+            video.frame_dir = str(images_root)  # images_root is Path(...)
+            video.save(update_fields=["frame_dir"])
+            self.stdout.write(
+                self.style.NOTICE(
+                    f"Set frame_dir for VideoFile id={video.id} to '{video.frame_dir}' "
+                    "for legacy image frames."
+                )
+            )
+        else:
+            self.stdout.write(
+                self.style.WARNING(
+                    f"VideoFile id={video.id} already has frame_dir='{video.frame_dir}'. "
+                    "Legacy Frames will be resolved relative to this directory."
+                )
+            )
+
+        # --- Use existing LabelSet (v1) ---
+        labelset = self._get_existing_labelset(
+            labelset_name=labelset_name,
+            labelset_version=labelset_version,
+        )
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Using LabelSet '{labelset.name}' (version={labelset.version}, id={labelset.id})."
+            )
+        )
+
+        # --- Create or reuse AIDataSet (image dataset) ---
+        if dry_run:
+            self.stdout.write(
+                self.style.WARNING("Dry run: AIDataSet will NOT be created.")
+            )
+            ai_dataset = None
+        else:
+            ai_dataset, created = AIDataSet.objects.get_or_create(
+                name=dataset_name,
+                defaults={
+                    "description": dataset_description,
+                    "dataset_type": AIDataSet.DATASET_TYPE_IMAGE,
+                    "ai_model_type": AIDataSet.AI_MODEL_TYPE_IMAGE_MULTILABEL,
+                    "is_active": True,
+                },
+            )
+            if created:
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"Created AIDataSet id={ai_dataset.id}, name='{ai_dataset.name}'."
+                    )
+                )
+            else:
+                # Use the helper method so this works even if we add video/text later
+                current_count = ai_dataset.get_annotations_queryset().count()
+                self.stdout.write(
+                    self.style.WARNING(
+                        f"Re-using existing AIDataSet id={ai_dataset.id}, name='{ai_dataset.name}'. "
+                        f"(Current annotation_count={current_count})"
+                    )
+                )
+
+        frame_counter = 0
+        annotation_counter = 0
+
+        # Use transaction unless dry-run
+        ctx = transaction.atomic if not dry_run else self._noop_context
+
+        with ctx():
+            with jsonl_path.open("r", encoding="utf-8") as f:
+                for line_num, line in enumerate(f, start=1):
+                    line = line.strip()
+                    if not line:
+                        continue
+
+                    try:
+                        item = json.loads(line)
+                    except json.JSONDecodeError as exc:
+                        raise CommandError(
+                            f"Invalid JSON on line {line_num} of {jsonl_path}: {exc}"
+                        )
+
+                    labels_list = item.get("labels", [])
+                    filename = item.get("filename")
+                    # old_examination_id and old_id are available if you want them later:
+                    old_id = item.get("old_id")
+                    old_exam_id = item.get("old_examination_id")
+
+                    if not filename:
+                        self.stdout.write(
+                            self.style.WARNING(
+                                f"Skipping line {line_num}: no 'filename' key."
+                            )
+                        )
+                        continue
+
+                    image_path = images_root / filename
+                    if not image_path.exists():
+                        self.stdout.write(
+                            self.style.WARNING(
+                                f"Image file does not exist for line {line_num}: {image_path}"
+                            )
+                        )
+                        # Still create Frame so DB + paths are consistent.
+
+                    # --- Create Frame ---
+                    frame_counter += 1
+                    frame = Frame(
+                        video=video,
+                        frame_number=frame_counter,
+                        relative_path=filename,  # filename is relative under images_root
+                        timestamp=None,
+                        old_examination_id=old_exam_id,  # keeping old examination id legacy exam id for grouping
+                        is_extracted=True,
+                    )
+                    if not dry_run:
+                        frame.save()
+
+                    # --- Create annotations for positive labels ---
+                    for label_name in labels_list:
+                        label = self._get_or_create_label_and_attach_to_labelset(
+                            label_name=label_name,
+                            labelset=labelset,
+                        )
+
+                        annotation_counter += 1
+                        annotation = ImageClassificationAnnotation(
+                            frame=frame,
+                            label=label,
+                            value=True,
+                            annotator="legacy_import",
+                        )
+                        if not dry_run:
+                            annotation.save()
+                            if ai_dataset is not None:
+                                # IMPORTANT CHANGE:
+                                # Use the AIDataSet helper, which for dataset_type='image'
+                                # returns the image_annotations manager.
+                                ai_dataset.get_annotations_queryset().add(annotation)
+
+            # --- Summary ---
+            if dry_run:
+                self.stdout.write(
+                    self.style.WARNING(
+                        f"[DRY RUN] Processed {frame_counter} Frames, {annotation_counter} Annotations. "
+                        "No database changes were committed."
+                    )
+                )
+            else:
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"Imported {frame_counter} Frames, {annotation_counter} "
+                        f"ImageClassificationAnnotations into AIDataSet id={ai_dataset.id}."
+                    )
+                )
+
+    # ------------------------------------------------------------------
+    # Helper methods
+    # ------------------------------------------------------------------
+
+    def _get_existing_labelset(
+        self, labelset_name: str, labelset_version: int
+    ) -> LabelSet:
+        try:
+            return LabelSet.objects.get(name=labelset_name, version=labelset_version)
+        except LabelSet.DoesNotExist as exc:
+            raise CommandError(
+                f"LabelSet name='{labelset_name}', version={labelset_version} does not exist. "
+                "Create it first (e.g. via fixtures or admin)."
+            ) from exc
+
+    def _get_or_create_label_and_attach_to_labelset(
+        self, label_name: str, labelset: LabelSet
+    ) -> Label:
+        label, _ = Label.objects.get_or_create(name=label_name)
+        # Attach to this labelset if missing
+        if label not in labelset.labels.all():
+            labelset.labels.add(label)
+        return label
+
+    class _noop_context:
+        """Simple no-op context manager used for dry-run."""
+
+        def __enter__(self):
+            return None
+
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            return False

endoreg_db/management/commands/load_name_data.py

@@ -3,7 +3,7 @@
 import os
 from django.core.management.base import BaseCommand
 from endoreg_db.utils import collect_center_names  # Import your function here
-from pathlib import Path
+
 
 class Command(BaseCommand):
     help = "Generate first_names.yaml and last_names.yaml from center data"
@@ -26,7 +26,6 @@ class Command(BaseCommand):
         pass
 
     def handle(self, *args, **options):
-
         # Run the function with the provided arguments
         try:
             collect_center_names()

endoreg_db/management/commands/load_pdf_type_data.py

@@ -1,7 +1,3 @@
-import os
-
-import yaml
-from django.conf import settings
 from django.core.management.base import BaseCommand
 
 from ...data import REPORT_TYPE_DATA_DIR
@@ -29,15 +25,15 @@ IMPORT_METADATA = {
             "endoscope_info_line",
             "examiner_info_line",
             "cut_off_below_lines",
-            "cut_off_above_lines"
-        ], # e.g. ["intervention_types"]
+            "cut_off_above_lines",
+        ],  # e.g. ["intervention_types"]
         "foreign_key_models": [
             ReportReaderFlag,
             ReportReaderFlag,
             ReportReaderFlag,
             ReportReaderFlag,
-            ReportReaderFlag
-        ] # e.g. [InterventionType]
+            ReportReaderFlag,
+        ],  # e.g. [InterventionType]
     }
 }
 

endoreg_db/management/commands/load_profession_data.py

@@ -1,4 +1,3 @@
-from django.conf import settings
 from django.core.management.base import BaseCommand
 
 from ...data import PROFESSION_DATA_DIR

endoreg_db/management/commands/load_report_reader_flag_data.py

@@ -1,7 +1,3 @@
-import os
-
-import yaml
-from django.conf import settings
 from django.core.management.base import BaseCommand
 
 from ...data import REPORT_READER_FLAG_DATA_DIR

endoreg_db/management/commands/load_requirement_data.py

@@ -62,10 +62,14 @@ def _validate_requirement_configuration(fields: dict, *, entry: dict, model):
             return True
         return any(not item for item in value)
 
-    missing = [key for key in ("requirement_types", "operators") if _values_missing(key)]
+    missing = [
+        key for key in ("requirement_types", "operators") if _values_missing(key)
+    ]
     if missing:
         missing_display = ", ".join(missing)
-        raise ValueError(f"{model.__name__} '{name}' is missing required configuration for: {missing_display}.")
+        raise ValueError(
+            f"{model.__name__} '{name}' is missing required configuration for: {missing_display}."
+        )
 
 
 IMPORT_METADATA = {

endoreg_db/management/commands/load_unit_data.py

@@ -1,7 +1,3 @@
-import os
-
-import yaml
-from django.conf import settings
 from django.core.management.base import BaseCommand
 
 from ...data import UNIT_DATA_DIR

endoreg_db/management/commands/load_user_groups.py

@@ -1,7 +1,5 @@
 from django.core.management.base import BaseCommand
-from django.contrib.auth.models import Group, Permission
-from django.contrib.contenttypes.models import ContentType
-from django.apps import apps
+from django.contrib.auth.models import Group
 
 
 class Command(BaseCommand):
@@ -9,13 +7,13 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
-            '--verbose',
-            action='store_true',
-            help='Display verbose output',
+            "--verbose",
+            action="store_true",
+            help="Display verbose output",
         )
 
     def handle(self, *args, **options):
-        verbose = options['verbose']
+        verbose = options["verbose"]
 
         # Create groups
         groups = ["demo", "verified", "agl", "endo_reg_user", "g_play_user", "ukw_user"]

endoreg_db/management/commands/model_input.py

@@ -0,0 +1,169 @@
+# endoreg_db/management/commands/model_input.py
+
+from __future__ import annotations
+
+from django.core.management.base import BaseCommand, CommandError
+
+from endoreg_db.models import AIDataSet
+from endoreg_db.utils.ai.data_loader_for_model_input import (
+    build_dataset_for_training,
+)
+from endoreg_db.utils.ai.model_training.config import (
+    TrainingConfig,
+)
+from endoreg_db.utils.ai.model_training.trainer_gastronet_multilabel import (
+    train_gastronet_multilabel,
+)
+
+
+class Command(BaseCommand):
+    help = (
+        "Build the dynamic multi-label dataset from AIDataSet and train a "
+        "GastroNet-ResNet50 multi-label model on it.\n"
+        "\n"
+        "This command:\n"
+        "- Uses AIDataSet.id to select annotations.\n"
+        "- Infers the LabelSet from used labels.\n"
+        "- Builds image_paths, label_vectors, and label_masks from DB.\n"
+        "- Prints a short debug dump.\n"
+        "- Trains a model using RN50 GastroNet checkpoint (if provided).\n"
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--dataset-id",
+            type=int,
+            required=True,
+            help="Primary key of the AIDataSet to use for training.",
+        )
+        parser.add_argument(
+            "--backbone-checkpoint",
+            type=str,
+            default=None,
+            help=(
+                "Path to RN50_GastroNet-1M_DINOv1.pth (or similar). "
+                "If omitted, ResNet50 is randomly initialized."
+            ),
+        )
+        parser.add_argument(
+            "--backbone-name",
+            type=str,
+            default="gastro_rn50",
+            help=(
+                "Backbone name, e.g. 'gastro_rn50' (default), "
+                "'resnet50_imagenet', 'resnet50_random', 'efficientnet_b0_imagenet', etc."
+            ),
+        )
+        parser.add_argument(
+            "--epochs",
+            type=int,
+            default=10,
+            help="Number of training epochs.",
+        )
+
+    def handle(self, *args, **options):
+        dataset_id = options["dataset_id"]
+        backbone_ckpt = options["backbone_checkpoint"]
+        backbone_name = options["backbone_name"]
+        num_epochs = options["epochs"]
+
+        try:
+            dataset = AIDataSet.objects.get(id=dataset_id)
+        except AIDataSet.DoesNotExist:
+            raise CommandError(f"AIDataSet with id={dataset_id} does not exist.")
+
+        # Basic info
+        self.stdout.write(
+            self.style.NOTICE(
+                f"Using AIDataSet id={dataset.id}, "
+                f"name={dataset.name!r}, "
+                f"dataset_type={dataset.dataset_type!r}, "
+                f"ai_model_type={dataset.ai_model_type!r}"
+            )
+        )
+
+        data = build_dataset_for_training(dataset)
+
+        image_paths = data["image_paths"]
+        label_vectors = data["label_vectors"]
+        label_masks = data["label_masks"]
+        labels = data["labels"]
+        labelset = data["labelset"]
+
+        self.stdout.write(self.style.NOTICE("Inferred LabelSet for this AIDataSet:"))
+        self.stdout.write(
+            f"  LabelSet id={labelset.id}, "
+            f"name={labelset.name!r}, "
+            f"version={labelset.version}"
+        )
+        self.stdout.write("  Labels (index, id, name):")
+        for idx, lbl in enumerate(labels):
+            self.stdout.write(f"    [{idx}] id={lbl.id}, name={lbl.name!r}")
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"\nBuilt training dataset from AIDataSet id={dataset.id}:\n"
+                f"- #samples: {len(image_paths)}\n"
+                f"- #labels:  {len(labels)}"
+            )
+        )
+
+        MAX_PRINT = 10
+        self.stdout.write(self.style.NOTICE("\nPer-sample debug output (first 10):"))
+        for i, (path, vec, mask) in enumerate(
+            zip(image_paths, label_vectors, label_masks)
+        ):
+            if i >= MAX_PRINT:
+                self.stdout.write(
+                    self.style.WARNING(
+                        f"... ({len(image_paths) - MAX_PRINT} more samples not shown)"
+                    )
+                )
+                break
+
+            self.stdout.write(
+                f"  Sample {i}:"
+                f"\n    path = {path!r}"
+                f"\n    vector (1/0/None) = {vec}"
+                f"\n    mask (1=use, 0=ignore) = {mask}"
+            )
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"\n Input for model training built successfully from AIDataSet id={dataset.id}."
+            )
+        )
+
+        # ------------------------------------------------------------------
+        # Ask user if we should really start training
+        # ------------------------------------------------------------------
+        self.stdout.write("")
+        confirm = (
+            input(
+                "Proceed with model training? "
+                "Type 'yes' and press Enter to continue, anything else to abort: "
+            )
+            .strip()
+            .lower()
+        )
+
+        if confirm != "yes":
+            self.stdout.write(
+                self.style.WARNING("Training aborted by user. No model was trained.")
+            )
+            return
+
+        # ---- Training ----
+        cfg = TrainingConfig(
+            dataset_id=dataset.id,
+            backbone_checkpoint=backbone_ckpt,
+            backbone_name=backbone_name,
+            num_epochs=num_epochs,
+        )
+        result = train_gastronet_multilabel(cfg)
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"\nTraining finished. Model saved to: {result['model_path']}"
+            )
+        )

endoreg_db/management/commands/register_ai_model.py

@@ -7,7 +7,7 @@
 # "labelset": "multilabel_classification", #labelset name, combination of name and version is unique
 # "labelset_version": 0,
 # "weights_path": "weights/multilabel_classification_0.pth", # path to weights file
-#}
+# }
 
 from django.core.management.base import BaseCommand
 from django.core.files import File
@@ -15,29 +15,33 @@ from endoreg_db.models import ModelMeta, ModelType, LabelSet
 import json
 from pathlib import Path
 
+
 class Command(BaseCommand):
     """
     Registers a new AI model in the database.
     """
-    help = 'Registers a new AI model in the database.'
+
+    help = "Registers a new AI model in the database."
 
     def add_arguments(self, parser):
-        parser.add_argument('model_meta_path', type=str)
+        parser.add_argument("model_meta_path", type=str)
 
     def handle(self, *args, **options):
-        model_meta_path = Path(options['model_meta_path'])
+        model_meta_path = Path(options["model_meta_path"])
 
-        with open(model_meta_path, 'r') as f:
+        with open(model_meta_path, "r") as f:
             model_meta = json.load(f)
 
         # get or create model type
-        model_type = ModelType.objects.get(name=model_meta['model_type'])
+        model_type = ModelType.objects.get(name=model_meta["model_type"])
 
         # get or create labelset
-        labelset = LabelSet.objects.get(name=model_meta['labelset'], version=model_meta['labelset_version'])
+        labelset = LabelSet.objects.get(
+            name=model_meta["labelset"], version=model_meta["labelset_version"]
+        )
 
         # Handle weights file
-        weights_path = model_meta['weights_path']
+        weights_path = model_meta["weights_path"]
         # weights path is realative to model_meta_path
         weights_path = model_meta_path.parent / weights_path
 
@@ -45,20 +49,22 @@ class Command(BaseCommand):
 
         # Make sure the path is correct and the file exists
         try:
-            with open(weights_path, 'rb') as file:
+            with open(weights_path, "rb") as file:
                 model_name_string = f"{model_meta['name']}_{model_meta['version']}"
-                weights = File(file, name = model_name_string)
+                weights = File(file, name=model_name_string)
                 # Create ModelMeta instance
                 model_meta_instance = ModelMeta.objects.create(
-                    name=model_meta['name'],
-                    version=model_meta['version'],
+                    name=model_meta["name"],
+                    version=model_meta["version"],
                     type=model_type,
                     labelset=labelset,
                     weights=weights,
-                    description=model_meta.get('description', '')  # Assuming description is optional
+                    description=model_meta.get(
+                        "description", ""
+                    ),  # Assuming description is optional
                 )
                 print(f"Successfully registered model {model_meta_instance}")
         except IOError:
-            print(f"Failed to open weights file at {weights_path}. Make sure the file exists.")
-
-
+            print(
+                f"Failed to open weights file at {weights_path}. Make sure the file exists."
+            )