endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

endoreg_db/management/commands/summarize_db_content.py

@@ -5,33 +5,41 @@ from django.utils.timezone import is_aware, make_naive
 import datetime
 import os
 from openpyxl import Workbook
-from django.conf import settings
 import csv
 
+
 class Command(BaseCommand):
-    help = 'Generates a structured report summarizing the database content of custom endoreg_db models and saves it to Excel and CSV files, excluding models with zero records from the files.' # Updated help text
+    help = "Generates a structured report summarizing the database content of custom endoreg_db models and saves it to Excel and CSV files, excluding models with zero records from the files."  # Updated help text
 
     def handle(self, *args, **options):
         """
         Generates a summary report of all models in the 'endoreg_db' app and exports the results to Excel and CSV files.
-        
+
         For each model, counts the total number of records and includes only models with at least one record in the output files. For models with records, attempts to display the range of values for common date or datetime fields and provides value counts for up to three categorical or ForeignKey fields, using heuristics based on field type and name. Handles missing app configuration, file saving errors, and aggregation exceptions gracefully, outputting progress and warnings to the console. Creates a 'data' directory within the app if it does not exist and saves the reports there.
         """
-        self.stdout.write(self.style.SUCCESS("Starting database content summarization for endoreg_db models..."))
+        self.stdout.write(
+            self.style.SUCCESS(
+                "Starting database content summarization for endoreg_db models..."
+            )
+        )
 
         try:
-            app_config = apps.get_app_config('endoreg_db')
+            app_config = apps.get_app_config("endoreg_db")
         except LookupError:
-            self.stdout.write(self.style.ERROR("Could not find the 'endoreg_db' app. Make sure it's correctly installed and configured."))
+            self.stdout.write(
+                self.style.ERROR(
+                    "Could not find the 'endoreg_db' app. Make sure it's correctly installed and configured."
+                )
+            )
             return
 
-        data_dir = os.path.join(app_config.path, 'data')
+        data_dir = os.path.join(app_config.path, "data")
         if not os.path.exists(data_dir):
             os.makedirs(data_dir)
             self.stdout.write(self.style.SUCCESS(f"Created directory: {data_dir}"))
-        
-        excel_file_path = os.path.join(data_dir, 'db_summary.xlsx')
-        csv_file_path = os.path.join(data_dir, 'db_summary.csv')
+
+        excel_file_path = os.path.join(data_dir, "db_summary.xlsx")
+        csv_file_path = os.path.join(data_dir, "db_summary.csv")
 
         # --- Excel Setup ---
         wb = Workbook()
@@ -42,20 +50,32 @@ class Command(BaseCommand):
         # --- End Excel Setup ---
 
         # --- CSV Setup ---
-        csv_data_for_file = [excel_headers] # Initialize with headers for CSV, will only store rows with count > 0
+        csv_data_for_file = [
+            excel_headers
+        ]  # Initialize with headers for CSV, will only store rows with count > 0
         # --- End CSV Setup ---
-            
+
         models = app_config.get_models()
 
         if not models:
-            self.stdout.write(self.style.WARNING("No models found in the 'endoreg_db' app."))
+            self.stdout.write(
+                self.style.WARNING("No models found in the 'endoreg_db' app.")
+            )
             try:
-                wb.save(excel_file_path) # wb will only have headers
-                self.stdout.write(self.style.SUCCESS(f"Empty summary report saved to {excel_file_path}"))
-                with open(csv_file_path, 'w', newline='') as f:
+                wb.save(excel_file_path)  # wb will only have headers
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"Empty summary report saved to {excel_file_path}"
+                    )
+                )
+                with open(csv_file_path, "w", newline="") as f:
                     writer = csv.writer(f)
-                    writer.writerows(csv_data_for_file) # csv_data_for_file will only contain headers
-                self.stdout.write(self.style.SUCCESS(f"Empty summary report saved to {csv_file_path}"))
+                    writer.writerows(
+                        csv_data_for_file
+                    )  # csv_data_for_file will only contain headers
+                self.stdout.write(
+                    self.style.SUCCESS(f"Empty summary report saved to {csv_file_path}")
+                )
             except Exception as e:
                 self.stdout.write(self.style.ERROR(f"Error saving files: {e}"))
             return
@@ -69,8 +89,12 @@ class Command(BaseCommand):
                 self.stdout.write(f"  Total records: {count}")
 
                 if count == 0:
-                    self.stdout.write(self.style.NOTICE("  No records found for this model. Skipping from file output."))
-                    continue # Skip further processing for this model if count is 0 for file output
+                    self.stdout.write(
+                        self.style.NOTICE(
+                            "  No records found for this model. Skipping from file output."
+                        )
+                    )
+                    continue  # Skip further processing for this model if count is 0 for file output
 
                 # --- Add to Excel (only if count > 0) ---
                 ws.append([model_name, count])
@@ -79,36 +103,63 @@ class Command(BaseCommand):
                 # --- Add to CSV data list (only if count > 0) ---
                 csv_data_for_file.append([model_name, count])
                 # --- End Add to CSV data list ---
-                
+
                 # Attempt to get date ranges for common date/datetime fields
-                date_fields_to_check = ['created_at', 'updated_at', 'timestamp', 'date',
-                                        'start_time', 'end_time', 'examination_date',
-                                        'birth_date', 'record_date']
+                date_fields_to_check = [
+                    "created_at",
+                    "updated_at",
+                    "timestamp",
+                    "date",
+                    "start_time",
+                    "end_time",
+                    "examination_date",
+                    "birth_date",
+                    "record_date",
+                ]
                 processed_date_field = False
                 for field_obj in model._meta.get_fields():
-                    if field_obj.name in date_fields_to_check and isinstance(field_obj, (fields.DateField, fields.DateTimeField)):
+                    if field_obj.name in date_fields_to_check and isinstance(
+                        field_obj, (fields.DateField, fields.DateTimeField)
+                    ):
                         try:
-                            aggregation = model.objects.aggregate(min_date=Min(field_obj.name), max_date=Max(field_obj.name))
-                            min_val = aggregation.get('min_date')
-                            max_val = aggregation.get('max_date')
+                            aggregation = model.objects.aggregate(
+                                min_date=Min(field_obj.name),
+                                max_date=Max(field_obj.name),
+                            )
+                            min_val = aggregation.get("min_date")
+                            max_val = aggregation.get("max_date")
 
                             if min_val is not None and max_val is not None:
-                                if isinstance(min_val, datetime.datetime) and is_aware(min_val):
-                                    min_val = make_naive(min_val).strftime('%Y-%m-%d %H:%M:%S')
+                                if isinstance(min_val, datetime.datetime) and is_aware(
+                                    min_val
+                                ):
+                                    min_val = make_naive(min_val).strftime(
+                                        "%Y-%m-%d %H:%M:%S"
+                                    )
                                 elif isinstance(min_val, datetime.date):
-                                    min_val = min_val.strftime('%Y-%m-%d')
-                                
-                                if isinstance(max_val, datetime.datetime) and is_aware(max_val):
-                                    max_val = make_naive(max_val).strftime('%Y-%m-%d %H:%M:%S')
+                                    min_val = min_val.strftime("%Y-%m-%d")
+
+                                if isinstance(max_val, datetime.datetime) and is_aware(
+                                    max_val
+                                ):
+                                    max_val = make_naive(max_val).strftime(
+                                        "%Y-%m-%d %H:%M:%S"
+                                    )
                                 elif isinstance(max_val, datetime.date):
-                                    max_val = max_val.strftime('%Y-%m-%d')
+                                    max_val = max_val.strftime("%Y-%m-%d")
 
-                                self.stdout.write(f"  {field_obj.verbose_name.capitalize()} range: {min_val} to {max_val}")
+                                self.stdout.write(
+                                    f"  {field_obj.verbose_name.capitalize()} range: {min_val} to {max_val}"
+                                )
                                 processed_date_field = True
-                                break # Process only the first found relevant date field for brevity
+                                break  # Process only the first found relevant date field for brevity
                         except Exception as e:
-                            self.stdout.write(self.style.WARNING(f"  Could not aggregate date range for '{field_obj.name}': {e}"))
-                
+                            self.stdout.write(
+                                self.style.WARNING(
+                                    f"  Could not aggregate date range for '{field_obj.name}': {e}"
+                                )
+                            )
+
                 # Attempt to get value counts for potential categorical/ForeignKey fields
                 categorical_fields_found = 0
                 MAX_CATEGORICAL_FIELDS_TO_ANALYZE = 3
@@ -121,69 +172,109 @@ class Command(BaseCommand):
                     is_potential_categorical = False
                     field_name_lower = field.name.lower()
 
-                    if field.many_to_one or field.one_to_one: # ForeignKey or OneToOneField
+                    if (
+                        field.many_to_one or field.one_to_one
+                    ):  # ForeignKey or OneToOneField
                         if field.related_model:
                             is_potential_categorical = True
-                    elif isinstance(field, (fields.CharField, fields.IntegerField, fields.BooleanField, fields.TextField)):
+                    elif isinstance(
+                        field,
+                        (
+                            fields.CharField,
+                            fields.IntegerField,
+                            fields.BooleanField,
+                            fields.TextField,
+                        ),
+                    ):
                         # Heuristic for common categorical fields by name or type
-                        if 'type' in field_name_lower or \
-                           'status' in field_name_lower or \
-                           'gender' in field_name_lower or \
-                           'category' in field_name_lower or \
-                           isinstance(field, fields.BooleanField):
-                           is_potential_categorical = True
+                        if (
+                            "type" in field_name_lower
+                            or "status" in field_name_lower
+                            or "gender" in field_name_lower
+                            or "category" in field_name_lower
+                            or isinstance(field, fields.BooleanField)
+                        ):
+                            is_potential_categorical = True
                         # For CharField/TextField, also consider if it has few distinct values (expensive to check upfront for all)
                         # We'll rely on the naming heuristic primarily or if it's a boolean.
 
                     if is_potential_categorical:
                         try:
-                            self.stdout.write(f"  Value counts for '{field.verbose_name.capitalize() if hasattr(field, 'verbose_name') else field.name}':")
-                            
+                            self.stdout.write(
+                                f"  Value counts for '{field.verbose_name.capitalize() if hasattr(field, 'verbose_name') else field.name}':"
+                            )
+
                             # QuerySet for distinct values and their counts
-                            values_qs = model.objects.values(field.name).annotate(count=Count(field.name)).order_by('-count')
-                            
+                            values_qs = (
+                                model.objects.values(field.name)
+                                .annotate(count=Count(field.name))
+                                .order_by("-count")
+                            )
+
                             distinct_values_count = values_qs.count()
 
                             if distinct_values_count == 0:
-                                self.stdout.write(f"    No distinct values found or field is often NULL.")
+                                self.stdout.write(
+                                    "    No distinct values found or field is often NULL."
+                                )
                                 continue
 
                             for i, item in enumerate(values_qs):
                                 if i >= MAX_DISTINCT_VALUES_TO_SHOW:
-                                    self.stdout.write(f"    ... and {distinct_values_count - MAX_DISTINCT_VALUES_TO_SHOW} more distinct values.")
+                                    self.stdout.write(
+                                        f"    ... and {distinct_values_count - MAX_DISTINCT_VALUES_TO_SHOW} more distinct values."
+                                    )
                                     break
                                 val = item[field.name]
-                                val_display = str(val) if val is not None else "None/NULL"
+                                val_display = (
+                                    str(val) if val is not None else "None/NULL"
+                                )
                                 # Truncate long string values for display
                                 if isinstance(val, str) and len(val_display) > 50:
                                     val_display = val_display[:47] + "..."
-                                self.stdout.write(f"    - {val_display}: {item['count']}")
-                            
+                                self.stdout.write(
+                                    f"    - {val_display}: {item['count']}"
+                                )
+
                             categorical_fields_found += 1
 
                         except Exception as e:
                             # Some fields (like generic relations or complex custom fields) might not support this directly
-                            self.stdout.write(self.style.WARNING(f"  Could not get value counts for '{field.name}': {e}"))
-            
+                            self.stdout.write(
+                                self.style.WARNING(
+                                    f"  Could not get value counts for '{field.name}': {e}"
+                                )
+                            )
+
             except Exception as e:
-                self.stdout.write(self.style.ERROR(f"  Error processing model {model_name}: {e}"))
+                self.stdout.write(
+                    self.style.ERROR(f"  Error processing model {model_name}: {e}")
+                )
 
         # --- Save Excel File ---
         try:
             wb.save(excel_file_path)
-            self.stdout.write(self.style.SUCCESS(f"\nDatabase summary report saved to {excel_file_path}"))
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"\nDatabase summary report saved to {excel_file_path}"
+                )
+            )
         except Exception as e:
             self.stdout.write(self.style.ERROR(f"\nError saving Excel file: {e}"))
         # --- End Save Excel File ---
 
         # --- Save CSV File ---
         try:
-            with open(csv_file_path, 'w', newline='') as f:
+            with open(csv_file_path, "w", newline="") as f:
                 writer = csv.writer(f)
                 writer.writerows(csv_data_for_file)
-            self.stdout.write(self.style.SUCCESS(f"Database summary report saved to {csv_file_path}"))
+            self.stdout.write(
+                self.style.SUCCESS(f"Database summary report saved to {csv_file_path}")
+            )
         except Exception as e:
             self.stdout.write(self.style.ERROR(f"Error saving CSV file: {e}"))
         # --- End Save CSV File ---
 
-        self.stdout.write(self.style.SUCCESS("\nDatabase content summarization finished."))
+        self.stdout.write(
+            self.style.SUCCESS("\nDatabase content summarization finished.")
+        )

endoreg_db/management/commands/train_image_multilabel_model.py

@@ -0,0 +1,144 @@
+# endoreg_db/management/commands/train_image_multilabel_model.py
+
+from __future__ import annotations
+
+from django.core.management.base import BaseCommand, CommandError
+
+from endoreg_db.models import AIDataSet
+from endoreg_db.utils.ai.data_loader_for_model_training import (
+    build_dataset_for_training,
+)
+
+
+class Command(BaseCommand):
+    help = (
+        "Train / fine-tune the image multi-label model on a given AIDataSet.\n"
+        "\n"
+        "This command is fully dynamic:\n"
+        "- It uses the AIDataSet row (id) to decide which annotations to use.\n"
+        "- It *infers* the LabelSet from the labels in those annotations.\n"
+        "- It builds image_paths, label_vectors, and label_masks for training.\n"
+        "- It prints detailed debug information about the chosen LabelSet and samples.\n"
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--dataset-id",
+            type=int,
+            required=True,
+            help="Primary key of the AIDataSet to use for training.",
+        )
+
+    def handle(self, *args, **options):
+        dataset_id = options["dataset_id"]
+
+        try:
+            dataset = AIDataSet.objects.get(id=dataset_id)
+        except AIDataSet.DoesNotExist:
+            raise CommandError(f"AIDataSet with id={dataset_id} does not exist.")
+
+        # ------------------------------------------------------------------
+        # Basic dataset info (AIDataSet row)
+        # ------------------------------------------------------------------
+        self.stdout.write(
+            self.style.NOTICE(
+                f"Using AIDataSet id={dataset.id}, "
+                f"name={dataset.name!r}, "
+                f"dataset_type={dataset.dataset_type!r}, "
+                f"ai_model_type={dataset.ai_model_type!r}"
+            )
+        )
+
+        # Build in-memory dataset (completely dynamic, labelset inferred automatically)
+        data = build_dataset_for_training(dataset)
+
+        image_paths = data["image_paths"]
+        label_vectors = data["label_vectors"]
+        label_masks = data["label_masks"]
+        labels = data["labels"]
+        labelset = data["labelset"]
+
+        # Optional: additional meta info, if present
+        frame_ids = data.get("frame_ids", [])
+        old_examination_ids = data.get("old_examination_ids", [])
+
+        # ------------------------------------------------------------------
+        # Debug: show which LabelSet was picked and its labels
+        # ------------------------------------------------------------------
+        self.stdout.write(self.style.NOTICE("Inferred LabelSet for this AIDataSet:"))
+        self.stdout.write(
+            f"  LabelSet id={labelset.id}, "
+            f"name={labelset.name!r}, "
+            f"version={labelset.version}"
+        )
+        self.stdout.write("  Labels (index, id, name):")
+        for idx, lbl in enumerate(labels):
+            self.stdout.write(f"    [{idx}] id={lbl.id}, name={lbl.name!r}")
+
+        # ------------------------------------------------------------------
+        # Summary of constructed dataset
+        # ------------------------------------------------------------------
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"\nBuilt training dataset from AIDataSet id={dataset.id}:\n"
+                f"- #samples: {len(image_paths)}\n"
+                f"- #labels:  {len(labels)}"
+            )
+        )
+
+        # ------------------------------------------------------------------
+        # Debug: print each frame's path, label vector and mask
+        # NOTE: If your dataset is very large, this will spam the console.
+        #       For now you want full transparency, so we print all.
+        # ------------------------------------------------------------------
+        MAX_PRINT = 50
+        self.stdout.write(self.style.NOTICE("\nPer-sample debug output:"))
+        for i, (path, vec, mask) in enumerate(
+            zip(image_paths, label_vectors, label_masks)
+        ):
+            if i >= MAX_PRINT:
+                self.stdout.write(
+                    self.style.WARNING(
+                        f"... ({len(image_paths) - MAX_PRINT} more samples not shown)"
+                    )
+                )
+                break
+
+            frame_id = frame_ids[i] if i < len(frame_ids) else None
+            old_exam = old_examination_ids[i] if i < len(old_examination_ids) else None
+
+            self.stdout.write(
+                f"  Sample {i}:"
+                f"\n    path = {path!r}"
+                f"\n    frame_id = {frame_id}"
+                f"\n    old_examination_id = {old_exam}"
+                f"\n    vector (1/0/None) = {vec}"
+                f"\n    mask (1=use, 0=ignore) = {mask}"
+            )
+
+        # ------------------------------------------------------------------
+        # TODO: Insert your actual training loop here
+        # ------------------------------------------------------------------
+        # Example (pseudo-code):
+        #
+        #   import torch
+        #   from torch.utils.data import Dataset, DataLoader
+        #
+        #   class EndoDataset(Dataset):
+        #       def __init__(self, image_paths, label_vectors, label_masks):
+        #           ...
+        #
+        #   ds = EndoDataset(image_paths, label_vectors, label_masks)
+        #   loader = DataLoader(ds, batch_size=32, shuffle=True)
+        #
+        #   for batch in loader:
+        #       # forward, compute loss using mask, backward, step, ...
+        #
+        # ------------------------------------------------------------------
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                "\nDataset construction finished. "
+                "You can now plug this into your model training code."
+            )
+        )