endoreg-db 0.6.3__py3-none-any.whl → 0.8.1__py3-none-any.whl

endoreg_db/models/ai_model/lightning/prediction_visualizer.py

@@ -1,55 +0,0 @@
-import streamlit as st
-import matplotlib.pyplot as plt
-import json
-from PIL import Image
-import argparse
-
-
-# Load the JSON file
-@st.cache_data
-def load_data(file_path):
-    with open(file_path, 'r') as f:
-        data = json.load(f)
-    return data
-
-# Streamlit App
-def main(file_path):
-    st.title("Visualize Predictions")
-
-    # Load the data
-    data = load_data(file_path)
-
-    # Dropdown to select label
-    selected_label = st.selectbox('Select Label', options=data['labels'])
-
-    # Get the index of the selected label
-    label_index = data['labels'].index(selected_label)
-
-    # Extract predictions for the selected label
-    selected_predictions = [pred[label_index] for pred in data['predictions']]
-
-    # Line plot for the selected label's prediction values
-    st.subheader(f"Line plot for {selected_label}")
-
-    fig, ax = plt.subplots(figsize=(10, 6))  # Explicitly create a figure and axis
-    ax.plot(selected_predictions)
-    ax.set_xlabel('Frame')
-    ax.set_ylabel('Prediction Value')
-    ax.set_title(f'Predictions for {selected_label}')
-
-    st.pyplot(fig)  # Pass the figure object explicitly
-
-
-    # Slider to select frame
-    frame_idx = st.slider('Select a frame', 0, len(data['paths']) - 1)
-
-    # Display the frame
-    image_path = data['paths'][frame_idx]
-    image = Image.open(image_path)
-    st.image(image, caption=f"Frame {frame_idx}", use_column_width=True)
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description="Visualize Predictions")
-    parser.add_argument("--file", type=str, help="Path to JSON file containing predictions", required=True)
-    args = parser.parse_args()
-    main(args.file)

endoreg_db/models/ai_model/lightning/run_visualizer.py

@@ -1,21 +0,0 @@
-# agl_visualization/run_visualizer.py
-
-import subprocess
-import argparse
-
-def main():
-    parser = argparse.ArgumentParser(description="Visualize Predictions")
-    parser.add_argument("--file", type=str, help="Path to JSON file containing predictions", required=True)
-    args = parser.parse_args()
-
-    # Replace this with the path where prediction_visualizer.py resides
-    streamlit_script_path = "agl_predict_endo_frame/prediction_visualizer.py"
-    
-    # Using subprocess to run the Streamlit command
-    subprocess.run(f'streamlit run {streamlit_script_path} -- --file {args.file}', shell=True)
-
-# if __name__ == "__main__":
-    # parser = argparse.ArgumentParser(description="Visualize Predictions")
-    # parser.add_argument("--file", type=str, help="Path to JSON file containing predictions", required=True)
-    # args = parser.parse_args()
-    # main(args.file)

endoreg_db/models/ai_model/model_meta.py

@@ -1,250 +0,0 @@
-"""
-This module defines the ModelMeta and ModelMetaManager classes for managing AI model metadata.
-"""
-
-import os
-from pathlib import Path
-from typing import Optional, TYPE_CHECKING
-import shutil
-
-from django.db import models
-from django.core.validators import FileExtensionValidator
-from icecream import ic
-
-if TYPE_CHECKING:
-    from endoreg_db.models import LabelSet, AiModel  # pylint: disable=import-outside-toplevel
-
-PSEUDO_DIR = Path(os.environ.get("DJANGO_PSEUDO_DIR", Path("./erc_data"))).expanduser()
-
-STORAGE_LOCATION = PSEUDO_DIR
-WEIGHTS_DIR_NAME = "db_model_weights"
-WEIGHTS_DIR = STORAGE_LOCATION / WEIGHTS_DIR_NAME
-
-if not WEIGHTS_DIR.exists():
-    WEIGHTS_DIR.mkdir(parents=True)
-
-
-class ModelMetaManager(models.Manager):
-    """
-    Custom manager for ModelMeta with additional query methods.
-    """
-
-    def get_by_natural_key(self, name: str, version: str) -> "ModelMeta":
-        """Get the model by its natural key"""
-        return self.get(name=name, version=version)
-
-
-class ModelMeta(models.Model):
-    """
-    Attributes:
-        name (str): The name of the model.
-        version (str): The version of the model.
-        type (ModelType): The type of the model.
-        labelset (LabelSet): Associated labels.
-        weights (FileField): Path to the model weights.
-    """
-
-    name = models.CharField(max_length=255)
-    version = models.CharField(max_length=255)
-
-    model = models.ForeignKey(
-        "AiModel", on_delete=models.CASCADE, related_name="models"
-    )
-
-    labelset = models.ForeignKey(
-        "LabelSet", on_delete=models.CASCADE, related_name="models"
-    )
-    activation = models.CharField(max_length=255, default="sigmoid")
-
-    weights = models.FileField(
-        upload_to=WEIGHTS_DIR_NAME,
-        validators=[FileExtensionValidator(allowed_extensions=["ckpt"])],
-        # storage=FileSystemStorage(location=STORAGE_LOCATION.resolve().as_posix()),
-        null=True,
-        blank=True,
-    )
-
-    # Means and stds for normalization
-    mean = models.CharField(max_length=255, default="0.45211223,0.27139644,0.19264949")
-    std = models.CharField(max_length=255, default="0.31418097,0.21088019,0.16059452")
-
-    size_x = models.IntegerField(default=716)
-    size_y = models.IntegerField(default=716)
-    axes = models.CharField(max_length=255, default="2,0,1")
-    batchsize = models.IntegerField(default=16)
-    num_workers = models.IntegerField(default=0)
-
-    description = models.TextField(blank=True, null=True)
-    date_created = models.DateTimeField(auto_now_add=True)
-
-    objects = ModelMetaManager()
-
-    if TYPE_CHECKING:
-        labelset: "LabelSet"
-        ai_model: "AiModel"
-
-    @classmethod
-    def create_from_file(
-        cls,
-        meta_name,
-        model_name,
-        model_meta_version,
-        labelset_name,
-        weights_file,
-        bump_version,
-        **kwargs,
-    ):
-        """Create a new model meta from a file"""
-
-        from endoreg_db.models import LabelSet, AiModel  # pylint: disable=import-outside-toplevel
-
-        ic("Start")
-        model_meta_version = kwargs.get("model_meta_version", None)
-
-        ai_model = AiModel.objects.get(name=model_name)
-
-        # check If ModelMeta with same name and model already exists
-        if not model_meta_version:
-            if cls.objects.filter(name=meta_name, model=ai_model).exists():
-                # get highest version number
-                highest_version = (
-                    cls.objects.filter(name=meta_name, model=ai_model)
-                    .latest("version")
-                    .version
-                )
-                version = int(highest_version) + 1
-            else:
-                version = 1
-        else:
-            version = model_meta_version
-
-        meta_exists = cls.objects.filter(
-            name=meta_name, model=ai_model, version=version
-        ).exists()
-        if meta_exists and not bump_version:
-            raise ValueError(
-                f"ModelMeta with name {meta_name} and version {version} already exists"
-            )
-
-        assert labelset_name is not None, "Labelset name must be provided"
-        labelset = LabelSet.objects.get(name=labelset_name)
-        assert labelset is not None, "Labelset not found"
-
-        weights_path = Path(weights_file)
-        # If not under our WEIGHTS_DIR, copy it there
-        if not str(weights_path).startswith(str(WEIGHTS_DIR)):
-            target_path = WEIGHTS_DIR / weights_path.name
-            if not target_path.exists():
-                shutil.copy(weights_path, target_path)
-            weights_file = target_path
-
-        return cls.objects.create(
-            name=meta_name,
-            model=ai_model,
-            version=version,
-            labelset=labelset,
-            weights=str(weights_file),
-            **kwargs,
-        )
-
-    @classmethod
-    def get_latest_version(cls, name) -> int:
-        """
-        get the model meta with the highest version. Assumes Model Name and Meta Name are the same
-        """
-        from endoreg_db.models import AiModel  # pylint: disable=import-outside-toplevel
-
-        ai_model = AiModel.objects.get(name=name)
-        ic(f"Model: {ai_model}")
-        assert ai_model is not None, "Model not found"
-
-        model_meta = cls.objects.filter(name=name, model=ai_model).latest("version")
-        ic(f"Latest ModelMeta: {model_meta}")
-
-        latest_version = model_meta.version
-        ic(f"Latest Version: {latest_version}")
-
-        return int(latest_version)
-
-    @classmethod
-    def get_activation_function(cls, activation: str):
-        """get the activation function"""
-        from torch import nn  # pylint: disable=import-outside-toplevel
-
-        lookup = {
-            "sigmoid": nn.Sigmoid(),
-            "softmax": nn.Softmax(),
-            "none": nn.Identity(),
-        }
-
-        return lookup[activation]
-
-    def get_inference_dataset_config(self):
-        """
-        Creates a dictionary with the configuration for the inference dataset.
-
-        Example:
-            sample_config = {
-                # mean and std for normalization
-                "mean": (0.45211223, 0.27139644, 0.19264949),
-                "std": (0.31418097, 0.21088019, 0.16059452),
-                # Image Size
-                "size_x": 716,
-                "size_y": 716,
-                # how to wrangle axes of the image before putting them in the network
-                "axes": [2,0,1],  # 2,1,0 for opencv
-                "batchsize": 16,
-                "num_workers": 0, # always 1 for Windows systems # FIXME: fix celery crash if multiprocessing
-                # maybe add sigmoid after prediction?
-                "activation": nn.Sigmoid(),
-                "labels": ['appendix',  'blood',  'diverticule',  'grasper',  'ileocaecalvalve',  'ileum',  'low_quality',  'nbi',  'needle',  'outside',  'polyp',  'snare',  'water_jet',  'wound']
-            }
-        """
-        mean = list(map(float, str(self.mean).split(",")))
-        std = list(map(float, str(self.std).split(",")))
-        axes = list(map(int, str(self.axes).split(",")))
-
-        config = {
-            "mean": mean,
-            "std": std,
-            "size_x": self.size_x,
-            "size_y": self.size_y,
-            "axes": axes,
-            "batchsize": self.batchsize,
-            "num_workers": self.num_workers,
-            "activation": self.get_activation_function(self.activation),
-            "labels": self.labelset.get_labels_in_order(),  # pylint: disable=no-member
-        }
-
-        ic(f"ModelMeta - get_inference_dataset_config: {config}")
-
-        return config
-
-    def natural_key(self):
-        """Return the natural key for this model"""
-        return (self.name, self.version)
-
-    def __str__(self):
-        return f"{self.name} (v: {self.version})"
-
-    def get_config_dict(self) -> dict:
-        """Return the configuration as a dictionary"""
-        conf_dict = {
-            "mean": list(map(float, str(self.mean).split(","))),
-            "std": list(map(float, str(self.std).split(","))),
-            "size_x": self.size_x,
-            "size_y": self.size_y,
-            "axes": list(map(int, str(self.axes).split(","))),
-            "batchsize": self.batchsize,
-            "num_workers": self.num_workers,
-            "activation": self.activation,
-            "labels": self.labelset.get_labels_in_order(),  # pylint: disable=no-member
-        }
-        return conf_dict
-
-    @classmethod
-    def get_by_name(cls, name: str, version: Optional[int]) -> "ModelMeta":
-        """Get the model by its name and version"""
-        if version is None:
-            version = cls.get_latest_version(name)
-        return cls.objects.get(name=name, version=version)

endoreg_db/models/annotation/__init__.py

@@ -1,32 +0,0 @@
-"""Module for annotation models"""
-
-from .image_classification import ImageClassificationAnnotation
-from .binary_classification_annotation_task import (
-    BinaryClassificationAnnotationTask,
-)
-from .anonymized_image_annotation import (
-    AnonymousImageAnnotation,
-    DroppedName,
-    AnonymizedImageLabel,
-    AnonymizedFile,
-    UploadedFile,
-)
-from .video_segmentation_annotation import (
-    VideoSegmentationLabel,
-    VideoSegmentationAnnotation,
-)
-
-from .video_segmentation_labelset import VideoSegmentationLabelSet
-
-__all__ = [
-    "ImageClassificationAnnotation",
-    "BinaryClassificationAnnotationTask",
-    "AnonymousImageAnnotation",
-    "DroppedName",
-    "AnonymizedImageLabel",
-    "AnonymizedFile",
-    "UploadedFile",
-    "VideoSegmentationLabel",
-    "VideoSegmentationLabelSet",
-    "VideoSegmentationAnnotation",
-]

endoreg_db/models/annotation/anonymized_image_annotation.py

@@ -1,115 +0,0 @@
-from django.db import models
-from django.utils.text import slugify
-from django.utils import timezone
-
-class AnonymizedImageLabel(models.Model):
-    """
-    Represents a label for anonymized image annotations.
-
-    Attributes:
-        name (str): Unique name of the label.
-        description (str): Optional description for the label.
-    """
-    name = models.CharField(max_length=255, unique=True)
-    description = models.TextField(blank=True, null=True)
-
-    def __str__(self) -> str:
-        return self.name
-
-
-class AnonymousImageAnnotation(models.Model):
-    """
-    Represents an annotation for anonymized images.
-
-    Attributes:
-        label (ForeignKey): Associated label for the image.
-        image_name (str): Name of the image file.
-        original_image_url (str): URL of the original image.
-        slug (str): Unique identifier for the annotation.
-        polyp_count (int): Number of polyps identified in the image.
-        comments (str): Additional comments about the annotation.
-        gender (str): Gender associated with the annotation (if applicable).
-        name_image_url (str): URL of the name image.
-        date_created (datetime): Date when the annotation was created.
-        processed (bool): Indicates if the annotation has been processed.
-    """
-    label = models.ForeignKey(AnonymizedImageLabel, on_delete=models.CASCADE)
-    image_name = models.CharField(max_length=255)
-    original_image_url = models.CharField(max_length=255, default='https://example.com/placeholder.jpg')
-    slug = models.SlugField(unique=True, blank=True, null=True)
-    polyp_count = models.IntegerField()
-    comments = models.TextField(blank=True, null=True)
-    gender = models.CharField(max_length=255)
-    name_image_url = models.CharField(max_length=255)
-    date_created = models.DateTimeField(auto_now_add=True)
-    processed = models.BooleanField(default=False)
-
-    def save(self, *args, **kwargs):
-        """
-        Overrides the save method to generate a slug if not already set.
-        """
-        if not self.slug:
-            self.slug = slugify(f"{self.label}-{self.image_name}")
-        super().save(*args, **kwargs)
-
-    def __str__(self) -> str:
-        return f"{self.image_name} - {self.label}"
-
-
-class DroppedName(models.Model):
-    """
-    Represents a dropped name in the annotation process.
-
-    Attributes:
-        annotation (ForeignKey): Associated annotation.
-        name (str): Dropped name.
-        gender (str): Gender associated with the name.
-        x (float): X-coordinate of the name.
-        y (float): Y-coordinate of the name.
-        name_image_url (str): URL of the name image.
-        box_coordinates (str): Optional coordinates of the bounding box.
-    """
-    annotation = models.ForeignKey(AnonymousImageAnnotation, related_name='dropped_names', on_delete=models.CASCADE)
-    name = models.CharField(max_length=255)
-    gender = models.CharField(max_length=255)
-    x = models.FloatField()
-    y = models.FloatField()
-    name_image_url = models.CharField(max_length=255)
-    box_coordinates = models.CharField(max_length=255, blank=True, null=True)
-
-    def __str__(self) -> str:
-        return f"{self.name} ({self.gender}) at ({self.x}, {self.y})"
-
-
-class UploadedFile(models.Model):
-    """
-    Represents a file uploaded to the system.
-
-    Attributes:
-        original_file (FileField): The uploaded original file.
-        upload_date (datetime): Date when the file was uploaded.
-        description (str): Optional description of the file.
-    """
-    original_file = models.FileField(upload_to='uploads/original/')
-    upload_date = models.DateTimeField(default=timezone.now)
-    description = models.TextField(blank=True, null=True)
-
-    def __str__(self) -> str:
-        return self.original_file.name
-
-
-class AnonymizedFile(models.Model):
-    """
-    Represents an anonymized version of an uploaded file.
-
-    Attributes:
-        original_file (OneToOneField): The original uploaded file.
-        anonymized_file (FileField): The anonymized file.
-        anonymization_date (datetime): Date when the file was anonymized.
-    """
-    original_file = models.OneToOneField(UploadedFile, on_delete=models.CASCADE, related_name='anonymized_file')
-    anonymized_file = models.FileField(upload_to='uploads/anonymized/')
-    anonymization_date = models.DateTimeField(default=timezone.now)
-
-    def __str__(self) -> str:
-        return f"Anonymized version of {self.original_file.original_file.name}"

endoreg_db/models/annotation/binary_classification_annotation_task.py

@@ -1,117 +0,0 @@
-from django.db import models
-# from rest_framework import serializers
-# from ..label import Label
-# from .image_classification import ImageClassificationAnnotation
-
-ANNOTATION_PER_S_THRESHOLD = 2
-
-# TODO Migrate
-# def clear_finished_legacy_tasks():
-#     """
-#     Deletes all finished LegacyBinaryClassificationAnnotationTask entries.
-#     """
-#     tasks = LegacyBinaryClassificationAnnotationTask.objects.filter(is_finished=True)
-#     tasks.delete()
-
-# def get_legacy_binary_classification_annotation_tasks_by_label(label:Label, n:int=100, legacy=False):
-#     clear_finished_legacy_tasks()
-#     """
-#     Retrieves legacy binary classification annotation tasks for a specific label.
-
-#     Args:
-#         label (Label): The label to filter tasks by.
-#         n (int): Maximum number of tasks to retrieve. Defaults to 100.
-#         legacy (bool): If True, includes legacy tasks. Defaults to False.
-#     """
-#     if legacy:
-#         # fetch all LegacyLabelVideoSegments with the given label
-#         _segments = LegacyLabelVideoSegment.objects.filter(label=label)
-#         frames_for_tasks = []
-
-#         for segment in _segments:
-#             # check if the segment has already been annotated
-#             annotations = list(ImageClassificationAnnotation.objects.filter(legacy_frame__in=segment.get_frames(), label=label))
-#             segment_len_in_s = segment.get_segment_len_in_s()
-
-#             target_annotation_number = segment_len_in_s * ANNOTATION_PER_S_THRESHOLD
-
-#             if len(annotations) < target_annotation_number:
-#                 get_frame_number = int(target_annotation_number - len(annotations))
-#                 frames = segment.get_frames_without_annotation(get_frame_number)
-#                 frames_for_tasks.extend(frames)
-
-#             if len(frames_for_tasks) >= n:
-#                 break
-
-#         # create tasks
-#         tasks = []
-#         for frame in frames_for_tasks:
-
-#             # get_or_create task
-#             task, created = LegacyBinaryClassificationAnnotationTask.objects.get_or_create(
-#                 label=label,
-#                 image_path=frame.image.path,
-#                 frame_id=frame.pk,
-#             )
-
-
-class AbstractBinaryClassificationAnnotationTask(models.Model):
-    """
-    Abstract base class for binary classification annotation tasks.
-
-    Attributes:
-        label (ForeignKey): The associated label.
-        is_finished (bool): Indicates if the task is completed.
-        date_created (datetime): The creation date of the task.
-        date_finished (datetime): The completion date of the task.
-        image_path (str): Path to the associated image.
-        image_type (str): The type of the image (e.g., "frame" or "legacy").
-        frame_id (int): Identifier of the associated frame.
-        labelstudio_project_id (int): The Label Studio project ID.
-        labelstudio_task_id (int): The Label Studio task ID.
-    """
-
-    label = models.ForeignKey("Label", on_delete=models.CASCADE)
-    is_finished = models.BooleanField(default=False)
-    date_created = models.DateTimeField(auto_now_add=True)
-    date_finished = models.DateTimeField(blank=True, null=True)
-    image_path = models.CharField(max_length=255, blank=True, null=True)
-    image_type = models.CharField(max_length=255, blank=True, null=True)
-    frame_id = models.IntegerField(blank=True, null=True)
-    labelstudio_project_id = models.IntegerField(blank=True, null=True)
-    labelstudio_task_id = models.IntegerField(blank=True, null=True)
-
-    class Meta:
-        abstract = True
-
-
-class BinaryClassificationAnnotationTask(AbstractBinaryClassificationAnnotationTask):
-    """
-    Represents a binary classification task for a frame.
-
-    Attributes:
-       frame (ForeignKey): The associated frame for this task.
-    """
-
-    frame = models.ForeignKey(
-        "Frame",
-        on_delete=models.CASCADE,
-        related_name="binary_classification_annotation_tasks",
-    )
-    image_type = models.CharField(
-        max_length=255, default="frame"
-    )  # Default image type for non-legacy tasks
-
-    def get_frame(self):
-        """
-        Retrieves the frame associated with this task.
-        """
-        from endoreg_db.models.data_file.frame import Frame
-
-        frame = self.frame
-        if not frame:
-            return None
-        else:
-            assert isinstance(frame, Frame)
-
-        return frame

endoreg_db/models/annotation/image_classification.py

@@ -1,86 +0,0 @@
-from django.db import models
-
-from typing import TYPE_CHECKING, Union
-
-if TYPE_CHECKING:
-    from endoreg_db.models import Frame, RawFrame, Label
-
-
-class ImageClassificationAnnotation(models.Model):
-    """
-    Represents an image classification annotation.
-
-    Attributes:
-        frame (ForeignKey): The frame associated with the annotation.
-        legacy_frame (ForeignKey): The legacy frame associated with the annotation.
-        legacy_image (ForeignKey): The legacy image associated with the annotation.
-        label (ForeignKey): The label assigned to the annotation.
-        value (bool): Indicates if the classification is valid.
-        annotator (str): The user who created the annotation.
-        date_created (datetime): The creation date of the annotation.
-        date_modified (datetime): The last modification date of the annotation.
-    """
-
-    # Foreign keys to Frame, LegacyFrame, and LegacyImage (only one of these should be set)
-    frame = models.ForeignKey(
-        "Frame",
-        on_delete=models.CASCADE,
-        blank=True,
-        null=True,
-        related_name="image_classification_annotations",
-    )
-
-    raw_frame = models.ForeignKey(
-        "RawFrame",
-        on_delete=models.CASCADE,
-        blank=True,
-        null=True,
-        related_name="image_classification_annotations",
-    )
-
-    label = models.ForeignKey(
-        "Label",
-        on_delete=models.CASCADE,
-        related_name="image_classification_annotations",
-    )
-    value = models.BooleanField()
-    float_value = models.FloatField(blank=True, null=True)
-    annotator = models.CharField(max_length=255, blank=True, null=True)
-    model_meta = models.ForeignKey(
-        "ModelMeta",
-        on_delete=models.SET_NULL,
-        related_name="image_classification_annotations",
-        default=None,
-        null=True,
-        blank=True,
-    )
-    date_created = models.DateTimeField(auto_now_add=True)
-    date_modified = models.DateTimeField(auto_now=True)
-    information_source = models.ForeignKey(
-        "InformationSource",
-        on_delete=models.SET_NULL,
-        related_name="image_classification_annotations",
-        default=None,
-        null=True,
-        blank=True,
-    )
-
-    if TYPE_CHECKING:
-        frame: "Frame"
-        raw_frame: "RawFrame"
-        label: "Label"
-
-    def __str__(self) -> str:
-        """
-        String representation of the annotation.
-        """
-        return f"{self.label.name} - {self.value}"
-
-    def get_frame(self):
-        """
-        Get the frame associated with the annotation.
-        """
-        if self.frame:
-            return self.frame
-        else:
-            return self.raw_frame