endoreg-db 0.6.4__py3-none-any.whl → 0.8.2__py3-none-any.whl

endoreg_db/models/ai_model/lightning/predict.py

@@ -1,172 +0,0 @@
-from .inference_dataset import InferenceDataset
-from torch.utils.data import DataLoader
-import torch
-from torch import nn
-import json
-from .postprocess import (
-    concat_pred_dicts,
-    make_smooth_preds,
-    find_true_pred_sequences
-)
-import numpy as np
-
-sample_config = {
-    # mean and std for normalization
-    "mean": (0.45211223, 0.27139644, 0.19264949),
-    "std": (0.31418097, 0.21088019, 0.16059452),
-    # Image Size
-    "size_x": 716,
-    "size_y": 716,
-    # how to wrangle axes of the image before putting them in the network
-    "axes": [2,0,1],  # 2,1,0 for opencv
-    "batchsize": 16,
-    "num_workers": 0, # always 1 for Windows systems # FIXME: fix celery crash if multiprocessing
-    # maybe add sigmoid after prediction?
-    "activation": nn.Sigmoid(),
-    "labels": [
-        'appendix',  'blood',  'diverticule',  'grasper',  'ileocaecalvalve',  'ileum',  'low_quality',  'nbi',  'needle',  'outside',  'polyp',  'snare',  'water_jet',  'wound'
-    ]
-}
-
-class Classifier():
-    def __init__(self, model=None, config=sample_config, verbose = False):
-        self.config = config
-        self.model = model
-        self.verbose = verbose
-
-    def pipe(self, paths, crops, verbose = None):
-        if verbose is None:
-            verbose = self.verbose
-
-        dataset = InferenceDataset(paths, crops, self.config)
-        if verbose:
-            print("Dataset created")
-
-        dl = DataLoader(
-            dataset=dataset,
-            batch_size=self.config["batchsize"],
-            num_workers=self.config["num_workers"],
-            shuffle = False,
-            pin_memory=True
-        )
-        if verbose:
-            print("Dataloader created")
-
-        predictions = []
-
-        with torch.inference_mode():
-            if self.verbose:
-                print("Starting inference")
-            for i,batch in enumerate(dl):
-                prediction = self.model(batch.cuda())
-                prediction = self.config["activation"](prediction).cpu().tolist()#.numpy().tolist()
-                predictions += prediction
-                if self.verbose and i==0:
-                    print("First batch done")
-
-        return predictions
-
-    def __call__(self, image, crop=None):
-        return self.pipe([image], [crop])
-
-    def readable(self, predictions):
-        return {label: prediction for label, prediction in zip(self.config["labels"], predictions)}
-    
-    def get_prediction_dict(self, predictions, paths):
-        json_dict = {
-            "labels": self.config["labels"],
-            "paths": paths,
-            "predictions": predictions
-        }
-        
-        return json_dict
-    
-    def get_prediction_json(self, predictions, paths, json_target_path: str = None):
-        if not json_target_path:
-            json_target_path = "predictions.json"
-
-        json_dict = self.get_prediction_dict(predictions, paths)
-
-        with open(json_target_path, 'w') as f:
-            json.dump(json_dict, f)
-        
-        if self.verbose:
-            print(f"Saved predictions to {json_target_path}")
-
-    def post_process_predictions(self, pred_dicts, window_size_s=1, fps=50, min_seq_len_s = 0.5):
-        '''
-        pred_dicts: list of dictionaries with the same keys
-        window_size_s: size of the window in seconds for smoothing
-        fps: frames per second
-        min_seq_len_s: minimum length of a sequence in seconds
-
-        Returns:
-        predictions: concatenated predictions
-        smooth_predictions: smoothed predictions
-        binary_predictions: binary predictions
-        raw_sequences: raw sequences
-        filtered_sequences: filtered sequences
-        '''
-        # Concatenate the predictions
-        predictions = concat_pred_dicts(pred_dicts)
-
-        smooth_predictions = {key:[] for key in predictions.keys()}
-        for key in predictions.keys():
-            smooth_predictions[key] = make_smooth_preds(
-                predictions[key],
-                window_size_s=window_size_s,
-                fps=fps
-            )
-
-        binary_predictions = {}
-        for key in smooth_predictions.keys():
-            binary_predictions[key] = np.array([p > 0.5 for p in smooth_predictions[key]])
-
-        raw_sequences = {}
-        for key in binary_predictions.keys():
-            raw_sequences[key] = find_true_pred_sequences(binary_predictions[key])
-
-        filtered_sequences = {}
-        min_seq_len = int(min_seq_len_s * fps)
-        for key in raw_sequences.keys():
-            filtered_sequences[key] = [s for s in raw_sequences[key] if s[1] - s[0] > min_seq_len]
-
-        return predictions, smooth_predictions, binary_predictions, raw_sequences, filtered_sequences
-
-    def post_process_predictions_serializable(
-        self, pred_dicts,
-        window_size_s = 1, fps = 50,
-        min_seq_len_s = 0.5
-    ):
-        result = self.post_process_predictions(
-            pred_dicts,
-            window_size_s,
-            fps, min_seq_len_s
-        )
-
-        for i, _dict in enumerate(result):
-            _keys = list(_dict.keys())
-            for key in _keys:
-                # if numpy array
-                if hasattr(_dict[key], "tolist"):
-                    result[i][key] = _dict[key].tolist()
-                
-                # check if list of tuples
-                # if so, make sure each tuple has 2 elements and split to two lists (start, stop)
-                if all(isinstance(x, tuple) for x in _dict[key]):
-                    if all(len(x) == 2 for x in _dict[key]):
-                        result[i][f"{key}_start"] = [int(x[0]) for x in _dict[key]]
-                        result[i][f"{key}_stop"] = [int(x[1]) for x in _dict[key]]
-                        del result[i][key]
-        
-        # make dict of dicts
-        result_dict = {
-            "predictions": result[0],
-            "smooth_predictions": result[1],
-            "binary_predictions": result[2],
-            "raw_sequences": result[3],
-            "filtered_sequences": result[4]
-        }
-
-        return result_dict
-                

endoreg_db/models/ai_model/lightning/prediction_visualizer.py

@@ -1,55 +0,0 @@
-import streamlit as st
-import matplotlib.pyplot as plt
-import json
-from PIL import Image
-import argparse
-
-
-# Load the JSON file
-@st.cache_data
-def load_data(file_path):
-    with open(file_path, 'r') as f:
-        data = json.load(f)
-    return data
-
-# Streamlit App
-def main(file_path):
-    st.title("Visualize Predictions")
-
-    # Load the data
-    data = load_data(file_path)
-
-    # Dropdown to select label
-    selected_label = st.selectbox('Select Label', options=data['labels'])
-
-    # Get the index of the selected label
-    label_index = data['labels'].index(selected_label)
-
-    # Extract predictions for the selected label
-    selected_predictions = [pred[label_index] for pred in data['predictions']]
-
-    # Line plot for the selected label's prediction values
-    st.subheader(f"Line plot for {selected_label}")
-
-    fig, ax = plt.subplots(figsize=(10, 6))  # Explicitly create a figure and axis
-    ax.plot(selected_predictions)
-    ax.set_xlabel('Frame')
-    ax.set_ylabel('Prediction Value')
-    ax.set_title(f'Predictions for {selected_label}')
-
-    st.pyplot(fig)  # Pass the figure object explicitly
-
-
-    # Slider to select frame
-    frame_idx = st.slider('Select a frame', 0, len(data['paths']) - 1)
-
-    # Display the frame
-    image_path = data['paths'][frame_idx]
-    image = Image.open(image_path)
-    st.image(image, caption=f"Frame {frame_idx}", use_column_width=True)
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description="Visualize Predictions")
-    parser.add_argument("--file", type=str, help="Path to JSON file containing predictions", required=True)
-    args = parser.parse_args()
-    main(args.file)

endoreg_db/models/ai_model/lightning/run_visualizer.py

@@ -1,21 +0,0 @@
-# agl_visualization/run_visualizer.py
-
-import subprocess
-import argparse
-
-def main():
-    parser = argparse.ArgumentParser(description="Visualize Predictions")
-    parser.add_argument("--file", type=str, help="Path to JSON file containing predictions", required=True)
-    args = parser.parse_args()
-
-    # Replace this with the path where prediction_visualizer.py resides
-    streamlit_script_path = "agl_predict_endo_frame/prediction_visualizer.py"
-    
-    # Using subprocess to run the Streamlit command
-    subprocess.run(f'streamlit run {streamlit_script_path} -- --file {args.file}', shell=True)
-
-# if __name__ == "__main__":
-    # parser = argparse.ArgumentParser(description="Visualize Predictions")
-    # parser.add_argument("--file", type=str, help="Path to JSON file containing predictions", required=True)
-    # args = parser.parse_args()
-    # main(args.file)

endoreg_db/models/ai_model/model_meta.py

@@ -1,240 +0,0 @@
-"""
-This module defines the ModelMeta and ModelMetaManager classes for managing AI model metadata.
-"""
-
-from pathlib import Path
-from typing import Optional, TYPE_CHECKING
-import shutil
-
-from django.db import models
-from django.core.validators import FileExtensionValidator
-from icecream import ic
-from ...utils import WEIGHTS_DIR
-
-if TYPE_CHECKING:
-    from endoreg_db.models import LabelSet, AiModel  # pylint: disable=import-outside-toplevel
-
-class ModelMetaManager(models.Manager):
-    """
-    Custom manager for ModelMeta with additional query methods.
-    """
-
-    def get_by_natural_key(self, name: str, version: str) -> "ModelMeta":
-        """Get the model by its natural key"""
-        return self.get(name=name, version=version)
-
-
-class ModelMeta(models.Model):
-    """
-    Attributes:
-        name (str): The name of the model.
-        version (str): The version of the model.
-        type (ModelType): The type of the model.
-        labelset (LabelSet): Associated labels.
-        weights (FileField): Path to the model weights.
-    """
-
-    name = models.CharField(max_length=255)
-    version = models.CharField(max_length=255)
-
-    model = models.ForeignKey(
-        "AiModel", on_delete=models.CASCADE, related_name="models"
-    )
-
-    labelset = models.ForeignKey(
-        "LabelSet", on_delete=models.CASCADE, related_name="models"
-    )
-    activation = models.CharField(max_length=255, default="sigmoid")
-
-    weights = models.FileField(
-        upload_to=WEIGHTS_DIR.name,
-        validators=[FileExtensionValidator(allowed_extensions=["ckpt"])],
-        # storage=FileSystemStorage(location=STORAGE_LOCATION.resolve().as_posix()),
-        null=True,
-        blank=True,
-    )
-
-    # Means and stds for normalization
-    mean = models.CharField(max_length=255, default="0.45211223,0.27139644,0.19264949")
-    std = models.CharField(max_length=255, default="0.31418097,0.21088019,0.16059452")
-
-    size_x = models.IntegerField(default=716)
-    size_y = models.IntegerField(default=716)
-    axes = models.CharField(max_length=255, default="2,0,1")
-    batchsize = models.IntegerField(default=16)
-    num_workers = models.IntegerField(default=0)
-
-    description = models.TextField(blank=True, null=True)
-    date_created = models.DateTimeField(auto_now_add=True)
-
-    objects = ModelMetaManager()
-
-    if TYPE_CHECKING:
-        labelset: "LabelSet"
-        ai_model: "AiModel"
-
-    @classmethod
-    def create_from_file(
-        cls,
-        meta_name,
-        model_name,
-        model_meta_version,
-        labelset_name,
-        weights_file,
-        bump_version,
-        **kwargs,
-    ):
-        """Create a new model meta from a file"""
-
-        from endoreg_db.models import LabelSet, AiModel  # pylint: disable=import-outside-toplevel
-
-        ic("Start")
-        model_meta_version = kwargs.get("model_meta_version", None)
-
-        ai_model = AiModel.objects.get(name=model_name)
-
-        # check If ModelMeta with same name and model already exists
-        if not model_meta_version:
-            if cls.objects.filter(name=meta_name, model=ai_model).exists():
-                # get highest version number
-                highest_version = (
-                    cls.objects.filter(name=meta_name, model=ai_model)
-                    .latest("version")
-                    .version
-                )
-                version = int(highest_version) + 1
-            else:
-                version = 1
-        else:
-            version = model_meta_version
-
-        meta_exists = cls.objects.filter(
-            name=meta_name, model=ai_model, version=version
-        ).exists()
-        if meta_exists and not bump_version:
-            raise ValueError(
-                f"ModelMeta with name {meta_name} and version {version} already exists"
-            )
-
-        assert labelset_name is not None, "Labelset name must be provided"
-        labelset = LabelSet.objects.get(name=labelset_name)
-        assert labelset is not None, "Labelset not found"
-
-        weights_path = Path(weights_file)
-        # If not under our WEIGHTS_DIR, copy it there
-        if not str(weights_path).startswith(str(WEIGHTS_DIR)):
-            target_path = WEIGHTS_DIR / weights_path.name
-            if not target_path.exists():
-                shutil.copy(weights_path, target_path)
-            weights_file = target_path
-
-        return cls.objects.create(
-            name=meta_name,
-            model=ai_model,
-            version=version,
-            labelset=labelset,
-            weights=str(weights_file),
-            **kwargs,
-        )
-
-    @classmethod
-    def get_latest_version(cls, name) -> int:
-        """
-        get the model meta with the highest version. Assumes Model Name and Meta Name are the same
-        """
-        from endoreg_db.models import AiModel  # pylint: disable=import-outside-toplevel
-
-        ai_model = AiModel.objects.get(name=name)
-        ic(f"Model: {ai_model}")
-        assert ai_model is not None, "Model not found"
-
-        model_meta = cls.objects.filter(name=name, model=ai_model).latest("version")
-        ic(f"Latest ModelMeta: {model_meta}")
-
-        latest_version = model_meta.version
-        ic(f"Latest Version: {latest_version}")
-
-        return int(latest_version)
-
-    @classmethod
-    def get_activation_function(cls, activation: str):
-        """get the activation function"""
-        from torch import nn  # pylint: disable=import-outside-toplevel
-
-        lookup = {
-            "sigmoid": nn.Sigmoid(),
-            "softmax": nn.Softmax(),
-            "none": nn.Identity(),
-        }
-
-        return lookup[activation]
-
-    def get_inference_dataset_config(self):
-        """
-        Creates a dictionary with the configuration for the inference dataset.
-
-        Example:
-            sample_config = {
-                # mean and std for normalization
-                "mean": (0.45211223, 0.27139644, 0.19264949),
-                "std": (0.31418097, 0.21088019, 0.16059452),
-                # Image Size
-                "size_x": 716,
-                "size_y": 716,
-                # how to wrangle axes of the image before putting them in the network
-                "axes": [2,0,1],  # 2,1,0 for opencv
-                "batchsize": 16,
-                "num_workers": 0, # always 1 for Windows systems # FIXME: fix celery crash if multiprocessing
-                # maybe add sigmoid after prediction?
-                "activation": nn.Sigmoid(),
-                "labels": ['appendix',  'blood',  'diverticule',  'grasper',  'ileocaecalvalve',  'ileum',  'low_quality',  'nbi',  'needle',  'outside',  'polyp',  'snare',  'water_jet',  'wound']
-            }
-        """
-        mean = list(map(float, str(self.mean).split(",")))
-        std = list(map(float, str(self.std).split(",")))
-        axes = list(map(int, str(self.axes).split(",")))
-
-        config = {
-            "mean": mean,
-            "std": std,
-            "size_x": self.size_x,
-            "size_y": self.size_y,
-            "axes": axes,
-            "batchsize": self.batchsize,
-            "num_workers": self.num_workers,
-            "activation": self.get_activation_function(self.activation),
-            "labels": self.labelset.get_labels_in_order(),  # pylint: disable=no-member
-        }
-
-        ic(f"ModelMeta - get_inference_dataset_config: {config}")
-
-        return config
-
-    def natural_key(self):
-        """Return the natural key for this model"""
-        return (self.name, self.version)
-
-    def __str__(self):
-        return f"{self.name} (v: {self.version})"
-
-    def get_config_dict(self) -> dict:
-        """Return the configuration as a dictionary"""
-        conf_dict = {
-            "mean": list(map(float, str(self.mean).split(","))),
-            "std": list(map(float, str(self.std).split(","))),
-            "size_x": self.size_x,
-            "size_y": self.size_y,
-            "axes": list(map(int, str(self.axes).split(","))),
-            "batchsize": self.batchsize,
-            "num_workers": self.num_workers,
-            "activation": self.activation,
-            "labels": self.labelset.get_labels_in_order(),  # pylint: disable=no-member
-        }
-        return conf_dict
-
-    @classmethod
-    def get_by_name(cls, name: str, version: Optional[int]) -> "ModelMeta":
-        """Get the model by its name and version"""
-        if version is None:
-            version = cls.get_latest_version(name)
-        return cls.objects.get(name=name, version=version)

endoreg_db/models/annotation/__init__.py

@@ -1,32 +0,0 @@
-"""Module for annotation models"""
-
-from .image_classification import ImageClassificationAnnotation
-from .binary_classification_annotation_task import (
-    BinaryClassificationAnnotationTask,
-)
-from .anonymized_image_annotation import (
-    AnonymousImageAnnotation,
-    DroppedName,
-    AnonymizedImageLabel,
-    AnonymizedFile,
-    UploadedFile,
-)
-from .video_segmentation_annotation import (
-    VideoSegmentationLabel,
-    VideoSegmentationAnnotation,
-)
-
-from .video_segmentation_labelset import VideoSegmentationLabelSet
-
-__all__ = [
-    "ImageClassificationAnnotation",
-    "BinaryClassificationAnnotationTask",
-    "AnonymousImageAnnotation",
-    "DroppedName",
-    "AnonymizedImageLabel",
-    "AnonymizedFile",
-    "UploadedFile",
-    "VideoSegmentationLabel",
-    "VideoSegmentationLabelSet",
-    "VideoSegmentationAnnotation",
-]

endoreg_db/models/annotation/anonymized_image_annotation.py

@@ -1,115 +0,0 @@
-from django.db import models
-from django.utils.text import slugify
-from django.utils import timezone
-
-class AnonymizedImageLabel(models.Model):
-    """
-    Represents a label for anonymized image annotations.
-
-    Attributes:
-        name (str): Unique name of the label.
-        description (str): Optional description for the label.
-    """
-    name = models.CharField(max_length=255, unique=True)
-    description = models.TextField(blank=True, null=True)
-
-    def __str__(self) -> str:
-        return self.name
-
-
-class AnonymousImageAnnotation(models.Model):
-    """
-    Represents an annotation for anonymized images.
-
-    Attributes:
-        label (ForeignKey): Associated label for the image.
-        image_name (str): Name of the image file.
-        original_image_url (str): URL of the original image.
-        slug (str): Unique identifier for the annotation.
-        polyp_count (int): Number of polyps identified in the image.
-        comments (str): Additional comments about the annotation.
-        gender (str): Gender associated with the annotation (if applicable).
-        name_image_url (str): URL of the name image.
-        date_created (datetime): Date when the annotation was created.
-        processed (bool): Indicates if the annotation has been processed.
-    """
-    label = models.ForeignKey(AnonymizedImageLabel, on_delete=models.CASCADE)
-    image_name = models.CharField(max_length=255)
-    original_image_url = models.CharField(max_length=255, default='https://example.com/placeholder.jpg')
-    slug = models.SlugField(unique=True, blank=True, null=True)
-    polyp_count = models.IntegerField()
-    comments = models.TextField(blank=True, null=True)
-    gender = models.CharField(max_length=255)
-    name_image_url = models.CharField(max_length=255)
-    date_created = models.DateTimeField(auto_now_add=True)
-    processed = models.BooleanField(default=False)
-
-    def save(self, *args, **kwargs):
-        """
-        Overrides the save method to generate a slug if not already set.
-        """
-        if not self.slug:
-            self.slug = slugify(f"{self.label}-{self.image_name}")
-        super().save(*args, **kwargs)
-
-    def __str__(self) -> str:
-        return f"{self.image_name} - {self.label}"
-
-
-class DroppedName(models.Model):
-    """
-    Represents a dropped name in the annotation process.
-
-    Attributes:
-        annotation (ForeignKey): Associated annotation.
-        name (str): Dropped name.
-        gender (str): Gender associated with the name.
-        x (float): X-coordinate of the name.
-        y (float): Y-coordinate of the name.
-        name_image_url (str): URL of the name image.
-        box_coordinates (str): Optional coordinates of the bounding box.
-    """
-    annotation = models.ForeignKey(AnonymousImageAnnotation, related_name='dropped_names', on_delete=models.CASCADE)
-    name = models.CharField(max_length=255)
-    gender = models.CharField(max_length=255)
-    x = models.FloatField()
-    y = models.FloatField()
-    name_image_url = models.CharField(max_length=255)
-    box_coordinates = models.CharField(max_length=255, blank=True, null=True)
-
-    def __str__(self) -> str:
-        return f"{self.name} ({self.gender}) at ({self.x}, {self.y})"
-
-
-class UploadedFile(models.Model):
-    """
-    Represents a file uploaded to the system.
-
-    Attributes:
-        original_file (FileField): The uploaded original file.
-        upload_date (datetime): Date when the file was uploaded.
-        description (str): Optional description of the file.
-    """
-    original_file = models.FileField(upload_to='uploads/original/')
-    upload_date = models.DateTimeField(default=timezone.now)
-    description = models.TextField(blank=True, null=True)
-
-    def __str__(self) -> str:
-        return self.original_file.name
-
-
-class AnonymizedFile(models.Model):
-    """
-    Represents an anonymized version of an uploaded file.
-
-    Attributes:
-        original_file (OneToOneField): The original uploaded file.
-        anonymized_file (FileField): The anonymized file.
-        anonymization_date (datetime): Date when the file was anonymized.
-    """
-    original_file = models.OneToOneField(UploadedFile, on_delete=models.CASCADE, related_name='anonymized_file')
-    anonymized_file = models.FileField(upload_to='uploads/anonymized/')
-    anonymization_date = models.DateTimeField(default=timezone.now)
-
-    def __str__(self) -> str:
-        return f"Anonymized version of {self.original_file.original_file.name}"