endoreg-db 0.6.4__py3-none-any.whl → 0.8.1__py3-none-any.whl

endoreg_db/models/data_file/base_classes/frame_helpers.py

@@ -1,17 +0,0 @@
-from django.core.files import File
-import io
-from pathlib import Path
-from typing import List
-
-
-def prepare_bulk_frames(frame_paths: List[Path]):
-    """
-    Reads the frame paths into memory as Django File objects.
-    This avoids 'seek of closed file' errors by using BytesIO for each frame.
-    """
-    for path in frame_paths:
-        frame_number = int(path.stem.split("_")[1])
-        with open(path, "rb") as f:
-            content = f.read()
-        file_obj = File(io.BytesIO(content), name=path.name)
-        yield frame_number, file_obj

endoreg_db/models/data_file/base_classes/prepare_bulk_frames.py

@@ -1,19 +0,0 @@
-from django.core.files import File
-
-
-import io
-from pathlib import Path
-from typing import List
-
-
-def prepare_bulk_frames(frame_paths: List[Path]):
-    """
-    Reads the frame paths into memory as Django File objects.
-    This avoids 'seek of closed file' errors by using BytesIO for each frame.
-    """
-    for path in frame_paths:
-        frame_number = int(path.stem.split("_")[1])
-        with open(path, "rb") as f:
-            content = f.read()
-        file_obj = File(io.BytesIO(content), name=path.name)
-        yield frame_number, file_obj

endoreg_db/models/data_file/base_classes/utils.py

@@ -1,58 +0,0 @@
-"""
-Utility functions for data file classes.
-"""
-
-import os
-from pathlib import Path
-import cv2
-import numpy as np
-
-
-
-def anonymize_frame(
-    raw_frame_path: Path, target_frame_path: Path, endo_roi, all_black: bool = False
-):
-    """
-    Anonymize the frame by blacking out all pixels that are not in the endoscope ROI.
-    """
-
-    frame = cv2.imread(raw_frame_path.as_posix())  # pylint: disable=no-member
-
-    # make black frame with same size as original frame
-    new_frame = np.zeros_like(frame)
-
-    if not all_black:
-        # endo_roi is dict with keys "x", "y", "width", "heigth"
-        x = endo_roi["x"]
-        y = endo_roi["y"]
-        width = endo_roi["width"]
-        height = endo_roi["height"]
-
-        # copy endoscope roi to black frame
-        new_frame[y : y + height, x : x + width] = frame[y : y + height, x : x + width]
-    cv2.imwrite(target_frame_path.as_posix(), new_frame)  # pylint: disable=no-member
-
-    return frame
-
-
-def copy_with_progress(src: str, dst: str, buffer_size=1024 * 1024):
-    """
-    Make a copy of a file with progress bar.
-
-    Args:
-        src (str): Source file path.
-        dst (str): Destination file path.
-        buffer_size (int): Buffer size for copying.
-    """
-    total_size = os.path.getsize(src)
-    copied_size = 0
-
-    with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
-        while True:
-            buf = fsrc.read(buffer_size)
-            if not buf:
-                break
-            fdst.write(buf)
-            copied_size += len(buf)
-            progress = copied_size / total_size * 100
-            print(f"\rProgress: {progress:.2f}%", end="")

endoreg_db/models/data_file/frame.py

@@ -1,29 +0,0 @@
-from typing import TYPE_CHECKING
-
-from endoreg_db.models.label.label import Label
-from .base_classes import AbstractFrame
-from django.db import models
-
-from ...utils import FRAME_DIR_NAME, RAW_FRAME_DIR_NAME
-
-if TYPE_CHECKING:
-    from endoreg_db.models import RawVideoFile, Video
-
-
-class RawFrame(AbstractFrame):
-    image = models.ImageField(upload_to=RAW_FRAME_DIR_NAME, blank=True, null=True)
-    video = models.ForeignKey(
-        "RawVideoFile", on_delete=models.CASCADE, related_name="frames"
-    )
-
-    if TYPE_CHECKING:
-        video: "RawVideoFile"
-        label: "Label"
-
-
-class Frame(AbstractFrame):
-    image = models.ImageField(upload_to=FRAME_DIR_NAME, blank=True, null=True)
-    video = models.ForeignKey("Video", on_delete=models.CASCADE, related_name="frames")
-
-    if TYPE_CHECKING:
-        video: "Video"

endoreg_db/models/data_file/import_classes/__init__.py

@@ -1,18 +0,0 @@
-from .raw_video import RawVideoFile
-from .raw_pdf import RawPdfFile
-
-# FileImporter class
-# This class is used to import data from a file into the database.
-# Expects a directory containing files to import.
-# creates correct import file object depending on file type
-
-# FileImporter class
-# This class is used to import data from a file into the database by creating objects for the files.
-# main method is import_files which expects a path to a directory containing files to import.
-# creates correct import file object depending on file type by checking the file extension
-
-
-__all__ = [
-    "RawPdfFile",
-    "RawVideoFile",
-]

endoreg_db/models/data_file/import_classes/processing_functions/__init__.py

@@ -1,35 +0,0 @@
-from .video import (
-    get_videos_scheduled_for_frame_extraction, 
-    extract_frames_from_video,
-    extract_frames_from_videos,
-
-    get_videos_scheduled_for_ocr,
-    videos_scheduled_for_ocr_preflight,
-    perform_ocr_on_video,
-    perform_ocr_on_videos,
-
-    videos_scheduled_for_initial_prediction_preflight,
-    get_videos_scheduled_for_initial_prediction,
-    get_multilabel_model,
-    get_multilabel_classifier,
-    get_crops,
-    perform_initial_prediction_on_video,
-    perform_initial_prediction_on_videos,
-
-    videos_scheduled_for_prediction_import_preflight,
-    get_videos_scheduled_for_prediction_import,
-    import_predictions_for_video,
-    import_predictions_for_videos,
-    
-
-    delete_frames_preflight,
-    get_videos_scheduled_for_frame_deletion,
-    delete_frames_for_video,
-    delete_frames,
-)
-
-from .pdf import (
-    get_pdf_files_scheduled_for_processing,
-    process_pdf_file,
-    process_pdf_files,
-)

endoreg_db/models/data_file/import_classes/processing_functions/pdf.py

@@ -1,28 +0,0 @@
-from ..raw_pdf import RawPdfFile
-from logging import getLogger
-#
-# # setup logging to pdf_import.log
-logger = getLogger('examination_pdf_import')
-
-def get_pdf_files_scheduled_for_processing():
-    reports = RawPdfFile.objects.filter(
-        state_report_processing_required=True
-    )
-    return reports
-
-def process_pdf_file(report:RawPdfFile):
-    if report.update(save=True, verbose=True):
-        logger.info(f"Report {report} processed successfully")
-        return True
-    else:
-        logger.error(f"Report {report} processing failed")
-        return False
-
-
-def process_pdf_files():
-    reports = get_pdf_files_scheduled_for_processing()
-    for report in reports:
-        process_pdf_file(report)
-    
-
-

endoreg_db/models/data_file/import_classes/processing_functions/video.py

@@ -1,260 +0,0 @@
-from ..raw_video import RawVideoFile
-import json 
-
-# # Starting point
-# Automated tasks generate RawVideoFile objects in our db.
-# Each object has state_{NAME} attributes.
-# We will create functions which query the db for RawVideoFile 
-# objects with specific state_{NAME} attributes.
-# Then, we perform the necessary operations on the RawVideoFile and
-# update the state_{NAME} attributes accordingly.
-
-# # Step 1 - Frame Extraction
-# function to query for videos scheduled for frame extraction,
-# these have state_frames_required and state_frames_extracted
-def get_videos_scheduled_for_frame_extraction():
-    return RawVideoFile.objects.filter(
-        state_frames_required=True,
-        state_frames_extracted=False
-    )
-
-def extract_frames_from_video(video:RawVideoFile):
-    # extract frames from video
-    video.extract_frames()
-
-    # update state_frames_extracted
-    video.state_frames_extracted = True
-    video.save()
-
-    return video
-
-def extract_frames_from_videos():
-    videos = get_videos_scheduled_for_frame_extraction()
-    for video in videos:
-        extract_frames_from_video(video)
-
-# # Step 2 - OCR
-# function to query for videos scheduled for OCR,
-# these have 
-# state_ocr_required = True and state_ocr_completed = False and state_frames_extracted = True
-def get_videos_scheduled_for_ocr():
-    return RawVideoFile.objects.filter(
-        state_ocr_required=True,
-        state_ocr_completed=False,
-        state_frames_extracted=True
-    )
-
-# function to set state_frames_required to True for videos 
-# which are scheduled for OCR but have not had frames extracted
-def videos_scheduled_for_ocr_preflight():
-    videos = RawVideoFile.objects.filter(
-        state_ocr_required=True,
-        state_ocr_completed=False,
-        state_frames_extracted=False
-    )
-    for video in videos:
-        video.state_frames_required = True
-        video.save()
-
-def perform_ocr_on_video(video:RawVideoFile):
-    # perform OCR on video
-    video.update_text_metadata()
-
-    # update state_ocr_completed
-    video.state_ocr_completed = True
-    video.save()
-
-    return video
-
-def perform_ocr_on_videos():
-    videos = get_videos_scheduled_for_ocr()
-    for video in videos:
-        perform_ocr_on_video(video)
-
-
-# # Step 3 - initial Prediction
-# function to query for videos scheduled for initial prediction,
-# these have
-# state_initial_prediction_required = True and state_initial_prediction_completed = False and state_frames_extracted = True
-def videos_scheduled_for_initial_prediction_preflight():
-    videos = RawVideoFile.objects.filter(
-        state_initial_prediction_required=True,
-        state_initial_prediction_completed=False,
-        state_frames_extracted=False
-    )
-    for video in videos:
-        video.state_frames_required = True
-        video.save()
-
-def get_videos_scheduled_for_initial_prediction():
-    return RawVideoFile.objects.filter(
-        state_initial_prediction_required=True,
-        state_initial_prediction_completed=False,
-        state_frames_extracted=True
-    )
-
-from pathlib import Path
-def get_multilabel_model(model_path:Path):
-    from agl_predict_endo_frame.model_loader import MultiLabelClassificationNet
-    model_path_str = model_path.resolve().as_posix()
-    model = MultiLabelClassificationNet.load_from_checkpoint(model_path_str)
-    model.cuda()
-    model.eval()
-    return model
-
-def get_multilabel_classifier(model, verbose:bool=False):
-    from agl_predict_endo_frame.predict import Classifier
-    classifier = Classifier(model, verbose = verbose)
-    return classifier
-
-def get_crops(video, paths):
-    endo_roi_dict = video.get_endo_roi()
-    # dict with x, y, width height
-    # crops is list of touples with (y_min, y_max, x_min, x_max)
-    crop_tuple = (
-        endo_roi_dict["y"],
-        endo_roi_dict["y"] + endo_roi_dict["height"],
-        endo_roi_dict["x"],
-        endo_roi_dict["x"] + endo_roi_dict["width"],  
-    )
-    crops = [crop_tuple for _ in paths]
-    return crops
-
-# model = MultiLabelClassificationNet.load_from_checkpoint("model/colo_segmentation_RegNetX800MF_6.ckpt")
-def perform_initial_prediction_on_video(
-    video:RawVideoFile, model_path,
-    window_size_s, min_seq_len_s
-):
-
-    model = get_multilabel_model(model_path)
-    classifier = get_multilabel_classifier(model, verbose = True)
-
-    paths = video.get_frame_paths()
-    string_paths = [p.resolve().as_posix() for p in paths]
-    crops = get_crops(video, string_paths)
-    fps = video.get_fps()
-
-    predictions = classifier.pipe(string_paths, crops)
-    readable_predictions = [classifier.readable(p) for p in predictions]
-    result_dict = classifier.post_process_predictions_serializable(
-        readable_predictions,
-        window_size_s = window_size_s,
-        min_seq_len_s = min_seq_len_s,
-        fps = fps
-    )
-
-    
-    # Predictions
-    _path = video.get_predictions_path()
-    with open(_path, "w") as f:
-        json.dump(result_dict["predictions"], f, indent = 4)
-
-    # smooth_predictions
-    _path = video.get_smooth_predictions_path()
-    with open(_path, "w") as f:
-        json.dump(result_dict["smooth_predictions"], f, indent = 4)
-
-    # binary_predictions
-    _path = video.get_binary_predictions_path()
-    with open(_path, "w") as f:
-        json.dump(result_dict["binary_predictions"], f, indent = 4)
-
-    # Raw Sequences
-    _path = video.get_raw_sequences_path()
-    with open(_path, "w") as f:
-        json.dump(result_dict["raw_sequences"], f, indent = 4)
-
-    # filtered_sequences
-    _path = video.get_filtered_sequences_path()
-    with open(_path, "w") as f:
-        json.dump(result_dict["filtered_sequences"], f, indent = 4)
-
-    
-    # update state_initial_prediction_completed
-    video.state_initial_prediction_required = False
-    video.state_initial_prediction_completed = True
-    video.state_initial_prediction_import_required = True
-    video.state_initial_prediction_import_completed = False
-    video.save()
-
-    return video
-
-def perform_initial_prediction_on_videos(
-    model_path,
-    window_size_s, min_seq_len_s
-):
-    videos = get_videos_scheduled_for_initial_prediction()
-    for video in videos:
-        perform_initial_prediction_on_video(
-        video,
-        model_path, window_size_s, min_seq_len_s
-    )
-
-def videos_scheduled_for_prediction_import_preflight():
-    videos = RawVideoFile.objects.filter(
-        state_initial_prediction_completed=True,
-        state_initial_prediction_import_completed=False
-    )
-    for video in videos:
-        video.state_initial_prediction_required = True
-        video.save()
-    
-def get_videos_scheduled_for_prediction_import():
-    return RawVideoFile.objects.filter(
-        state_initial_prediction_import_required=True,
-        state_initial_prediction_import_completed=False,
-        state_initial_prediction_completed=True
-    )
-
-def import_predictions_for_video(video:RawVideoFile):
-    # import predictions for video
-    pass
-
-    # update state_prediction_import_completed
-    # video.state_initial_prediction_import_required = False
-    # video.state_initial_prediction_import_completed = True
-    # video.save()
-
-    return video
-
-def import_predictions_for_videos():
-    videos = get_videos_scheduled_for_prediction_import()
-    for video in videos:
-        import_predictions_for_video(video)
-
-# # Step 4 - Delete Frames if not needed anymore
-# function to query for videos scheduled for frame deletion,
-# first we need to set state_frames_required = False for videos with:
-# state_ocr_required = False and state_ocr_completed = True and
-# state_initial_prediction_required = False and state_initial_prediction_completed = True
-def delete_frames_preflight():
-    videos = RawVideoFile.objects.filter(
-        state_ocr_required=False,
-        state_ocr_completed=True,
-        state_initial_prediction_required=False,
-        state_initial_prediction_completed=True
-    )
-    for video in videos:
-        video.state_frames_required = False
-        video.save()
-
-# function to query for videos scheduled for frame deletion,
-# frames should be deleted if state_frames_required = False
-def get_videos_scheduled_for_frame_deletion():
-    return RawVideoFile.objects.filter(
-        state_frames_required=False
-    )
-
-def delete_frames_for_video(video:RawVideoFile):
-    # delete frames for video
-
-    # update state_frames_deleted
-    video.state_frames_extracted = False
-    video.save()
-
-    return video
-
-def delete_frames():
-    videos = get_videos_scheduled_for_frame_deletion()
-    for video in videos:
-        delete_frames_for_video(video)