endoreg-db 0.5.2__py3-none-any.whl → 0.6.0__py3-none-any.whl

endoreg_db/utils/dataloader.py

@@ -1,92 +0,0 @@
-import os
-import yaml
-from django.core.exceptions import ObjectDoesNotExist
-
-def load_model_data_from_yaml(command, model_name, metadata, verbose):
-    """
-    Load model data from YAML files.
-    
-    Args:
-        command: Command object for stdout writing.
-        model_name: Name of the model being loaded.
-        metadata: Metadata including directory and foreign key information.
-        verbose: Boolean indicating whether to print verbose output.
-    """
-    if verbose:
-        command.stdout.write(f"Start loading {model_name}")
-    model = metadata["model"]
-    dir_path = metadata["dir"]
-    foreign_keys = metadata["foreign_keys"]
-    foreign_key_models = metadata["foreign_key_models"]
-
-    _files = [f for f in os.listdir(dir_path) if f.endswith('.yaml')]
-    # sort
-    _files.sort()
-    for file in _files:
-        with open(os.path.join(dir_path, file), 'r') as file:
-            yaml_data = yaml.safe_load(file)
-        
-        load_data_with_foreign_keys(command, model, yaml_data, foreign_keys, foreign_key_models, verbose)
-
-def load_data_with_foreign_keys(command, model, yaml_data, foreign_keys, foreign_key_models, verbose):
-    """
-    Load data handling foreign keys and many-to-many relationships.
-    
-    Args:
-        command: Command object for stdout writing.
-        model: The Django model for the data.
-        yaml_data: Data loaded from YAML.
-        foreign_keys: List of foreign keys.
-        foreign_key_models: Corresponding models for the foreign keys.
-        verbose: Boolean indicating whether to print verbose output.
-    """
-    for entry in yaml_data:
-        fields = entry.get('fields', {})
-        name = fields.pop('name', None)
-        m2m_relationships = {}  # Store many-to-many relationships
-        print(entry)
-
-        # Handle foreign keys and many-to-many relationships
-        for fk_field, fk_model in zip(foreign_keys, foreign_key_models):
-            print(fk_field, fk_model)
-            target_keys = fields.pop(fk_field, None)
-            
-            # Ensure the foreign key exists
-            if target_keys is None:
-                if verbose:
-                    command.stdout.write(command.style.WARNING(f"Foreign key {fk_field} not found in fields"))
-                continue  # Skip if no foreign key provided
-
-            # Process many-to-many fields or foreign keys
-            if isinstance(target_keys, list):  # Assume many-to-many relationship
-                related_objects = []
-                for key in target_keys:
-                    obj, created = fk_model.objects.get_or_create(name=key)
-                    if created and verbose:
-                        command.stdout.write(command.style.SUCCESS(f"Created {fk_model.__name__} {key}"))
-                    related_objects.append(obj)
-                m2m_relationships[fk_field] = related_objects
-            else:  # Single foreign key relationship
-                try:
-                    if model == "endoreg_db.case_template_rule":
-                        print(fk_model, target_keys)
-                    obj = fk_model.objects.get_by_natural_key(target_keys)
-                except ObjectDoesNotExist:
-                    if verbose:
-                        command.stdout.write(command.style.WARNING(f"{fk_model.__name__} with key {target_keys} not found"))
-                    continue
-                fields[fk_field] = obj
-    
-        # Create or update the main object
-        if name is None:
-            obj, created = model.objects.get_or_create(**fields)
-        else:
-            obj, created = model.objects.update_or_create(defaults=fields, name=name)
-        if created and verbose:
-            command.stdout.write(command.style.SUCCESS(f'Created {model.__name__} {name}'))
-        elif verbose:
-            command.stdout.write(command.style.WARNING(f'Skipped {model.__name__} {name}, already exists'))
-
-        # Set many-to-many relationships
-        for field_name, related_objs in m2m_relationships.items():
-            getattr(obj, field_name).set(related_objs)

endoreg_db/utils/file_operations.py

@@ -1,30 +0,0 @@
-import shutil
-from pathlib import Path
-from endoreg_db.utils.uuid import get_uuid
-
-def get_uuid_filename(file:Path) -> tuple[str, str]:
-    """
-    Returns a new filename with a uuid
-    """
-    # Get the file extension
-    file_extension = file.suffix
-    # Generate a new file name
-    uuid = get_uuid()
-    new_file_name = f"{uuid}{file_extension}"
-    return new_file_name, uuid
-    
-def rename_file_uuid(old_file:Path):
-    """
-    Rename a file by assigning a uuid while preserving file extension. Returns new filepath and uuid
-    """
-    # Get the file extension
-    file_extension = old_file.suffix
-    # Generate a new file name
-    uuid = get_uuid()
-    new_file_name = f"{uuid}{file_extension}"
-
-    # Rename the file
-    new_file = old_file.with_name(new_file_name)
-    shutil.move(old_file.resolve().as_posix(), new_file.resolve().as_posix())
-
-    return new_file, uuid

endoreg_db/utils/hashs.py

@@ -1,34 +0,0 @@
-import hashlib
-from pathlib import Path
-
-def get_video_hash(video_path):
-    """
-    Get the hash of a video file.
-    """
-    # Open the video file in read-binary mode:
-    with open(video_path, 'rb') as f:
-        # Create the hash object, passing in the video contents for hashing:
-        hash_object = hashlib.sha256(f.read())
-        # Get the hexadecimal representation of the hash
-        video_hash = hash_object.hexdigest()
-        assert len(video_hash) <= 255, "Hash length exceeds 255 characters"
-
-    return video_hash
-
-def get_pdf_hash(pdf_path:Path):
-    """
-    Get the hash of a pdf file.
-    """
-    pdf_hash = None
-
-    # Open the file in binary mode and read its contents
-    with open(pdf_path, 'rb') as f:
-        pdf_contents = f.read()
-        # Create a hash object using SHA-256 algorithm
-
-    hash_object = hashlib.sha256(pdf_contents, usedforsecurity=False)
-    # Get the hexadecimal representation of the hash
-    pdf_hash = hash_object.hexdigest()
-    assert len(pdf_hash) <= 255, "Hash length exceeds 255 characters"
-
-    return pdf_hash

endoreg_db/utils/legacy_ocr.py

@@ -1,201 +0,0 @@
-import pytesseract
-# import cv2
-from endoreg_db.models import EndoscopyProcessor
-import os
-from collections import Counter
-from tempfile import TemporaryDirectory
-import re
-from datetime import datetime
-from typing import Dict, List
-import numpy as np
-
-N_FRAMES_MEAN_OCR = 2
-
-# Helper function to process date strings
-def process_date_text(date_text):
-    """
-    Processes a string of text that represents a date and returns a datetime.date object.
-
-    Args:
-        date_text (str): A string of text that represents a date.
-
-    Returns:
-        datetime.date: A datetime.date object representing the parsed date, or None if the text cannot be parsed.
-    """
-    try:
-        # Remove any non-digit characters
-        date_text_clean = re.sub(r'\D', '', date_text)
-        # Reformat to 'ddmmyyyy' if necessary
-        if len(date_text_clean) == 8:
-            return datetime.strptime(date_text_clean, "%d%m%Y").date()
-        elif len(date_text_clean) == 14:
-            return datetime.strptime(date_text_clean, "%d%m%Y%H%M%S").date()
-    except ValueError:
-        # Return None if the text cannot be parsed into a date
-        # set date to 1/1/1900
-        return datetime.strptime("01011900", "%d%m%Y").date()
-
-# Helper function to process patient names
-def process_name_text(name_text):
-    """
-    Remove all numbers, punctuation, and whitespace from a string of text and return the result.
-    """
-    name = re.sub(r'[0-9!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~\s]+', '', name_text).strip()
-    # capitalize first letter of each word
-    name = ' '.join([word.capitalize() for word in name.split()])
-    return name
-
-
-# Helper function to process endoscope type text
-def process_general_text(endoscope_text):
-    """
-    This function takes in a string of text from an endoscope and returns a cleaned version of the text.
-    """
-    return ' '.join(endoscope_text.split())
-
-def roi_values_valid(roi):
-    """
-    Check if all values in an ROI dictionary are valid (>=0).
-    """
-    return all([value >= 0 for value in roi.values()])
-
-# Function to extract text from ROIs
-def extract_text_from_rois(image_path, processor:EndoscopyProcessor):
-    """
-    Extracts text from regions of interest (ROIs) in an image using OCR.
-
-    Args:
-        image_path (str): The path to the image file.
-        processor (EndoscopyProcessor): An instance of the EndoscopyProcessor class.
-
-    Returns:
-        dict: A dictionary containing the extracted text for each ROI.
-    """
-    # Read the image using OpenCV
-    image = cv2.imread(image_path)
-
-    # Initialize the dictionary to hold the extracted text
-    extracted_texts = {}
-
-    # Define your ROIs and their corresponding post-processing functions in tuples
-    rois_with_postprocessing = [
-        ('examination_date', processor.get_roi_examination_date, process_date_text),
-        ("patient_first_name", processor.get_roi_patient_first_name, process_name_text),
-        ('patient_last_name', processor.get_roi_patient_last_name, process_name_text),
-        ('patient_dob', processor.get_roi_patient_dob, process_date_text),
-        ('endoscope_type', processor.get_roi_endoscope_type, process_general_text),
-        ('endoscope_sn', processor.get_roi_endoscopy_sn, process_general_text),
-    ]
-
-    # Extract and post-process text for each ROI
-    for roi_name, roi_function, post_process in rois_with_postprocessing:
-        # Get the ROI dictionary
-        roi = roi_function()
-        
-        # Check if the ROI has values
-        
-        if roi_values_valid(roi):
-            # Crop the image to the ROI
-            x, y, w, h = roi['x'], roi['y'], roi['width'], roi['height']
-            roi_cropped = image[y:y+h, x:x+w]
-            # Convert to grayscale
-            gray = cv2.cvtColor(roi_cropped, cv2.COLOR_BGR2GRAY)
-
-            # Invert colors for white text on black background
-            gray = cv2.bitwise_not(gray)
-
-            # Binarize the image - using Otsu's method
-            _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
-
-            # Dilate the image to improve the contour of the pixelated text
-            kernel = np.ones((2,2), np.uint8)
-            dilation = cv2.dilate(binary, kernel, iterations=1)
-
-            # OCR configuration: Recognize white text on black background without corrections
-            config = '--psm 10 -c tessedit_char_whitelist=0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-üöäÜÖÄß'
-
-            # Use pytesseract to do OCR on the preprocessed ROI
-            text = pytesseract.image_to_string(dilation, config=config).strip()
-
-            # Post-process extracted text
-            processed_text = post_process(text)
-            # processed_text = text
-            
-            # Store the processed text in the dictionary
-            extracted_texts[roi_name] = processed_text
-
-        else:
-            pass
-            # ic(roi_name)
-            # ic(roi)
-            # ic("No values for this ROI")
-
-    return extracted_texts
-
-
-def get_most_frequent_values(rois_texts: Dict[str, List[str]]) -> Dict[str, str]:
-    """
-    Given a dictionary of ROIs and their corresponding texts, returns a dictionary of the most frequent text for each ROI.
-
-    Args:
-        rois_texts: A dictionary where the keys are the names of the ROIs and the values are lists of texts.
-
-    Returns:
-        A dictionary where the keys are the names of the ROIs and the values are the most frequent text for each ROI.
-    """
-    most_frequent = {}
-    for key in rois_texts.keys():
-        counter = Counter([text for text in rois_texts[key] if text])
-        # ic(key)
-        # ic(counter)
-        most_frequent[key], _ = counter.most_common(1)[0] if counter else (None, None)
-    return most_frequent
-
-def process_video(video_path, processor):
-    """
-    Processes a video file by extracting text from regions of interest (ROIs) in each frame.
-
-    Args:
-        video_path (str): The path to the video file to process.
-        processor (OCRProcessor): An instance of the OCRProcessor class that defines the ROIs to extract text from.
-
-    Returns:
-        dict: A dictionary containing the most frequent text values extracted from each ROI.
-    """
-    # Create a temporary directory to store frames
-    with TemporaryDirectory() as temp_dir:
-        # ic(temp_dir)
-        # Capture the video
-        video = cv2.VideoCapture(video_path)
-        success, frame_number = True, 0
-        rois_texts = {roi_name: [] for roi_name in processor.get_rois().keys()}
-        frames_for_mean_extraction = 0
-
-        while success:
-            success, frame = video.read()
-
-            # Check if this is the 200th frame
-            if frame_number % 1000 == 0 and success:
-                frame_path = os.path.join(temp_dir, f"frame_{frame_number}.jpg")
-                cv2.imwrite(frame_path, frame)  # Save the frame as a JPEG file
-                # cv2.imwrite(f"_tmp/frame_{frame_number}.jpg", frame)
-                
-                # Extract text from ROIs
-                extracted_texts = extract_text_from_rois(frame_path, processor)
-                # ic(extracted_texts)
-                
-                # Store the extracted text from each ROI
-                for key, text in extracted_texts.items():
-                    rois_texts[key].append(text)
-                frames_for_mean_extraction += 1
-
-            frame_number += 1
-
-            if frames_for_mean_extraction >= N_FRAMES_MEAN_OCR: break
-
-        # Release the video capture object
-        video.release()
-
-        # Get the most frequent values for each ROI
-        return get_most_frequent_values(rois_texts)
-

endoreg_db/utils/ocr.py

@@ -1,190 +0,0 @@
-import pytesseract
-from PIL import Image, ImageOps, ImageFilter
-import os
-from collections import Counter
-from tempfile import TemporaryDirectory
-import re
-from datetime import datetime
-from typing import Dict, List
-import numpy as np
-from endoreg_db.utils.cropping import crop_and_insert
-
-
-
-N_FRAMES_MEAN_OCR = 2
-
-# Helper function to process date strings
-def process_date_text(date_text):
-    """
-    Processes a string of text that represents a date and returns a datetime.date object.
-
-    Args:
-        date_text (str): A string of text that represents a date.
-
-    Returns:
-        datetime.date: A datetime.date object representing the parsed date, or None if the text cannot be parsed.
-    """
-    try:
-        # Remove any non-digit characters
-        date_text_clean = re.sub(r'\D', '', date_text)
-        # Reformat to 'ddmmyyyy' if necessary
-        if len(date_text_clean) == 8:
-            return datetime.strptime(date_text_clean, "%d%m%Y").date()
-        elif len(date_text_clean) == 14:
-            return datetime.strptime(date_text_clean, "%d%m%Y%H%M%S").date()
-    except ValueError:
-        # Return None if the text cannot be parsed into a date
-        # set date to 1/1/1900
-        return datetime.strptime("01011900", "%d%m%Y").date()
-
-# Helper function to process patient names
-def process_name_text(name_text):
-    """
-    Remove all numbers, punctuation, and whitespace from a string of text and return the result.
-    """
-    name = re.sub(r'[0-9!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~\s]+', '', name_text).strip()
-    # capitalize first letter of each word
-    name = ' '.join([word.capitalize() for word in name.split()])
-    return name
-
-
-# Helper function to process endoscope type text
-def process_general_text(endoscope_text):
-    """
-    This function takes in a string of text from an endoscope and returns a cleaned version of the text.
-    """
-    return ' '.join(endoscope_text.split())
-
-def roi_values_valid(roi):
-    """
-    Check if all values in an ROI dictionary are valid (>=0).
-    """
-    return all([value >= 0 for value in roi.values()])
-
-# Function to extract text from ROIs
-def extract_text_from_rois(image_path, processor):
-    """
-    Extracts text from regions of interest (ROIs) in an image using OCR.
-
-    Args:
-        image_path (str): The path to the image file.
-        processor (EndoscopyProcessor): An instance of the EndoscopyProcessor class.
-
-    Returns:
-        dict: A dictionary containing the extracted text for each ROI.
-    """
-    # Read the image using Pillow
-    image = Image.open(image_path)
-    image_dimensions = image.size # (width, height)
-
-    ####### Adjust Image #######
-    # Convert to grayscale
-    gray = image.convert('L')
-
-    # Invert colors for white text on black background
-    inverted = ImageOps.invert(gray)
-
-    # Initialize the dictionary to hold the extracted text
-    extracted_texts = {}
-
-    # Define your ROIs and their corresponding post-processing functions in tuples
-    rois_with_postprocessing = [
-        ('examination_date', processor.get_roi_examination_date, process_date_text),
-        ("patient_first_name", processor.get_roi_patient_first_name, process_name_text),
-        ('patient_last_name', processor.get_roi_patient_last_name, process_name_text),
-        ('patient_dob', processor.get_roi_patient_dob, process_date_text),
-        ('endoscope_type', processor.get_roi_endoscope_type, process_general_text),
-        ('endoscope_sn', processor.get_roi_endoscopy_sn, process_general_text),
-    ]
-
-    # Extract and post-process text for each ROI
-    for roi_name, roi_function, post_process in rois_with_postprocessing:
-        # Get the ROI dictionary
-        roi = roi_function()
-        
-        # Check if the ROI has values
-        
-        if roi_values_valid(roi):
-            x, y, w, h = roi['x'], roi['y'], roi['width'], roi['height']
-            
-            # Get white image with original shape and just the roi remaining
-            roi_image = crop_and_insert(inverted, x,y,h,w)
-
-            # OCR configuration: Recognize white text on black background without corrections
-            config = '--psm 10 -c tessedit_char_whitelist=0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-üöäÜÖÄß'
-
-            # Use pytesseract to do OCR on the preprocessed ROI
-            text = pytesseract.image_to_string(roi_image, config=config).strip()
-
-            # Post-process extracted text
-            processed_text = post_process(text)
-            
-            extracted_texts[roi_name] = processed_text
-
-        else:
-            pass
-
-    return extracted_texts
-
-def get_most_frequent_values(rois_texts: Dict[str, List[str]]) -> Dict[str, str]:
-    """
-    Given a dictionary of ROIs and their corresponding texts, returns a dictionary of the most frequent text for each ROI.
-
-    Args:
-        rois_texts: A dictionary where the keys are the names of the ROIs and the values are lists of texts.
-
-    Returns:
-        A dictionary where the keys are the names of the ROIs and the values are the most frequent text for each ROI.
-    """
-    most_frequent = {}
-    for key in rois_texts.keys():
-        counter = Counter([text for text in rois_texts[key] if text])
-        most_frequent[key], _ = counter.most_common(1)[0] if counter else (None, None)
-    return most_frequent
-
-def process_video(video_path, processor):
-    """
-    Processes a video file by extracting text from regions of interest (ROIs) in each frame.
-
-    Args:
-        video_path (str): The path to the video file to process.
-        processor (OCRProcessor): An instance of the OCRProcessor class that defines the ROIs to extract text from.
-
-    Returns:
-        dict: A dictionary containing the most frequent text values extracted from each ROI.
-    """
-    # Create a temporary directory to store frames
-    with TemporaryDirectory() as temp_dir:
-        # Capture the video
-        video = cv2.VideoCapture(video_path)
-        success, frame_number = True, 0
-        rois_texts = {roi_name: [] for roi_name in processor.get_rois().keys()}
-        frames_for_mean_extraction = 0
-
-        while success:
-            success, frame = video.read()
-
-            # Check if this is the 200th frame
-            if frame_number % 1000 == 0 and success:
-                frame_path = os.path.join(temp_dir, f"frame_{frame_number}.jpg")
-                cv2.imwrite(frame_path, frame)  # Save the frame as a JPEG file
-                # cv2.imwrite(f"_tmp/frame_{frame_number}.jpg", frame)
-                
-                # Extract text from ROIs
-                extracted_texts = extract_text_from_rois(frame_path, processor)
-                
-                # Store the extracted text from each ROI
-                for key, text in extracted_texts.items():
-                    rois_texts[key].append(text)
-                frames_for_mean_extraction += 1
-
-            frame_number += 1
-
-            if frames_for_mean_extraction >= N_FRAMES_MEAN_OCR: break
-
-        # Release the video capture object
-        video.release()
-
-        # Get the most frequent values for each ROI
-        return get_most_frequent_values(rois_texts)
-

endoreg_db/utils/uuid.py

@@ -1,4 +0,0 @@
-import uuid
-
-def get_uuid():
-    return uuid.uuid4()

endoreg_db/utils/video_metadata.py

@@ -1,87 +0,0 @@
-# import cv2
-from pathlib import Path
-import datetime
-
-def _extract_metadata_from_filename(filepath:Path, time_format = None): # deprecated
-    """
-    Extracts metadata from a video filename.
-
-    Args:
-        filepath (Path): The path to the video file.
-        time_format (str, optional): The format of the date in the filename. Defaults to None.
-
-    Returns:
-        dict: A dictionary containing the extracted metadata.
-
-    Example return value:
-        {
-            'examination_date': '2021-03-03',
-            'patient_last_name': 'Doe',
-            'patient_first_name': 'John',
-            'patient_dob': '1990-01-01'
-        }
-    """
-    # get filename without suffix
-    filename = filepath.stem    
-
-    if not time_format:
-        time_format = "%d.%m.%Y"
-
-    _info = [_.strip() for _ in filename.split("_")]
-
-    if len(_info) == 4:
-        examination_date, last_name, first_name, birthdate = _info
-    
-    else:
-        examination_date, last_name, first_name, birthdate = None, None, None, None
-
-    metadata = {}
-    metadata['examination_date'] = examination_date
-    metadata['patient_last_name'] = last_name
-    metadata['patient_first_name'] = first_name
-    metadata['patient_dob'] = birthdate
-
-    try:
-        metadata['examination_date'] = datetime.datetime.strptime(examination_date, time_format).date()
-        metadata['examination_date'] = metadata['examination_date'].strftime("%Y-%m-%d")
-    except:
-        metadata['examination_date'] = None
-
-    try:
-        metadata['patient_dob'] = datetime.datetime.strptime(birthdate, time_format).date()
-        metadata['patient_dob'] = metadata['patient_dob'].strftime("%Y-%m-%d")
-    except:
-        metadata['patient_dob'] = None
-
-    return metadata
-
-def _get_video_metadata(file_path): # Deprecated
-    """
-    Returns the framerate, dimensions, and duration of a video file.
-
-    Parameters:
-    file_path (str): The path to the video file.
-
-    Returns:
-    tuple: A tuple containing the framerate (float), dimensions (tuple of ints), and duration (float) of the video.
-    """
-    # Open the video file
-    video = cv2.VideoCapture(file_path)
-
-    # Get the video framerate
-    fps = video.get(cv2.CAP_PROP_FPS)
-
-    # Get the video dimensions
-    width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
-    height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
-    dimensions = (width, height)
-
-    # Get the video duration
-    frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
-    duration = frame_count / fps
-
-    # Release the video file
-    video.release()
-
-    # Return the metadata
-    return fps, dimensions, duration

endoreg_db-0.5.2.dist-info/METADATA

@@ -1,27 +0,0 @@
-Metadata-Version: 2.3
-Name: endoreg-db
-Version: 0.5.2
-Summary: EndoReg Db Django App
-Requires-Python: >=3.11
-Requires-Dist: agl-report-reader>=0.4.0
-Requires-Dist: django-bootstrap5>=24.3
-Requires-Dist: django>=5.1.3
-Requires-Dist: djangorestframework>=3.15.2
-Requires-Dist: ffmpeg-python>=0.2.0
-Requires-Dist: opencv-python>=4.10.0.84
-Requires-Dist: pandas>=2.2.3
-Requires-Dist: pillow>=11.0.0
-Requires-Dist: pytesseract>=0.3.13
-Requires-Dist: pyyaml>=6.0.2
-Requires-Dist: scipy>=1.14.1
-Requires-Dist: tqdm>=4.67.0
-Description-Content-Type: text/markdown
-
-# endoreg-db
-endoreg-db
-
-
-## Poetry add pypi token
-poetry config pypi-token.pypi your-api-token
-
-