endoreg-db 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl

endoreg_db/utils/file_operations.py

@@ -0,0 +1,30 @@
+import shutil
+from pathlib import Path
+from endoreg_db.utils.uuid import get_uuid
+
+def get_uuid_filename(file:Path) -> tuple[str, str]:
+    """
+    Returns a new filename with a uuid
+    """
+    # Get the file extension
+    file_extension = file.suffix
+    # Generate a new file name
+    uuid = get_uuid()
+    new_file_name = f"{uuid}{file_extension}"
+    return new_file_name, uuid
+    
+def rename_file_uuid(old_file:Path):
+    """
+    Rename a file by assigning a uuid while preserving file extension. Returns new filepath and uuid
+    """
+    # Get the file extension
+    file_extension = old_file.suffix
+    # Generate a new file name
+    uuid = get_uuid()
+    new_file_name = f"{uuid}{file_extension}"
+
+    # Rename the file
+    new_file = old_file.with_name(new_file_name)
+    shutil.move(old_file.resolve().as_posix(), new_file.resolve().as_posix())
+
+    return new_file, uuid

endoreg_db/utils/hashs.py

@@ -0,0 +1,152 @@
+import hashlib
+from pathlib import Path
+from datetime import datetime, date
+
+import os
+
+SALT = os.getenv("DJANGO_SALT", "default_salt")
+
+
+def get_video_hash(video_path):
+    """
+    Get the hash of a video file.
+    """
+    # Open the video file in read-binary mode:
+    with open(video_path, "rb") as f:
+        # Create the hash object, passing in the video contents for hashing:
+        hash_object = hashlib.sha256(f.read())
+        # Get the hexadecimal representation of the hash
+        video_hash = hash_object.hexdigest()
+        assert len(video_hash) <= 255, "Hash length exceeds 255 characters"
+
+    return video_hash
+
+
+def get_pdf_hash(pdf_path: Path):
+    """
+    Get the hash of a pdf file.
+    """
+    pdf_hash = None
+
+    # Open the file in binary mode and read its contents
+    with open(pdf_path, "rb") as f:
+        pdf_contents = f.read()
+        # Create a hash object using SHA-256 algorithm
+
+    hash_object = hashlib.sha256(pdf_contents, usedforsecurity=False)
+    # Get the hexadecimal representation of the hash
+    pdf_hash = hash_object.hexdigest()
+    assert len(pdf_hash) <= 255, "Hash length exceeds 255 characters"
+
+    return pdf_hash
+
+
+def _get_date_hash_string(date_obj: date) -> str:
+    # if date is datetime object, convert to date
+    if isinstance(date_obj, datetime):
+        # warnings.warn("Date is a datetime object. Converting to date object.")
+        date_obj = date_obj.date()
+    elif isinstance(date_obj, str):
+        # warnings.warn(f"Date is a string ({date_obj}). Converting to date object.")
+        date_obj = datetime.strptime(date_obj, "%Y-%m-%d").date()
+
+    assert isinstance(date_obj, date), "Date must be a date object"
+    # if date is 1900-01-01, make it an empty string
+    if date_obj == date(1900, 1, 1):
+        date_str = ""
+    else:
+        date_str = date_obj.strftime("%Y-%m-%d")
+
+    return date_str
+
+
+def get_hash_string(
+    first_name: str = "",
+    last_name: str = "",
+    dob: date = date(1900, 1, 1),
+    center_name: str = "",
+    examination_date: date = date(1900, 1, 1),
+    endoscope_sn: str = "",
+    salt: str = "",
+):
+    """
+    Get the string to be hashed for a patient's first name, last name, date of birth, examination date, and endoscope serial number.
+    """
+    if not salt:
+        salt = SALT
+
+    examination_date_str = _get_date_hash_string(examination_date)
+    dob_str = _get_date_hash_string(dob)
+
+    # Concatenate the patient's first name, last name, date of birth, examination date, endoscope serial number, and salt:
+    hash_str = f"{first_name}{last_name}{dob_str}{center_name}{dob_str}{examination_date_str}{endoscope_sn}{salt}"
+    return hash_str
+
+
+def get_patient_hash(
+    first_name: str, last_name: str, dob: date, center: str, salt: str = ""
+):
+    """
+    Get the hash of a patient's first name, last name, and date of birth.
+    """
+    # Concatenate the patient's first name, last name, date of birth, and salt:
+    hash_str = get_hash_string(
+        first_name=first_name,
+        last_name=last_name,
+        dob=dob,
+        center_name=center,
+        salt=salt,
+    )
+    # Create a hash object using SHA-256 algorithm
+    hash_object = hashlib.sha256(hash_str.encode())
+    # Get the hexadecimal representation of the hash
+    patient_hash = hash_object.hexdigest()
+
+    return patient_hash
+
+
+def get_patient_examination_hash(
+    first_name: str,
+    last_name: str,
+    dob: date,
+    center: str,
+    examination_date: date,
+    salt: str = "",
+):
+    """
+    Get the hash of a patient's first name, last name, date of birth, and examination date.
+    """
+    # Concatenate the patient's first name, last name, date of birth, examination date, and salt:
+    hash_str = get_hash_string(
+        first_name=first_name,
+        last_name=last_name,
+        center_name=center,
+        dob=dob,
+        examination_date=examination_date,
+        salt=salt,
+    )
+    # Create a hash object using SHA-256 algorithm
+    hash_object = hashlib.sha256(hash_str.encode())
+    # Get the hexadecimal representation of the hash
+    patient_examination_hash = hash_object.hexdigest()
+
+    return patient_examination_hash
+
+
+def get_examiner_hash(first_name, last_name, center_name, salt):
+    """
+    Get the hash of an examiner's first name, last name, and center name.
+    """
+    # Concatenate the examiner's first name, last name, center name, and salt:
+    hash_str = get_hash_string(
+        first_name=first_name,
+        last_name=last_name,
+        center_name=center_name,
+        salt=salt,
+    )
+    # Create a hash object using SHA-256 algorithm
+    hash_object = hashlib.sha256(hash_str.encode())
+    # Get the hexadecimal representation of the hash
+    examiner_hash = hash_object.hexdigest()
+
+    return examiner_hash

endoreg_db/utils/legacy_ocr.py

@@ -0,0 +1,201 @@
+import pytesseract
+# import cv2
+from endoreg_db.models import EndoscopyProcessor
+import os
+from collections import Counter
+from tempfile import TemporaryDirectory
+import re
+from datetime import datetime
+from typing import Dict, List
+import numpy as np
+
+N_FRAMES_MEAN_OCR = 2
+
+# Helper function to process date strings
+def process_date_text(date_text):
+    """
+    Processes a string of text that represents a date and returns a datetime.date object.
+
+    Args:
+        date_text (str): A string of text that represents a date.
+
+    Returns:
+        datetime.date: A datetime.date object representing the parsed date, or None if the text cannot be parsed.
+    """
+    try:
+        # Remove any non-digit characters
+        date_text_clean = re.sub(r'\D', '', date_text)
+        # Reformat to 'ddmmyyyy' if necessary
+        if len(date_text_clean) == 8:
+            return datetime.strptime(date_text_clean, "%d%m%Y").date()
+        elif len(date_text_clean) == 14:
+            return datetime.strptime(date_text_clean, "%d%m%Y%H%M%S").date()
+    except ValueError:
+        # Return None if the text cannot be parsed into a date
+        # set date to 1/1/1900
+        return datetime.strptime("01011900", "%d%m%Y").date()
+
+# Helper function to process patient names
+def process_name_text(name_text):
+    """
+    Remove all numbers, punctuation, and whitespace from a string of text and return the result.
+    """
+    name = re.sub(r'[0-9!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~\s]+', '', name_text).strip()
+    # capitalize first letter of each word
+    name = ' '.join([word.capitalize() for word in name.split()])
+    return name
+
+
+# Helper function to process endoscope type text
+def process_general_text(endoscope_text):
+    """
+    This function takes in a string of text from an endoscope and returns a cleaned version of the text.
+    """
+    return ' '.join(endoscope_text.split())
+
+def roi_values_valid(roi):
+    """
+    Check if all values in an ROI dictionary are valid (>=0).
+    """
+    return all([value >= 0 for value in roi.values()])
+
+# Function to extract text from ROIs
+def extract_text_from_rois(image_path, processor:EndoscopyProcessor):
+    """
+    Extracts text from regions of interest (ROIs) in an image using OCR.
+
+    Args:
+        image_path (str): The path to the image file.
+        processor (EndoscopyProcessor): An instance of the EndoscopyProcessor class.
+
+    Returns:
+        dict: A dictionary containing the extracted text for each ROI.
+    """
+    # Read the image using OpenCV
+    image = cv2.imread(image_path)
+
+    # Initialize the dictionary to hold the extracted text
+    extracted_texts = {}
+
+    # Define your ROIs and their corresponding post-processing functions in tuples
+    rois_with_postprocessing = [
+        ('examination_date', processor.get_roi_examination_date, process_date_text),
+        ("patient_first_name", processor.get_roi_patient_first_name, process_name_text),
+        ('patient_last_name', processor.get_roi_patient_last_name, process_name_text),
+        ('patient_dob', processor.get_roi_patient_dob, process_date_text),
+        ('endoscope_type', processor.get_roi_endoscope_type, process_general_text),
+        ('endoscope_sn', processor.get_roi_endoscopy_sn, process_general_text),
+    ]
+
+    # Extract and post-process text for each ROI
+    for roi_name, roi_function, post_process in rois_with_postprocessing:
+        # Get the ROI dictionary
+        roi = roi_function()
+        
+        # Check if the ROI has values
+        
+        if roi_values_valid(roi):
+            # Crop the image to the ROI
+            x, y, w, h = roi['x'], roi['y'], roi['width'], roi['height']
+            roi_cropped = image[y:y+h, x:x+w]
+            # Convert to grayscale
+            gray = cv2.cvtColor(roi_cropped, cv2.COLOR_BGR2GRAY)
+
+            # Invert colors for white text on black background
+            gray = cv2.bitwise_not(gray)
+
+            # Binarize the image - using Otsu's method
+            _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+
+            # Dilate the image to improve the contour of the pixelated text
+            kernel = np.ones((2,2), np.uint8)
+            dilation = cv2.dilate(binary, kernel, iterations=1)
+
+            # OCR configuration: Recognize white text on black background without corrections
+            config = '--psm 10 -c tessedit_char_whitelist=0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-üöäÜÖÄß'
+
+            # Use pytesseract to do OCR on the preprocessed ROI
+            text = pytesseract.image_to_string(dilation, config=config).strip()
+
+            # Post-process extracted text
+            processed_text = post_process(text)
+            # processed_text = text
+            
+            # Store the processed text in the dictionary
+            extracted_texts[roi_name] = processed_text
+
+        else:
+            pass
+            # ic(roi_name)
+            # ic(roi)
+            # ic("No values for this ROI")
+
+    return extracted_texts
+
+
+def get_most_frequent_values(rois_texts: Dict[str, List[str]]) -> Dict[str, str]:
+    """
+    Given a dictionary of ROIs and their corresponding texts, returns a dictionary of the most frequent text for each ROI.
+
+    Args:
+        rois_texts: A dictionary where the keys are the names of the ROIs and the values are lists of texts.
+
+    Returns:
+        A dictionary where the keys are the names of the ROIs and the values are the most frequent text for each ROI.
+    """
+    most_frequent = {}
+    for key in rois_texts.keys():
+        counter = Counter([text for text in rois_texts[key] if text])
+        # ic(key)
+        # ic(counter)
+        most_frequent[key], _ = counter.most_common(1)[0] if counter else (None, None)
+    return most_frequent
+
+def process_video(video_path, processor):
+    """
+    Processes a video file by extracting text from regions of interest (ROIs) in each frame.
+
+    Args:
+        video_path (str): The path to the video file to process.
+        processor (OCRProcessor): An instance of the OCRProcessor class that defines the ROIs to extract text from.
+
+    Returns:
+        dict: A dictionary containing the most frequent text values extracted from each ROI.
+    """
+    # Create a temporary directory to store frames
+    with TemporaryDirectory() as temp_dir:
+        # ic(temp_dir)
+        # Capture the video
+        video = cv2.VideoCapture(video_path)
+        success, frame_number = True, 0
+        rois_texts = {roi_name: [] for roi_name in processor.get_rois().keys()}
+        frames_for_mean_extraction = 0
+
+        while success:
+            success, frame = video.read()
+
+            # Check if this is the 200th frame
+            if frame_number % 1000 == 0 and success:
+                frame_path = os.path.join(temp_dir, f"frame_{frame_number}.jpg")
+                cv2.imwrite(frame_path, frame)  # Save the frame as a JPEG file
+                # cv2.imwrite(f"_tmp/frame_{frame_number}.jpg", frame)
+                
+                # Extract text from ROIs
+                extracted_texts = extract_text_from_rois(frame_path, processor)
+                # ic(extracted_texts)
+                
+                # Store the extracted text from each ROI
+                for key, text in extracted_texts.items():
+                    rois_texts[key].append(text)
+                frames_for_mean_extraction += 1
+
+            frame_number += 1
+
+            if frames_for_mean_extraction >= N_FRAMES_MEAN_OCR: break
+
+        # Release the video capture object
+        video.release()
+
+        # Get the most frequent values for each ROI
+        return get_most_frequent_values(rois_texts)
+

endoreg_db/utils/names.py

@@ -0,0 +1,74 @@
+# Use faker library to generate fake names by gender
+# Use german names by default
+
+from faker import Faker
+import gender_guesser.detector as gender_detector
+
+
+def create_mock_examiner_name() -> tuple[str, str]:
+    """
+    Generate a mock examiner's name using the Faker library.
+    This function creates a tuple with a first name and a last name for a mock examiner. It utilizes the "de_DE" locale for generating German names.
+    Returns:
+        tuple[str, str]: A tuple containing the first name and the last name.
+    """
+
+    fake = Faker("de_DE")
+    first_name = fake.first_name()
+    last_name = fake.last_name()
+    return first_name, last_name
+
+
+def create_mock_patient_name(gender: str) -> tuple[str, str]:
+    """
+    Generate a mock patient's name based on the provided gender using the Faker library.
+    This function creates a tuple with a first name and a last name for a mock patient. It utilizes the "de_DE" locale for generating German names. When the input gender string is checked:
+    - If it contains "male", a male name is generated.
+    - If it contains "female", a female name is generated.
+    - Otherwise, a generic name is generated without considering gender.
+    Parameters:
+        gender (str): A string indicating the gender to be used for generating the name.
+    Returns:
+        tuple[str, str]: A tuple containing the first name and the last name.
+    """
+
+    fake = Faker("de_DE")
+
+    if "male" in gender.lower():
+        gender = "male"
+    elif "female" in gender.lower():
+        gender = "female"
+
+    if gender == "male":
+        first_name = fake.first_name_male()
+        last_name = fake.last_name_male()
+
+    elif gender == "female":
+        first_name = fake.first_name_female()
+        last_name = fake.last_name_female()
+
+    else:
+        first_name = fake.first_name()
+        last_name = fake.last_name()
+
+    return first_name, last_name
+
+
+def guess_name_gender(name: str) -> str:
+    """
+    Guesses the gender for a given name by using a gender detector and retrieving the corresponding Gender model instance.
+    Parameters:
+        name (str): The name for which the gender is to be determined.
+    Returns:
+        Gender: The Gender object corresponding to the detected gender name.
+    Raises:
+        Gender.DoesNotExist: If no Gender object matching the detected gender is found.
+        Exception: For any other exceptions that occur during gender detection or database lookup.
+    """
+
+    from endoreg_db.models import Gender
+
+    detector = gender_detector.Detector(case_sensitive=False)
+    gender_name = detector.get_gender(name)
+    gender = Gender.objects.get(name=gender_name)
+    return gender

endoreg_db/utils/ocr.py

@@ -0,0 +1,190 @@
+import pytesseract
+from PIL import Image, ImageOps, ImageFilter
+import os
+from collections import Counter
+from tempfile import TemporaryDirectory
+import re
+from datetime import datetime
+from typing import Dict, List
+import numpy as np
+from endoreg_db.utils.cropping import crop_and_insert
+
+
+
+N_FRAMES_MEAN_OCR = 2
+
+# Helper function to process date strings
+def process_date_text(date_text):
+    """
+    Processes a string of text that represents a date and returns a datetime.date object.
+
+    Args:
+        date_text (str): A string of text that represents a date.
+
+    Returns:
+        datetime.date: A datetime.date object representing the parsed date, or None if the text cannot be parsed.
+    """
+    try:
+        # Remove any non-digit characters
+        date_text_clean = re.sub(r'\D', '', date_text)
+        # Reformat to 'ddmmyyyy' if necessary
+        if len(date_text_clean) == 8:
+            return datetime.strptime(date_text_clean, "%d%m%Y").date()
+        elif len(date_text_clean) == 14:
+            return datetime.strptime(date_text_clean, "%d%m%Y%H%M%S").date()
+    except ValueError:
+        # Return None if the text cannot be parsed into a date
+        # set date to 1/1/1900
+        return datetime.strptime("01011900", "%d%m%Y").date()
+
+# Helper function to process patient names
+def process_name_text(name_text):
+    """
+    Remove all numbers, punctuation, and whitespace from a string of text and return the result.
+    """
+    name = re.sub(r'[0-9!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~\s]+', '', name_text).strip()
+    # capitalize first letter of each word
+    name = ' '.join([word.capitalize() for word in name.split()])
+    return name
+
+
+# Helper function to process endoscope type text
+def process_general_text(endoscope_text):
+    """
+    This function takes in a string of text from an endoscope and returns a cleaned version of the text.
+    """
+    return ' '.join(endoscope_text.split())
+
+def roi_values_valid(roi):
+    """
+    Check if all values in an ROI dictionary are valid (>=0).
+    """
+    return all([value >= 0 for value in roi.values()])
+
+# Function to extract text from ROIs
+def extract_text_from_rois(image_path, processor):
+    """
+    Extracts text from regions of interest (ROIs) in an image using OCR.
+
+    Args:
+        image_path (str): The path to the image file.
+        processor (EndoscopyProcessor): An instance of the EndoscopyProcessor class.
+
+    Returns:
+        dict: A dictionary containing the extracted text for each ROI.
+    """
+    # Read the image using Pillow
+    image = Image.open(image_path)
+    image_dimensions = image.size # (width, height)
+
+    ####### Adjust Image #######
+    # Convert to grayscale
+    gray = image.convert('L')
+
+    # Invert colors for white text on black background
+    inverted = ImageOps.invert(gray)
+
+    # Initialize the dictionary to hold the extracted text
+    extracted_texts = {}
+
+    # Define your ROIs and their corresponding post-processing functions in tuples
+    rois_with_postprocessing = [
+        ('examination_date', processor.get_roi_examination_date, process_date_text),
+        ("patient_first_name", processor.get_roi_patient_first_name, process_name_text),
+        ('patient_last_name', processor.get_roi_patient_last_name, process_name_text),
+        ('patient_dob', processor.get_roi_patient_dob, process_date_text),
+        ('endoscope_type', processor.get_roi_endoscope_type, process_general_text),
+        ('endoscope_sn', processor.get_roi_endoscopy_sn, process_general_text),
+    ]
+
+    # Extract and post-process text for each ROI
+    for roi_name, roi_function, post_process in rois_with_postprocessing:
+        # Get the ROI dictionary
+        roi = roi_function()
+        
+        # Check if the ROI has values
+        
+        if roi_values_valid(roi):
+            x, y, w, h = roi['x'], roi['y'], roi['width'], roi['height']
+            
+            # Get white image with original shape and just the roi remaining
+            roi_image = crop_and_insert(inverted, x,y,h,w)
+
+            # OCR configuration: Recognize white text on black background without corrections
+            config = '--psm 10 -c tessedit_char_whitelist=0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-üöäÜÖÄß'
+
+            # Use pytesseract to do OCR on the preprocessed ROI
+            text = pytesseract.image_to_string(roi_image, config=config).strip()
+
+            # Post-process extracted text
+            processed_text = post_process(text)
+            
+            extracted_texts[roi_name] = processed_text
+
+        else:
+            pass
+
+    return extracted_texts
+
+def get_most_frequent_values(rois_texts: Dict[str, List[str]]) -> Dict[str, str]:
+    """
+    Given a dictionary of ROIs and their corresponding texts, returns a dictionary of the most frequent text for each ROI.
+
+    Args:
+        rois_texts: A dictionary where the keys are the names of the ROIs and the values are lists of texts.
+
+    Returns:
+        A dictionary where the keys are the names of the ROIs and the values are the most frequent text for each ROI.
+    """
+    most_frequent = {}
+    for key in rois_texts.keys():
+        counter = Counter([text for text in rois_texts[key] if text])
+        most_frequent[key], _ = counter.most_common(1)[0] if counter else (None, None)
+    return most_frequent
+
+def process_video(video_path, processor):
+    """
+    Processes a video file by extracting text from regions of interest (ROIs) in each frame.
+
+    Args:
+        video_path (str): The path to the video file to process.
+        processor (OCRProcessor): An instance of the OCRProcessor class that defines the ROIs to extract text from.
+
+    Returns:
+        dict: A dictionary containing the most frequent text values extracted from each ROI.
+    """
+    # Create a temporary directory to store frames
+    with TemporaryDirectory() as temp_dir:
+        # Capture the video
+        video = cv2.VideoCapture(video_path)
+        success, frame_number = True, 0
+        rois_texts = {roi_name: [] for roi_name in processor.get_rois().keys()}
+        frames_for_mean_extraction = 0
+
+        while success:
+            success, frame = video.read()
+
+            # Check if this is the 200th frame
+            if frame_number % 1000 == 0 and success:
+                frame_path = os.path.join(temp_dir, f"frame_{frame_number}.jpg")
+                cv2.imwrite(frame_path, frame)  # Save the frame as a JPEG file
+                # cv2.imwrite(f"_tmp/frame_{frame_number}.jpg", frame)
+                
+                # Extract text from ROIs
+                extracted_texts = extract_text_from_rois(frame_path, processor)
+                
+                # Store the extracted text from each ROI
+                for key, text in extracted_texts.items():
+                    rois_texts[key].append(text)
+                frames_for_mean_extraction += 1
+
+            frame_number += 1
+
+            if frames_for_mean_extraction >= N_FRAMES_MEAN_OCR: break
+
+        # Release the video capture object
+        video.release()
+
+        # Get the most frequent values for each ROI
+        return get_most_frequent_values(rois_texts)
+

endoreg_db/utils/parse_and_generate_yaml.py

@@ -0,0 +1,46 @@
+import yaml
+from pathlib import Path
+
+# get this files path
+file_path = Path(__file__)
+module_root = file_path.parent.parent
+data_dir = module_root / "data"
+
+
+def collect_center_names(
+):
+    input_file_path = data_dir / "center/data.yaml"
+    fist_name_dir = data_dir / "names_first"
+    last_name_dir = data_dir / "names_last"
+    # Load the input YAML file
+    with open(input_file_path, 'r', encoding='utf-8') as file:
+        data = yaml.safe_load(file)
+    
+    # Containers for first and last names
+    first_names_set = set()
+    last_names_set = set()
+    
+    # Extract first and last names from the YAML data
+    for entry in data:
+        fields = entry.get('fields', {})
+        first_names_set.update(fields.get('first_names', []))
+        last_names_set.update(fields.get('last_names', []))
+    
+    # Create a list of dictionaries for first and last names
+    first_names_data = [
+        {"model": "endoreg_db.first_name", "fields": {"name": name}}
+        for name in sorted(first_names_set)
+    ]
+    last_names_data = [
+        {"model": "endoreg_db.last_name", "fields": {"name": name}}
+        for name in sorted(last_names_set)
+    ]
+    
+    # Write the data to separate YAML files
+    with open(fist_name_dir/"first_names.yaml", "w", encoding='utf-8') as first_file:
+        yaml.dump(first_names_data, first_file, allow_unicode=True, sort_keys=False)
+    
+    with open(last_name_dir/"last_names.yaml", "w", encoding='utf-8') as last_file:
+        yaml.dump(last_names_data, last_file, allow_unicode=True, sort_keys=False)
+
+    # print("Generated first_names.yaml and last_names.yaml successfully.")