endoreg-db 0.3.6__py3-none-any.whl → 0.8.6.1__py3-none-any.whl

endoreg_db/utils/ocr.py

@@ -1,197 +1,190 @@
-import pytesseract
-from PIL import Image, ImageOps, ImageFilter
-import os
-from collections import Counter
-from tempfile import TemporaryDirectory
-import re
-from datetime import datetime
-from typing import Dict, List
-from icecream import ic
-import numpy as np
-from endoreg_db.utils.cropping import crop_and_insert
-
-
-
-N_FRAMES_MEAN_OCR = 2
-
-# Helper function to process date strings
-def process_date_text(date_text):
-    """
-    Processes a string of text that represents a date and returns a datetime.date object.
-
-    Args:
-        date_text (str): A string of text that represents a date.
-
-    Returns:
-        datetime.date: A datetime.date object representing the parsed date, or None if the text cannot be parsed.
-    """
-    try:
-        # Remove any non-digit characters
-        date_text_clean = re.sub(r'\D', '', date_text)
-        # Reformat to 'ddmmyyyy' if necessary
-        if len(date_text_clean) == 8:
-            return datetime.strptime(date_text_clean, "%d%m%Y").date()
-        elif len(date_text_clean) == 14:
-            return datetime.strptime(date_text_clean, "%d%m%Y%H%M%S").date()
-    except ValueError:
-        # Return None if the text cannot be parsed into a date
-        # set date to 1/1/1900
-        return datetime.strptime("01011900", "%d%m%Y").date()
-
-# Helper function to process patient names
-def process_name_text(name_text):
-    """
-    Remove all numbers, punctuation, and whitespace from a string of text and return the result.
-    """
-    name = re.sub(r'[0-9!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~\s]+', '', name_text).strip()
-    # capitalize first letter of each word
-    name = ' '.join([word.capitalize() for word in name.split()])
-    return name
-
-
-# Helper function to process endoscope type text
-def process_general_text(endoscope_text):
-    """
-    This function takes in a string of text from an endoscope and returns a cleaned version of the text.
-    """
-    return ' '.join(endoscope_text.split())
-
-def roi_values_valid(roi):
-    """
-    Check if all values in an ROI dictionary are valid (>=0).
-    """
-    return all([value >= 0 for value in roi.values()])
-
-# Function to extract text from ROIs
-def extract_text_from_rois(image_path, processor):
-    """
-    Extracts text from regions of interest (ROIs) in an image using OCR.
-
-    Args:
-        image_path (str): The path to the image file.
-        processor (EndoscopyProcessor): An instance of the EndoscopyProcessor class.
-
-    Returns:
-        dict: A dictionary containing the extracted text for each ROI.
-    """
-    # Read the image using Pillow
-    image = Image.open(image_path)
-    image_dimensions = image.size # (width, height)
-
-    ####### Adjust Image #######
-    # Convert to grayscale
-    gray = image.convert('L')
-
-    # Invert colors for white text on black background
-    inverted = ImageOps.invert(gray)
-
-    # Initialize the dictionary to hold the extracted text
-    extracted_texts = {}
-
-    # Define your ROIs and their corresponding post-processing functions in tuples
-    rois_with_postprocessing = [
-        ('examination_date', processor.get_roi_examination_date, process_date_text),
-        ("patient_first_name", processor.get_roi_patient_first_name, process_name_text),
-        ('patient_last_name', processor.get_roi_patient_last_name, process_name_text),
-        ('patient_dob', processor.get_roi_patient_dob, process_date_text),
-        ('endoscope_type', processor.get_roi_endoscope_type, process_general_text),
-        ('endoscope_sn', processor.get_roi_endoscopy_sn, process_general_text),
-    ]
-
-    # Extract and post-process text for each ROI
-    for roi_name, roi_function, post_process in rois_with_postprocessing:
-        # Get the ROI dictionary
-        roi = roi_function()
-        
-        # Check if the ROI has values
-        
-        if roi_values_valid(roi):
-            x, y, w, h = roi['x'], roi['y'], roi['width'], roi['height']
-            
-            # Get white image with original shape and just the roi remaining
-            roi_image = crop_and_insert(inverted, x,y,h,w)
-
-            # OCR configuration: Recognize white text on black background without corrections
-            config = '--psm 10 -c tessedit_char_whitelist=0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-üöäÜÖÄß'
-
-            # Use pytesseract to do OCR on the preprocessed ROI
-            text = pytesseract.image_to_string(roi_image, config=config).strip()
-
-            # Post-process extracted text
-            processed_text = post_process(text)
-            
-            extracted_texts[roi_name] = processed_text
-
-        else:
-            ic(roi_name)
-            ic(roi)
-            ic("No values for this ROI")
-
-    return extracted_texts
-
-def get_most_frequent_values(rois_texts: Dict[str, List[str]]) -> Dict[str, str]:
-    """
-    Given a dictionary of ROIs and their corresponding texts, returns a dictionary of the most frequent text for each ROI.
-
-    Args:
-        rois_texts: A dictionary where the keys are the names of the ROIs and the values are lists of texts.
-
-    Returns:
-        A dictionary where the keys are the names of the ROIs and the values are the most frequent text for each ROI.
-    """
-    most_frequent = {}
-    for key in rois_texts.keys():
-        counter = Counter([text for text in rois_texts[key] if text])
-        ic(key)
-        ic(counter)
-        most_frequent[key], _ = counter.most_common(1)[0] if counter else (None, None)
-    return most_frequent
-
-def process_video(video_path, processor):
-    """
-    Processes a video file by extracting text from regions of interest (ROIs) in each frame.
-
-    Args:
-        video_path (str): The path to the video file to process.
-        processor (OCRProcessor): An instance of the OCRProcessor class that defines the ROIs to extract text from.
-
-    Returns:
-        dict: A dictionary containing the most frequent text values extracted from each ROI.
-    """
-    # Create a temporary directory to store frames
-    with TemporaryDirectory() as temp_dir:
-        ic(temp_dir)
-        # Capture the video
-        video = cv2.VideoCapture(video_path)
-        success, frame_number = True, 0
-        rois_texts = {roi_name: [] for roi_name in processor.get_rois().keys()}
-        frames_for_mean_extraction = 0
-
-        while success:
-            success, frame = video.read()
-
-            # Check if this is the 200th frame
-            if frame_number % 1000 == 0 and success:
-                frame_path = os.path.join(temp_dir, f"frame_{frame_number}.jpg")
-                cv2.imwrite(frame_path, frame)  # Save the frame as a JPEG file
-                # cv2.imwrite(f"_tmp/frame_{frame_number}.jpg", frame)
-                
-                # Extract text from ROIs
-                extracted_texts = extract_text_from_rois(frame_path, processor)
-                ic(extracted_texts)
-                
-                # Store the extracted text from each ROI
-                for key, text in extracted_texts.items():
-                    rois_texts[key].append(text)
-                frames_for_mean_extraction += 1
-
-            frame_number += 1
-
-            if frames_for_mean_extraction >= N_FRAMES_MEAN_OCR: break
-
-        # Release the video capture object
-        video.release()
-
-        # Get the most frequent values for each ROI
-        return get_most_frequent_values(rois_texts)
-
+import pytesseract
+from PIL import Image, ImageOps, ImageFilter
+import os
+from collections import Counter
+from tempfile import TemporaryDirectory
+import re
+from datetime import datetime
+from typing import Dict, List
+import numpy as np
+from endoreg_db.utils.cropping import crop_and_insert
+
+
+
+N_FRAMES_MEAN_OCR = 2
+
+# Helper function to process date strings
+def process_date_text(date_text):
+    """
+    Processes a string of text that represents a date and returns a datetime.date object.
+
+    Args:
+        date_text (str): A string of text that represents a date.
+
+    Returns:
+        datetime.date: A datetime.date object representing the parsed date, or None if the text cannot be parsed.
+    """
+    try:
+        # Remove any non-digit characters
+        date_text_clean = re.sub(r'\D', '', date_text)
+        # Reformat to 'ddmmyyyy' if necessary
+        if len(date_text_clean) == 8:
+            return datetime.strptime(date_text_clean, "%d%m%Y").date()
+        elif len(date_text_clean) == 14:
+            return datetime.strptime(date_text_clean, "%d%m%Y%H%M%S").date()
+    except ValueError:
+        # Return None if the text cannot be parsed into a date
+        # set date to 1/1/1900
+        return datetime.strptime("01011900", "%d%m%Y").date()
+
+# Helper function to process patient names
+def process_name_text(name_text):
+    """
+    Remove all numbers, punctuation, and whitespace from a string of text and return the result.
+    """
+    name = re.sub(r'[0-9!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~\s]+', '', name_text).strip()
+    # capitalize first letter of each word
+    name = ' '.join([word.capitalize() for word in name.split()])
+    return name
+
+
+# Helper function to process endoscope type text
+def process_general_text(endoscope_text):
+    """
+    This function takes in a string of text from an endoscope and returns a cleaned version of the text.
+    """
+    return ' '.join(endoscope_text.split())
+
+def roi_values_valid(roi):
+    """
+    Check if all values in an ROI dictionary are valid (>=0).
+    """
+    return all([value >= 0 for value in roi.values()])
+
+# Function to extract text from ROIs
+def extract_text_from_rois(image_path, processor):
+    """
+    Extracts text from regions of interest (ROIs) in an image using OCR.
+
+    Args:
+        image_path (str): The path to the image file.
+        processor (EndoscopyProcessor): An instance of the EndoscopyProcessor class.
+
+    Returns:
+        dict: A dictionary containing the extracted text for each ROI.
+    """
+    # Read the image using Pillow
+    image = Image.open(image_path)
+    image_dimensions = image.size # (width, height)
+
+    ####### Adjust Image #######
+    # Convert to grayscale
+    gray = image.convert('L')
+
+    # Invert colors for white text on black background
+    inverted = ImageOps.invert(gray)
+
+    # Initialize the dictionary to hold the extracted text
+    extracted_texts = {}
+
+    # Define your ROIs and their corresponding post-processing functions in tuples
+    rois_with_postprocessing = [
+        ('examination_date', processor.get_roi_examination_date, process_date_text),
+        ("patient_first_name", processor.get_roi_patient_first_name, process_name_text),
+        ('patient_last_name', processor.get_roi_patient_last_name, process_name_text),
+        ('patient_dob', processor.get_roi_patient_dob, process_date_text),
+        ('endoscope_type', processor.get_roi_endoscope_type, process_general_text),
+        ('endoscope_sn', processor.get_roi_endoscopy_sn, process_general_text),
+    ]
+
+    # Extract and post-process text for each ROI
+    for roi_name, roi_function, post_process in rois_with_postprocessing:
+        # Get the ROI dictionary
+        roi = roi_function()
+        
+        # Check if the ROI has values
+        
+        if roi_values_valid(roi):
+            x, y, w, h = roi['x'], roi['y'], roi['width'], roi['height']
+            
+            # Get white image with original shape and just the roi remaining
+            roi_image = crop_and_insert(inverted, x,y,h,w)
+
+            # OCR configuration: Recognize white text on black background without corrections
+            config = '--psm 10 -c tessedit_char_whitelist=0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-üöäÜÖÄß'
+
+            # Use pytesseract to do OCR on the preprocessed ROI
+            text = pytesseract.image_to_string(roi_image, config=config).strip()
+
+            # Post-process extracted text
+            processed_text = post_process(text)
+            
+            extracted_texts[roi_name] = processed_text
+
+        else:
+            pass
+
+    return extracted_texts
+
+def get_most_frequent_values(rois_texts: Dict[str, List[str]]) -> Dict[str, str]:
+    """
+    Given a dictionary of ROIs and their corresponding texts, returns a dictionary of the most frequent text for each ROI.
+
+    Args:
+        rois_texts: A dictionary where the keys are the names of the ROIs and the values are lists of texts.
+
+    Returns:
+        A dictionary where the keys are the names of the ROIs and the values are the most frequent text for each ROI.
+    """
+    most_frequent = {}
+    for key in rois_texts.keys():
+        counter = Counter([text for text in rois_texts[key] if text])
+        most_frequent[key], _ = counter.most_common(1)[0] if counter else (None, None)
+    return most_frequent
+
+def process_video(video_path, processor):
+    """
+    Processes a video file by extracting text from regions of interest (ROIs) in each frame.
+
+    Args:
+        video_path (str): The path to the video file to process.
+        processor (OCRProcessor): An instance of the OCRProcessor class that defines the ROIs to extract text from.
+
+    Returns:
+        dict: A dictionary containing the most frequent text values extracted from each ROI.
+    """
+    # Create a temporary directory to store frames
+    with TemporaryDirectory() as temp_dir:
+        # Capture the video
+        video = cv2.VideoCapture(video_path)
+        success, frame_number = True, 0
+        rois_texts = {roi_name: [] for roi_name in processor.get_rois().keys()}
+        frames_for_mean_extraction = 0
+
+        while success:
+            success, frame = video.read()
+
+            # Check if this is the 200th frame
+            if frame_number % 1000 == 0 and success:
+                frame_path = os.path.join(temp_dir, f"frame_{frame_number}.jpg")
+                cv2.imwrite(frame_path, frame)  # Save the frame as a JPEG file
+                # cv2.imwrite(f"_tmp/frame_{frame_number}.jpg", frame)
+                
+                # Extract text from ROIs
+                extracted_texts = extract_text_from_rois(frame_path, processor)
+                
+                # Store the extracted text from each ROI
+                for key, text in extracted_texts.items():
+                    rois_texts[key].append(text)
+                frames_for_mean_extraction += 1
+
+            frame_number += 1
+
+            if frames_for_mean_extraction >= N_FRAMES_MEAN_OCR: break
+
+        # Release the video capture object
+        video.release()
+
+        # Get the most frequent values for each ROI
+        return get_most_frequent_values(rois_texts)
+

endoreg_db/utils/parse_and_generate_yaml.py

@@ -0,0 +1,46 @@
+import yaml
+from pathlib import Path
+
+# get this files path
+file_path = Path(__file__)
+module_root = file_path.parent.parent
+data_dir = module_root / "data"
+
+
+def collect_center_names(
+):
+    input_file_path = data_dir / "center/data.yaml"
+    fist_name_dir = data_dir / "names_first"
+    last_name_dir = data_dir / "names_last"
+    # Load the input YAML file
+    with open(input_file_path, 'r', encoding='utf-8') as file:
+        data = yaml.safe_load(file)
+    
+    # Containers for first and last names
+    first_names_set = set()
+    last_names_set = set()
+    
+    # Extract first and last names from the YAML data
+    for entry in data:
+        fields = entry.get('fields', {})
+        first_names_set.update(fields.get('first_names', []))
+        last_names_set.update(fields.get('last_names', []))
+    
+    # Create a list of dictionaries for first and last names
+    first_names_data = [
+        {"model": "endoreg_db.first_name", "fields": {"name": name}}
+        for name in sorted(first_names_set)
+    ]
+    last_names_data = [
+        {"model": "endoreg_db.last_name", "fields": {"name": name}}
+        for name in sorted(last_names_set)
+    ]
+    
+    # Write the data to separate YAML files
+    with open(fist_name_dir/"first_names.yaml", "w", encoding='utf-8') as first_file:
+        yaml.dump(first_names_data, first_file, allow_unicode=True, sort_keys=False)
+    
+    with open(last_name_dir/"last_names.yaml", "w", encoding='utf-8') as last_file:
+        yaml.dump(last_names_data, last_file, allow_unicode=True, sort_keys=False)
+
+    # print("Generated first_names.yaml and last_names.yaml successfully.")

endoreg_db/utils/paths.py

@@ -0,0 +1,95 @@
+"""
+Centralizes path management for the application.
+
+This module sets up all necessary directories for storage and import operations.
+It provides a unified dictionary 'data_paths' for accessing all path objects.
+"""
+
+from logging import getLogger
+
+from sphinx.search import no
+logger = getLogger(__name__)
+
+from pathlib import Path
+from typing import Dict
+import os
+# Alternative approach using env_path helper, deprecated since monorepo setup. Alright for single install, env is always preferred.
+#from endoreg_db.config.env import env_path
+
+STORAGE_DIR = Path(os.getenv("STORAGE_DIR", "storage"))
+
+storage_dir_env = os.getenv("STORAGE_DIR")
+if storage_dir_env is None:
+    raise RuntimeError("STORAGE_DIR environment variable is not set.")
+storage_dir_str = Path(storage_dir_env)
+STORAGE_DIR = storage_dir_str
+if not STORAGE_DIR.exists():
+    STORAGE_DIR.mkdir(parents=True, exist_ok=True)
+
+PREFIX_RAW = "raw_"
+IMPORT_DIR_NAME = "import"
+EXPORT_DIR_NAME = "export"
+
+VIDEO_DIR_NAME = "videos"
+ANONYM_VIDEO_DIR_NAME = "anonym_videos" # Added for processed videos
+FRAME_DIR_NAME = "frames"
+PDF_DIR_NAME = "pdfs" # Changed from reports
+WEIGHTS_DIR_NAME = "model_weights"
+EXAMINATION_DIR_NAME = "examinations"
+
+RAW_VIDEO_DIR_NAME = f"{PREFIX_RAW}videos"
+RAW_FRAME_DIR_NAME = f"{PREFIX_RAW}frames"
+RAW_PDF_DIR_NAME = f"{PREFIX_RAW}pdfs" # Changed from reports
+
+# Define data subdirectories under STORAGE_DIR
+VIDEO_DIR = STORAGE_DIR / VIDEO_DIR_NAME
+ANONYM_VIDEO_DIR = STORAGE_DIR / ANONYM_VIDEO_DIR_NAME # Added
+FRAME_DIR = STORAGE_DIR / FRAME_DIR_NAME
+PDF_DIR = STORAGE_DIR / PDF_DIR_NAME # Changed
+WEIGHTS_DIR = STORAGE_DIR / WEIGHTS_DIR_NAME
+RAW_VIDEO_DIR = STORAGE_DIR / RAW_VIDEO_DIR_NAME
+RAW_FRAME_DIR = STORAGE_DIR / RAW_FRAME_DIR_NAME
+RAW_PDF_DIR = STORAGE_DIR / RAW_PDF_DIR_NAME # Changed
+
+IMPORT_DIR = STORAGE_DIR / IMPORT_DIR_NAME
+VIDEO_IMPORT_DIR = IMPORT_DIR / VIDEO_DIR_NAME
+FRAME_IMPORT_DIR = IMPORT_DIR / FRAME_DIR_NAME
+PDF_IMPORT_DIR = IMPORT_DIR / PDF_DIR_NAME # Changed
+WEIGHTS_IMPORT_DIR = IMPORT_DIR / WEIGHTS_DIR_NAME
+
+EXPORT_DIR = STORAGE_DIR / EXPORT_DIR_NAME
+
+data_paths:Dict[str,Path] = {
+    "storage": STORAGE_DIR,
+    "video": VIDEO_DIR,
+    "anonym_video": ANONYM_VIDEO_DIR, # Added
+    "frame": FRAME_DIR,
+    "pdf": PDF_DIR, # Changed
+    "import": IMPORT_DIR,
+    "video_import": VIDEO_IMPORT_DIR,
+    "frame_import": FRAME_IMPORT_DIR,
+    "pdf_import": PDF_IMPORT_DIR, # Changed
+    "raw_video": RAW_VIDEO_DIR,
+    "raw_frame": RAW_FRAME_DIR,
+    "raw_pdf": RAW_PDF_DIR, # Changed
+    "weights": WEIGHTS_DIR,
+    "weights_import": WEIGHTS_IMPORT_DIR,
+    "export": EXPORT_DIR,
+    "video_export": EXPORT_DIR / VIDEO_DIR_NAME,
+    "anonym_video_export": EXPORT_DIR / ANONYM_VIDEO_DIR_NAME, # Added
+    "frame_export": EXPORT_DIR / FRAME_DIR_NAME,
+    "pdf_export": EXPORT_DIR / PDF_DIR_NAME, # Changed
+    "weights_export": EXPORT_DIR / WEIGHTS_DIR_NAME,
+    "examination_export": EXPORT_DIR / EXAMINATION_DIR_NAME,
+    "raw_video_export": EXPORT_DIR / RAW_VIDEO_DIR_NAME,
+    "raw_frame_export": EXPORT_DIR / RAW_FRAME_DIR_NAME,
+    "raw_pdf_export": EXPORT_DIR / RAW_PDF_DIR_NAME, # Changed
+}
+
+logger.info(f"Storage directory: {STORAGE_DIR.resolve()}")
+logger.info(f"Export directory: {EXPORT_DIR.resolve()}")
+
+# for key, path in data_paths.items():
+#     path.mkdir(parents=True, exist_ok=True)
+
+#     logger.info(f"{key.capitalize()} directory: {path.resolve()}")