endoreg-db 0.3.6__py3-none-any.whl → 0.8.6.1__py3-none-any.whl

endoreg_db/utils/dataloader.py

@@ -1,92 +1,175 @@
-import os
-import yaml
-from django.core.exceptions import ObjectDoesNotExist
-
-def load_model_data_from_yaml(command, model_name, metadata, verbose):
-    """
-    Load model data from YAML files.
-    
-    Args:
-        command: Command object for stdout writing.
-        model_name: Name of the model being loaded.
-        metadata: Metadata including directory and foreign key information.
-        verbose: Boolean indicating whether to print verbose output.
-    """
-    if verbose:
-        command.stdout.write(f"Start loading {model_name}")
-    model = metadata["model"]
-    dir_path = metadata["dir"]
-    foreign_keys = metadata["foreign_keys"]
-    foreign_key_models = metadata["foreign_key_models"]
-
-    _files = [f for f in os.listdir(dir_path) if f.endswith('.yaml')]
-    # sort
-    _files.sort()
-    for file in _files:
-        with open(os.path.join(dir_path, file), 'r') as file:
-            yaml_data = yaml.safe_load(file)
-        
-        load_data_with_foreign_keys(command, model, yaml_data, foreign_keys, foreign_key_models, verbose)
-
-def load_data_with_foreign_keys(command, model, yaml_data, foreign_keys, foreign_key_models, verbose):
-    """
-    Load data handling foreign keys and many-to-many relationships.
-    
-    Args:
-        command: Command object for stdout writing.
-        model: The Django model for the data.
-        yaml_data: Data loaded from YAML.
-        foreign_keys: List of foreign keys.
-        foreign_key_models: Corresponding models for the foreign keys.
-        verbose: Boolean indicating whether to print verbose output.
-    """
-    for entry in yaml_data:
-        fields = entry.get('fields', {})
-        name = fields.pop('name', None)
-        m2m_relationships = {}  # Store many-to-many relationships
-        print(entry)
-
-        # Handle foreign keys and many-to-many relationships
-        for fk_field, fk_model in zip(foreign_keys, foreign_key_models):
-            print(fk_field, fk_model)
-            target_keys = fields.pop(fk_field, None)
-            
-            # Ensure the foreign key exists
-            if target_keys is None:
-                if verbose:
-                    command.stdout.write(command.style.WARNING(f"Foreign key {fk_field} not found in fields"))
-                continue  # Skip if no foreign key provided
-
-            # Process many-to-many fields or foreign keys
-            if isinstance(target_keys, list):  # Assume many-to-many relationship
-                related_objects = []
-                for key in target_keys:
-                    obj, created = fk_model.objects.get_or_create(name=key)
-                    if created and verbose:
-                        command.stdout.write(command.style.SUCCESS(f"Created {fk_model.__name__} {key}"))
-                    related_objects.append(obj)
-                m2m_relationships[fk_field] = related_objects
-            else:  # Single foreign key relationship
-                try:
-                    if model == "endoreg_db.case_template_rule":
-                        print(fk_model, target_keys)
-                    obj = fk_model.objects.get_by_natural_key(target_keys)
-                except ObjectDoesNotExist:
-                    if verbose:
-                        command.stdout.write(command.style.WARNING(f"{fk_model.__name__} with key {target_keys} not found"))
-                    continue
-                fields[fk_field] = obj
-    
-        # Create or update the main object
-        if name is None:
-            obj, created = model.objects.get_or_create(**fields)
-        else:
-            obj, created = model.objects.update_or_create(defaults=fields, name=name)
-        if created and verbose:
-            command.stdout.write(command.style.SUCCESS(f'Created {model.__name__} {name}'))
-        elif verbose:
-            command.stdout.write(command.style.WARNING(f'Skipped {model.__name__} {name}, already exists'))
-
-        # Set many-to-many relationships
-        for field_name, related_objs in m2m_relationships.items():
-            getattr(obj, field_name).set(related_objs)
+import os
+import yaml
+from django.core.exceptions import ObjectDoesNotExist
+from django.db import OperationalError, transaction
+
+
+def load_model_data_from_yaml(command, model_name, metadata, verbose):
+    """
+    Load model data from YAML files.
+
+    Args:
+        command: Command object for stdout writing.
+        model_name: Name of the model being loaded.
+        metadata: Metadata including directory and foreign key information.
+        verbose: Boolean indicating whether to print verbose output.
+    """
+    if verbose:
+        command.stdout.write(f"Start loading {model_name}")
+    model = metadata["model"]
+    dir_path = metadata["dir"]
+    foreign_keys = metadata["foreign_keys"]
+    foreign_key_models = metadata["foreign_key_models"]
+
+    _files = [f for f in os.listdir(dir_path) if f.endswith(".yaml")]
+    # sort
+    _files.sort()
+    for file in _files:
+        with open(os.path.join(dir_path, file), "r", encoding="utf-8") as file:
+            yaml_data = yaml.safe_load(file)
+
+        load_data_with_foreign_keys(
+            command, model, yaml_data, foreign_keys, foreign_key_models, verbose
+        )
+
+
+def load_data_with_foreign_keys(
+    command, model, yaml_data, foreign_keys, foreign_key_models, verbose
+):
+    """
+    Load YAML data into Django model instances with FK and M2M support.
+    
+    Processes each YAML entry to create or update a model instance. For each entry, the
+    function extracts field data and uses the presence of a 'name' field to decide whether
+    to update an existing instance or create a new one. Foreign key fields listed in
+    foreign_keys are handled by retrieving related objects via natural keys. When a field
+    contains a list, it is treated as a many-to-many relationship and the corresponding
+    objects are set after the instance is saved. Missing or unresolved foreign keys trigger
+    warnings if verbose output is enabled.
+    
+    Parameters:
+        model: The Django model class representing the data.
+        yaml_data: A list of dictionaries representing YAML entries.
+        foreign_keys: A list of foreign key field names to process from each entry.
+        foreign_key_models: The corresponding Django model classes for each foreign key.
+        verbose: If True, prints detailed output and warnings during processing.
+    """
+    for entry in yaml_data:
+        fields = entry.get("fields", {})
+        name = fields.pop("name", None)
+
+
+        ####################
+        #TODO REMOVE AFTER TRANSLATION SUPPORT IS ADDED
+        SKIP_NAMES=[
+            "name_de",  # German name, not used
+            "name_en",  # English name, not used
+            "description_de",  # German description
+            "description_en",  # English description
+        ]
+
+        
+        # Remove fields that are not needed
+        for skip_name in SKIP_NAMES:
+            if skip_name in fields:
+                fields.pop(skip_name)
+        ########################
+
+        m2m_relationships = {}  # Store many-to-many relationships
+        # print(entry)
+
+        # Handle foreign keys and many-to-many relationships
+        for fk_field, fk_model in zip(foreign_keys, foreign_key_models):
+            # Skip fields that are not in the data
+            if fk_field not in fields:
+                continue
+
+            target_keys = fields.pop(fk_field, None)
+
+            # Ensure the foreign key exists
+            if target_keys is None:
+                if verbose:
+                    command.stdout.write(
+                        command.style.WARNING(
+                            f"Foreign key {fk_field} not found in fields"
+                        )
+                    )
+                continue  # Skip if no foreign key provided
+
+            # Process many-to-many fields or foreign keys
+            if isinstance(target_keys, list):  # Assume many-to-many relationship
+                related_objects = []
+                for key in target_keys:
+                    try:
+                        obj = fk_model.objects.get_by_natural_key(key)
+                    except ObjectDoesNotExist:
+                        if verbose:
+                            command.stdout.write(
+                                command.style.WARNING(
+                                    f"{fk_model.__name__} with key {key} not found"
+                                )
+                            )
+                        continue
+                    related_objects.append(obj)
+                m2m_relationships[fk_field] = related_objects
+            else:  # Single foreign key relationship
+                try:
+                    obj = fk_model.objects.get_by_natural_key(target_keys)
+                except ObjectDoesNotExist:
+                    if verbose:
+                        command.stdout.write(
+                            command.style.WARNING(
+                                f"{fk_model.__name__} with key {target_keys} not found"
+                            )
+                        )
+                    continue
+                fields[fk_field] = obj
+
+        # Create or update the main object (avoid update_or_create to prevent SQLite locks)
+        def _save_instance():
+            if name is None:
+                # Try to find an existing object by all provided fields
+                obj = model.objects.filter(**fields).first()
+                if obj is None:
+                    obj = model.objects.create(**fields)
+                    created = True
+                else:
+                    created = False
+            else:
+                obj = model.objects.filter(name=name).first()
+                if obj is None:
+                    obj = model.objects.create(name=name, **fields)
+                    created = True
+                else:
+                    # Update fields
+                    for k, v in fields.items():
+                        setattr(obj, k, v)
+                    obj.save()
+                    created = False
+            return obj, created
+
+        try:
+            # Attempt save inside a transaction for consistency
+            with transaction.atomic():
+                obj, created = _save_instance()
+        except OperationalError:
+            # Retry once on SQLite lock
+            obj, created = _save_instance()
+
+        if created and verbose:
+            command.stdout.write(
+                command.style.SUCCESS(f"Created {model.__name__} {name}")
+            )
+        elif verbose:
+            pass
+
+        # Set many-to-many relationships
+        for field_name, related_objs in m2m_relationships.items():
+            if related_objs:  # Only set if there are objects to set
+                getattr(obj, field_name).set(related_objs)
+                if verbose:
+                    command.stdout.write(
+                        command.style.SUCCESS(
+                            f"Set {len(related_objs)} {field_name} for {model.__name__} {name}"
+                        )
+                    )

endoreg_db/utils/dates.py

@@ -0,0 +1,60 @@
+from datetime import datetime, date, timedelta
+from random import randint
+from calendar import monthrange
+from django.utils import timezone
+import datetime
+
+# TODO replace used random_day_by_year function implementation when
+# creating pseudo patients with new function "random date by age_at_date and examination_date"
+
+
+def random_day_by_age_at_date(age_at_date: int, examination_date: date) -> date:
+    """
+    Return a random birth day for a patient with the specified age at the specified examination date.
+    """
+    examination_year = examination_date.year
+    latest_birthdate = examination_date.replace(year=examination_year - age_at_date)
+    valid_dates = [latest_birthdate - timedelta(days=i) for i in range(365)]
+
+    select = randint(0, len(valid_dates) - 1)
+    birth_date = valid_dates[select]
+
+    return birth_date
+
+
+def random_day_by_year(year: int) -> date:
+    """
+    Return a random birth day within the specified year.
+    """
+    month = randint(1, 12)
+    day = randint(1, monthrange(year, month)[1])
+
+    return date(year, month, day)
+
+
+def random_day_by_month_year(month: int, year) -> date:
+    """
+    Return a random birth day within the specified month and year.
+    """
+
+    day = randint(1, monthrange(year, month)[1])
+    return date(year, month, day)
+
+
+def ensure_aware_datetime(dt):
+    """
+    Ensures a datetime object is timezone-aware.
+    If the datetime is naive (has no timezone info), the current timezone is applied.
+    
+    Args:
+        dt: A datetime object that may be naive
+        
+    Returns:
+        A timezone-aware datetime object
+    """
+    if dt is None:
+        return None
+        
+    if isinstance(dt, datetime.datetime) and timezone.is_naive(dt):
+        return timezone.make_aware(dt)
+    return dt

endoreg_db/utils/env.py

@@ -0,0 +1,33 @@
+import os
+
+DEBUG = os.getenv("DEBUG", "false").lower() == "true"
+
+def get_env_var(var_name: str, default: str = None) -> str | None:
+    """
+    Get the value of an environment variable, with an optional default value.
+    If the environment variable is set, we need to remove flanking quotation marks and spaces.
+    if the environment variable is not set, we set it to the default value.
+    :param var_name: The name of the environment variable.
+    :param default: The default value to return if the environment variable is not set.
+    :return: The value of the environment variable or the default value.
+    """
+    value = os.environ.get(var_name)
+    if value:
+        value = value.strip('"\'')  # Strip both single and double quotes
+        if DEBUG:
+            print(f"Environment variable {var_name}: {value}")
+        return value
+    return default
+
+def set_env_var(var_name: str, value: str) -> None:
+    """
+    Set the value of an environment variable.
+    :param var_name: The name of the environment variable.
+    :param value: The value to set.
+    """
+    os.environ[var_name] = value
+    if DEBUG:
+        print(f"Set environment variable {var_name}: {value}")
+
+DJANGO_SETTINGS_MODULE = get_env_var("DJANGO_SETTINGS_MODULE") or "endoreg_db.settings_dev"
+

endoreg_db/utils/extract_specific_frames.py

@@ -0,0 +1,72 @@
+from pathlib import Path
+import shutil
+
+def extract_selected_frames(
+    video_path: Path,
+    frame_numbers: list,
+    output_dir: Path,
+    fps: int = 50,
+    quality: int = 2,
+    ext: str = "png"
+):
+    """
+    Extract specific frames from a video using the same quality logic as the original extractor.
+
+    This will:
+    - Convert frame numbers to timestamps using the provided FPS.
+    - Overwrite any existing frames for a clean run.
+    - Save frames with consistent formatting (zero-padded).
+
+    Args:
+        video_path (Path): Path to the input video.
+        frame_numbers (list): List of frame numbers to extract.
+        output_dir (Path): Where to save the frames.
+        fps (int): Frames per second of the video.
+        quality (int): Quality setting (2 is default).
+        ext (str): File extension for output (e.g., 'png/jpg').
+    """
+    # Optional: Clean up old frames before extraction
+    if output_dir.exists():
+        shutil.rmtree(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    for frame_number in frame_numbers:
+        # Convert frame number to timestamp in seconds
+        timestamp_sec = frame_number / fps
+
+        # Build output filename
+        output_file = output_dir / f"frame_{str(frame_number).zfill(7)}.{ext}"
+
+        # Extract and save the frame
+        extract_single_frame(
+            input_path=str(video_path),
+            timestamp=timestamp_sec,
+            output_path=str(output_file),
+            quality=quality,
+            ext=ext
+        )
+import subprocess
+from pathlib import Path
+
+def extract_single_frame(input_path: str, timestamp: float, output_path: str, quality: int = 2, ext: str = "png"):
+    """
+    Extract a single frame from a video using ffmpeg.
+
+    Args:
+        input_path (str): Path to the video file.
+        timestamp (float): Time in seconds to extract the frame.
+        output_path (str): Where to save the extracted frame.
+        quality (int): pnj quality (2 = high, 31 = low).
+        ext (str): File extension (e.g., 'jpg').
+    """
+    cmd = [
+        "ffmpeg",
+        "-loglevel", "error",
+        "-ss", f"{timestamp:.3f}",
+        "-i", str(input_path),
+        "-frames:v", "1",
+        "-q:v", str(quality),
+        str(output_path)
+    ]
+
+    subprocess.run(cmd, check=True)

endoreg_db/utils/file_operations.py

@@ -1,30 +1,58 @@
-import shutil
-from pathlib import Path
-from endoreg_db.utils.uuid import get_uuid
-
-def get_uuid_filename(file:Path) -> tuple[str, str]:
-    """
-    Returns a new filename with a uuid
-    """
-    # Get the file extension
-    file_extension = file.suffix
-    # Generate a new file name
-    uuid = get_uuid()
-    new_file_name = f"{uuid}{file_extension}"
-    return new_file_name, uuid
-    
-def rename_file_uuid(old_file:Path):
-    """
-    Rename a file by assigning a uuid while preserving file extension. Returns new filepath and uuid
-    """
-    # Get the file extension
-    file_extension = old_file.suffix
-    # Generate a new file name
-    uuid = get_uuid()
-    new_file_name = f"{uuid}{file_extension}"
-
-    # Rename the file
-    new_file = old_file.with_name(new_file_name)
-    shutil.move(old_file.resolve().as_posix(), new_file.resolve().as_posix())
-
-    return new_file, uuid
+import shutil
+from pathlib import Path
+from endoreg_db.utils.uuid import get_uuid
+import os
+
+def get_uuid_filename(file:Path) -> tuple[str, str]:
+    """
+    Returns a new filename with a uuid
+    """
+    # Get the file extension
+    file_extension = file.suffix
+    # Generate a new file name
+    uuid = get_uuid()
+    new_file_name = f"{uuid}{file_extension}"
+    return new_file_name, uuid
+    
+def rename_file_uuid(old_file:Path):
+    """
+    Rename a file by assigning a uuid while preserving file extension. Returns new filepath and uuid
+    """
+    # Get the file extension
+    file_extension = old_file.suffix
+    # Generate a new file name
+    uuid = get_uuid()
+    new_file_name = f"{uuid}{file_extension}"
+
+    # Rename the file
+    new_file = old_file.with_name(new_file_name)
+    shutil.move(old_file.as_posix(), new_file.as_posix())
+
+    return new_file, uuid
+
+def copy_with_progress(src: str, dst: str, buffer_size=1024 * 1024):
+    """
+    Make a copy of a file with progress bar.
+
+    Args:
+        src (str): Source file path.
+        dst (str): Destination file path.
+        buffer_size (int): Buffer size for copying.
+    """
+    # Ensure the destination directory exists
+    os.makedirs(os.path.dirname(dst), exist_ok=True)
+    total_size = os.path.getsize(src)
+    copied_size = 0
+
+    with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
+        while True:
+            buf = fsrc.read(buffer_size)
+            if not buf:
+                break
+            fdst.write(buf)
+            copied_size += len(buf)
+            progress = copied_size / total_size * 100
+            print(f"\rProgress: {progress:.2f}%", end="")
+
+    # Print newline once copying is finished so the next log starts on a new line
+    print()

endoreg_db/utils/fix_video_path_direct.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+"""
+Direct fix for video file path issues without complex Django setup.
+"""
+
+import sqlite3
+from pathlib import Path
+import os
+import argparse
+
+def fix_video_paths_direct(db_path, storage_dir):
+    """Fix video file paths directly in SQLite database."""
+    
+    db_path = Path(db_path)
+    storage_dir = Path(storage_dir)
+    
+    print("🔧 DIRECT VIDEO PATH FIX")
+    print("=" * 40)
+    
+    if not db_path.exists():
+        print(f"❌ Database not found at {db_path}")
+        return
+    
+    # Find actual video files
+    print("1. Scanning for video files...")
+    actual_files = {}
+    for pattern in ['**/*.mp4', '**/*.avi', '**/*.mov', '**/*.mkv']:
+        for file_path in storage_dir.glob(pattern):
+            if file_path.is_file() and file_path.stat().st_size > 0:
+                filename = file_path.name
+                if '_' in filename:
+                    uuid_part = filename.split('_')[0]
+                else:
+                    uuid_part = filename.split('.')[0]
+                
+                relative_path = file_path.relative_to(storage_dir)
+                actual_files[uuid_part] = {
+                    'absolute_path': str(file_path),
+                    'relative_path': str(relative_path),
+                    'size_mb': file_path.stat().st_size / (1024*1024)
+                }
+    
+    print(f"Found {len(actual_files)} video files")
+    
+    # Connect to database and check video records
+    print("\n2. Checking database records...")
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        
+        # Find video file table (might be different names)
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '%video%'")
+        tables = cursor.fetchall()
+        print(f"Found video tables: {[t[0] for t in tables]}")
+        
+        # Check for VideoFile records
+        video_table = None
+        for table in ['endoreg_db_videofile', 'videofile', 'video_file']:
+            try:
+                cursor.execute(f"SELECT COUNT(*) FROM {table}")
+                count = cursor.fetchone()[0]
+                if count > 0:
+                    video_table = table
+                    print(f"Using table: {table} ({count} records)")
+                    break
+            except sqlite3.OperationalError:
+                continue
+        
+        if not video_table:
+            print("❌ No video table found in database")
+            return
+        
+        # Get video record 5
+        cursor.execute(f"SELECT id, uuid, raw_file FROM {video_table} WHERE id = 5")
+        record = cursor.fetchone()
+        
+        if not record:
+            print("❌ Video ID 5 not found in database")
+            return
+        
+        video_id, uuid_str, current_raw_file = record
+        print(f"\n3. Video ID 5 details:")
+        print(f"   UUID: {uuid_str}")
+        print(f"   Current raw_file: {current_raw_file}")
+        
+        # Check if current file exists
+        if current_raw_file:
+            current_full_path = storage_dir / current_raw_file
+            file_exists = current_full_path.exists()
+            print(f"   File exists: {file_exists}")
+            if not file_exists:
+                print(f"   Full path checked: {current_full_path}")
+        
+        # Find matching file
+        if uuid_str in actual_files:
+            file_info = actual_files[uuid_str]
+            print(f"\n4. Found matching file:")
+            print(f"   Path: {file_info['absolute_path']}")
+            print(f"   Size: {file_info['size_mb']:.1f} MB")
+            print(f"   Relative path: {file_info['relative_path']}")
+            
+            if current_raw_file != file_info['relative_path']:
+                print(f"\n5. 🔧 FIXING PATH:")
+                print(f"   FROM: {current_raw_file}")
+                print(f"   TO:   {file_info['relative_path']}")
+                
+                # Update the database with transaction safety
+                try:
+                    cursor.execute(f"UPDATE {video_table} SET raw_file = ? WHERE id = ?", 
+                                   (file_info['relative_path'], video_id))
+                    conn.commit()
+                    print("✅ Database updated successfully!")
+                except Exception as e:
+                    conn.rollback()
+                    print(f"❌ Failed to update database: {e}")
+                
+                print("\n6. 🎯 NEXT STEPS:")
+                print("   1. Restart your Django development server")
+                print("   2. Test video streaming in the frontend")
+                print("   3. Check browser developer tools for any remaining errors")
+            else:
+                print("\n✅ Path is already correct in database")
+        else:
+            print(f"\n❌ No matching file found for UUID {uuid_str}")
+            print("Available UUIDs in storage:")
+            for uuid_key in list(actual_files.keys())[:5]:
+                print(f"   - {uuid_key}")
+        
+        conn.close()
+        
+    except Exception as e:
+        print(f"❌ Database error: {e}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Direct fix for video file path issues in SQLite DB.")
+    parser.add_argument('--db-path', type=str, default=os.environ.get('ENDOREG_DB_PATH', './db.sqlite3'),
+                        help='Path to the SQLite database file (default: ./db.sqlite3 or $ENDOREG_DB_PATH)')
+    parser.add_argument('--storage-dir', type=str, default=os.environ.get('ENDOREG_STORAGE_DIR', './storage'),
+                        help='Path to the storage directory (default: ./storage or $ENDOREG_STORAGE_DIR)')
+    args = parser.parse_args()
+    fix_video_paths_direct(args.db_path, args.storage_dir)