endoreg-db 0.3.6__py3-none-any.whl → 0.8.6.1__py3-none-any.whl

endoreg_db/management/commands/import_fallback_video.py

@@ -0,0 +1,203 @@
+from pathlib import Path
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from endoreg_db.models import (
+    VideoFile, 
+    Center, 
+    Label, 
+    VideoSegmentationLabelSet,
+    VideoSegmentationLabel,
+    EndoscopyProcessor
+)
+
+class Command(BaseCommand):
+    help = 'Import fallback test video and create default labels'
+
+    def _ensure_default_objects(self):
+        """
+        Ensures that default Center and EndoscopyProcessor objects exist, creating them if necessary.
+        
+        Returns:
+            tuple: A tuple containing the Center and EndoscopyProcessor objects.
+        """
+        center, created = Center.objects.get_or_create(
+            name="Default Center",
+            defaults={
+                'description': 'Fallback center for test videos',
+                'location': 'Test Location'
+            }
+        )
+        if created:
+            self.stdout.write(f"Created center: {center.name}")
+
+        processor, created = EndoscopyProcessor.objects.get_or_create(
+            name="Default Processor",
+            defaults={
+                'description': 'Fallback processor for test videos'
+            }
+        )
+        if created:
+            self.stdout.write(f"Created processor: {processor.name}")
+        
+        return center, processor
+
+    def add_arguments(self, parser):
+        """
+        Adds the --video-path argument to specify the path of the test video file.
+        
+        This optional argument allows users to provide a custom path to the test video file to be imported. If not specified, a default path is used.
+        """
+        parser.add_argument(
+            '--video-path',
+            type=str,
+            default='~/test-data/video/lux-gastro-video.mp4',
+            help='Path to the test video file'
+        )
+
+    def handle(self, *args, **options):
+        """
+        Handles the import of a test video and setup of default labels for frontend testing.
+        
+        Checks for the existence of the specified video file. If found, ensures default Center and EndoscopyProcessor objects exist, creates default annotation labels, and imports the video into the database. If the video file is missing or import fails, creates fallback database entries to enable frontend testing.
+        """
+        video_path_str = options['video_path']
+        video_path = Path(video_path_str).expanduser()
+        
+        self.stdout.write(f"Looking for video at: {video_path}")
+        
+        if not video_path.exists():
+            self.stdout.write(
+                self.style.WARNING(f"Video file not found at {video_path}")
+            )
+            # Create a fallback entry anyway for frontend testing
+            self.create_fallback_entries()
+            return
+        
+        # Create or get default center and processor
+        center, processor = self._ensure_default_objects()
+        
+        # Create default labels
+        self.create_default_labels()
+        
+        # Import the video if it exists
+        try:
+            video_file = VideoFile.create_from_file_initialized(
+                file_path=video_path,
+                center_name=center.name,
+                processor_name=processor.name
+            )
+            self.stdout.write(
+                self.style.SUCCESS(f"Successfully imported video: {video_file.uuid}")
+            )
+        except Exception as e:
+            self.stdout.write(
+                self.style.ERROR(f"Failed to import video: {e}")
+            )
+            # Create fallback entries for frontend testing
+            self.create_fallback_entries()
+
+    def create_default_labels(self):
+        """
+        Creates a default set of video segmentation labels and associates them with a label set.
+        
+        This method ensures that a predefined group of labels, each with multilingual names, colors, and order priorities, exists in the database for endoscopy video annotation. It creates or retrieves both specialized segmentation labels and general labels for compatibility, and links the segmentation labels to a default label set named "Default Endoscopy Labels."
+        """
+        
+        # Create default labelset
+        labelset, created = VideoSegmentationLabelSet.objects.get_or_create(
+            name="Default Endoscopy Labels",
+            defaults={
+                'description': 'Default labels for endoscopy video annotation'
+            }
+        )
+        if created:
+            self.stdout.write(f"Created labelset: {labelset.name}")
+        
+        # Fetch existing labels in the labelset to avoid N+1 queries
+        existing_labels_in_set = set(labelset.labels.values_list('pk', flat=True))
+        
+        # Default labels for endoscopy
+        default_labels = [
+            {'name': 'outside', 'name_de': 'Außerhalb', 'name_en': 'Outside', 'color': '#00bcd4', 'order_priority': 1},
+            {'name': 'appendix', 'name_de': 'Appendix', 'name_en': 'Appendix', 'color': '#ff9800', 'order_priority': 2},
+            {'name': 'blood', 'name_de': 'Blut', 'name_en': 'Blood', 'color': '#f44336', 'order_priority': 3},
+            {'name': 'diverticule', 'name_de': 'Divertikel', 'name_en': 'Diverticule', 'color': '#9c27b0', 'order_priority': 4},
+            {'name': 'grasper', 'name_de': 'Greifer', 'name_en': 'Grasper', 'color': '#CBEDCA', 'order_priority': 5},
+            {'name': 'ileocaecalvalve', 'name_de': 'Ileozäkalklappe', 'name_en': 'Ileocaecal Valve', 'color': '#3f51b5', 'order_priority': 6},
+            {'name': 'ileum', 'name_de': 'Ileum', 'name_en': 'Ileum', 'color': '#2196f3', 'order_priority': 7},
+            {'name': 'low_quality', 'name_de': 'Niedrige Bildqualität', 'name_en': 'Low Quality', 'color': '#9e9e9e', 'order_priority': 8},
+            {'name': 'nbi', 'name_de': 'Narrow Band Imaging', 'name_en': 'NBI', 'color': '#795548', 'order_priority': 9},
+            {'name': 'needle', 'name_de': 'Nadel', 'name_en': 'Needle', 'color': '#e91e63', 'order_priority': 10},
+            {'name': 'polyp', 'name_de': 'Polyp', 'name_en': 'Polyp', 'color': '#8bc34a', 'order_priority': 11},
+            {'name': 'snare', 'name_de': 'Snare', 'name_en': 'Snare', 'color': '#ff5722', 'order_priority': 12},
+            {'name': 'water_jet', 'name_de': 'Wasserstrahl', 'name_en': 'Water Jet', 'color': '#03a9f4', 'order_priority': 13},
+            {'name': 'wound', 'name_de': 'Wunde', 'name_en': 'Wound', 'color': '#607d8b', 'order_priority': 14},
+        ]
+        
+        for label_data in default_labels:
+            # Create VideoSegmentationLabel
+            vs_label, created = VideoSegmentationLabel.objects.get_or_create(
+                name=label_data['name'],
+                defaults={
+                    'name_de': label_data['name_de'],
+                    'name_en': label_data['name_en'],
+                    'color': label_data['color'],
+                    'order_priority': label_data['order_priority'],
+                    'description': f"Default {label_data['name_en']} label"
+                }
+            )
+            
+            # Create general Label (for compatibility)
+            general_label, created = Label.objects.get_or_create(
+                name=label_data['name'],
+                defaults={
+                    'description': f"Default {label_data['name_en']} label"
+                }
+            )
+            
+            if created:
+                self.stdout.write(f"Created label: {label_data['name']}")
+            
+            # Link to labelset
+            if vs_label.pk not in existing_labels_in_set:
+                labelset.labels.add(vs_label)
+                existing_labels_in_set.add(vs_label.pk) # Keep the set in sync
+        
+        labelset.save()
+        self.stdout.write(f"Labelset configured with {labelset.labels.count()} labels")
+
+    def create_fallback_entries(self):
+        """
+        Creates fallback database entries for testing when the video file is unavailable.
+        
+        This method ensures that default annotation labels, a default center, and a default endoscopy processor exist. If a fallback video entry named "lux-gastro-video.mp4" does not already exist, it creates a minimal `VideoFile` record with preset metadata and initializes its state and frame directory for frontend testing.
+        """
+        with transaction.atomic():
+            # Create default labels anyway
+            self.create_default_labels()
+            
+            # Create a placeholder VideoFile entry for frontend testing
+            center, processor = self._ensure_default_objects()
+            
+            # Check if we already have a fallback video
+            if not VideoFile.objects.filter(original_file_name="lux-gastro-video.mp4").exists():
+                # Create a minimal VideoFile entry for frontend testing
+                video_file = VideoFile.objects.create(
+                    original_file_name="lux-gastro-video.mp4",
+                    video_hash="fallback_hash_12345",
+                    center=center,
+                    processor=processor,
+                    fps=30.0,
+                    duration=120.0,  # 2 minutes fallback
+                    frame_count=3600,
+                    width=1920,
+                    height=1080
+                )
+                
+                # Initialize the video file
+                video_file.get_or_create_state()
+                video_file.set_frame_dir()
+                
+                self.stdout.write(
+                    self.style.SUCCESS(f"Created fallback video entry: {video_file.uuid}")
+                )

endoreg_db/management/commands/import_report.py

@@ -0,0 +1,298 @@
+from pathlib import Path
+import os
+import sys
+from django.core.management import BaseCommand
+from endoreg_db.models import (
+    RawPdfFile,
+    PdfType,
+)
+from endoreg_db.helpers.data_loader import (
+    load_data
+)
+from icecream import ic
+# python manage.py import_report tests/assets/lux-gastro-report.pdf --verbose --start_ollama
+# Dynamic import path manipulation to ensure local development version is used
+def ensure_local_lx_anonymizer():
+    """
+    Checks for a local development version of the lx-anonymizer package and adds it to sys.path if available.
+    
+    Returns:
+        True if the local lx-anonymizer directory was found and added to sys.path; False otherwise.
+    """
+    script_dir = Path(__file__).parent.parent.parent.parent.parent  # /home/admin/dev/lx-annotate/endoreg-db
+    local_lx_anonymizer_path = script_dir / "lx-anonymizer"
+    
+    if local_lx_anonymizer_path.exists() and local_lx_anonymizer_path.is_dir():
+        # Add the directory containing lx_anonymizer to the Python path
+        if str(local_lx_anonymizer_path) not in sys.path:
+            sys.path.insert(0, str(local_lx_anonymizer_path))
+            print(f"Using local lx-anonymizer from: {local_lx_anonymizer_path}")
+            return True
+    return False
+
+# Try to use local version, fall back to installed version
+local_version_available = ensure_local_lx_anonymizer()
+
+
+
+# Now import from lx_anonymizer
+try:
+    from lx_anonymizer.ollama_service import ollama_service
+except ImportError:
+    print("Could not import init_ollama_service from local or installed lx_anonymizer")
+    raise
+
+class Command(BaseCommand):
+    """Management Command to import a report file to the database"""
+
+    help = """
+        Imports a .pdf file to the database.
+        1. Get center by center name from db (default: university_hospital_wuerzburg)
+    """
+
+    def add_arguments(self, parser):
+        """
+        Defines command-line arguments for the import_report management command.
+        
+        Adds options for specifying the report file path, center name, report directory, deletion and save behavior, and controls for initializing the Ollama LLM service.
+        """
+        parser.add_argument(
+            "--verbose",
+            action="store_true",
+            help="Display verbose output for all commands",
+        )
+
+        parser.add_argument(
+            "file_path",
+            type=str,
+            help="Path to the Report file to import",
+        )
+        parser.add_argument(
+            "--center_name",
+            type=str,
+            default="university_hospital_wuerzburg",
+            help="Name of the center to associate with the report",
+        )
+
+        parser.add_argument(
+            "--report_dir_root",
+            type=str,
+            default="~/test-data/db_report_dir",
+            help="Path to the report directory",
+        )
+
+        parser.add_argument(
+            "--delete_source",
+            action="store_true",
+            default=False,
+            help="Delete the source report file after importing",
+        )
+
+        parser.add_argument(
+            "--save",
+            action="store_true",
+            default=False,
+            help="Save the report object to the database",
+        )
+        
+        parser.add_argument(
+            "--start_ollama",
+            action="store_true",
+            default=False,
+            help="Start Ollama server for LLM processing",
+        )
+        
+        parser.add_argument(
+            "--ollama_debug",
+            action="store_true",
+            default=False,
+            help="Enable debug mode for Ollama",
+        )
+        
+        parser.add_argument(
+            "--ollama_timeout",
+            type=int,
+            default=30,
+            help="Maximum time to wait for Ollama to start in seconds",
+        )
+
+    def handle(self, *args, **options):
+        """
+        Handles the import of a PDF report file into the database, with optional LLM service initialization and anonymization.
+        
+        This method validates input options, optionally starts the Ollama LLM service, ensures the existence of required files and directories, determines the report type, processes the PDF for text and metadata extraction, anonymizes content, and saves the resulting data to the database. Provides verbose output and error handling throughout the process.
+        """
+        # Load initial or prerequisite data for the application.
+        # This may include loading default values, configurations, or lookup table data
+        # necessary for the import process or other application functionalities.
+        try:
+            load_data()
+            self.stdout.write(self.style.SUCCESS("Successfully loaded initial data."))
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"Failed to load initial data: {e}"))
+            # Depending on the criticality of load_data(), you might want to exit or handle differently.
+            # For now, we'll just log the error and continue.
+        
+            
+        verbose = options["verbose"]
+        center_name = options["center_name"]
+        report_dir_root = options["report_dir_root"]
+        file_path = options["file_path"]
+        delete_source = options["delete_source"]
+        save = options["save"]
+        start_ollama = options["start_ollama"]
+        ollama_debug = options["ollama_debug"]
+        ollama_timeout = options["ollama_timeout"]
+
+        if not isinstance(delete_source, bool):
+            raise ValueError("delete_source must be a boolean")
+
+        self.stdout.write(self.style.SUCCESS(f"Starting report import for {file_path}..."))
+        
+        if local_version_available:
+            self.stdout.write(self.style.SUCCESS("Using local development version of lx-anonymizer"))
+        
+        ollama_proc = None  # Track Ollama process if started
+        try:
+            # Initialize Ollama service if requested
+            if start_ollama:
+                self.stdout.write(self.style.SUCCESS("Initializing Ollama service..."))
+                try:
+                    # Set Ollama environment variables
+                    os.environ["OLLAMA_MAX_WAIT_TIME"] = str(ollama_timeout)
+                    os.environ["OLLAMA_DEBUG"] = "true" if ollama_debug else "false"
+                    
+                    # Try to find Ollama binary location from env or common paths
+                    ollama_bin = os.environ.get("OLLAMA_BIN")
+                    if not ollama_bin:
+                        # Try common Nix store paths first
+                        for path in ["/run/current-system/sw/bin/ollama", 
+                                    "/nix/store/*/bin/ollama"]:
+                            import glob
+                            matches = glob.glob(path)
+                            if matches:
+                                ollama_bin = matches[0]
+                                break
+                    
+                    if ollama_bin:
+                        self.stdout.write(self.style.SUCCESS(f"Using Ollama binary at: {ollama_bin}"))
+                        os.environ["OLLAMA_BIN"] = ollama_bin
+                    
+                    # Start Ollama server process if not already running
+                    import subprocess
+                    import shutil
+                    
+                    # Check if ollama is already running
+                    try:
+                        import requests
+                        resp = requests.get("http://127.0.0.1:11434/api/version", timeout=1)
+                        if resp.status_code == 200:
+                            self.stdout.write(self.style.SUCCESS("Ollama is already running"))
+                        else:
+                            self.stdout.write(self.style.WARNING(f"Ollama returned status code {resp.status_code}"))
+                    except requests.exceptions.RequestException:
+                        self.stdout.write(self.style.WARNING("Ollama is not running, attempting to start..."))
+                        # Find ollama binary
+                        ollama_path = ollama_bin or shutil.which("ollama")
+                        if ollama_path:
+                            self.stdout.write(self.style.SUCCESS(f"Starting Ollama using {ollama_path}"))
+                            # Start ollama serve in background
+                            ollama_proc = subprocess.Popen([
+                                ollama_path, "serve"
+                            ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, start_new_session=True)
+                            self.stdout.write(self.style.SUCCESS("Ollama server started in background"))
+                        else:
+                            self.stdout.write(self.style.ERROR("Ollama binary not found in PATH"))
+                    
+                    # Start the service with explicit initialization
+                    ollama_service(auto_start=True)
+                    self.stdout.write(self.style.SUCCESS("Ollama service initialized successfully"))
+                except Exception as e:
+                    self.stdout.write(self.style.ERROR(f"Failed to initialize Ollama service: {e}"))
+                    self.stdout.write(self.style.WARNING("Continuing without Ollama - some features may not work"))
+
+            # Ensure the report file exists
+            file_path = Path(file_path).expanduser()
+            if not file_path.exists():
+                self.stdout.write(self.style.ERROR(f"Report file not found: {file_path}"))
+                return
+
+            # Ensure the report directory exists
+            report_dir_root = Path(report_dir_root).expanduser()
+            report_dir_root.mkdir(parents=True, exist_ok=True)
+
+            # Create the report file object
+            self.stdout.write(self.style.SUCCESS(f"Creating RawPdfFile object from {file_path}..."))
+            report_file_obj = RawPdfFile.create_from_file(
+                file_path=file_path,
+                center_name=center_name,
+                delete_source=delete_source,
+                save=save,
+            )
+            if not report_file_obj:
+                self.stdout.write(self.style.ERROR("Failed to create RawPdfFile object."))
+                return
+            
+            report_file_obj.anonymized = False
+            
+            # Assign pdfType to the report file object
+            if "report" in file_path.name:
+                pdf_type_name = "ukw-endoscopy-examination-report-generic"
+            elif "histo" in file_path.name:
+                pdf_type_name = "ukw-endoscopy-histology-report-generic"
+            elif "AW_PA" in file_path.name:
+                pdf_type_name = "rkh-endoscopy-histology-report-generic"
+            elif "AW" in file_path.name:
+                pdf_type_name = "rkh-endoscopy-examination-report-generic"
+            else:
+                raise ValueError(f"Unknown report type: {file_path.name}")
+
+            self.stdout.write(self.style.SUCCESS(f"Using PDF type: {pdf_type_name}"))
+            try:
+                pdf_type = PdfType.objects.get(name=pdf_type_name)
+            except PdfType.DoesNotExist:
+                self.stdout.write(self.style.ERROR(f"PdfType with name '{pdf_type_name}' does not exist."))
+                return
+            report_file_obj.pdf_type = pdf_type
+
+            rr_config = report_file_obj.get_report_reader_config()
+            pdf_path = report_file_obj.file.path
+            
+            # Import at this point to avoid initializing the module too early
+            from lx_anonymizer import ReportReader
+            self.stdout.write(self.style.SUCCESS("Creating ReportReader..."))
+            rr = ReportReader(**rr_config)
+
+            self.stdout.write(self.style.SUCCESS(f"Processing report: {pdf_path}"))
+            text, anonymized_text, report_meta = rr.process_report(
+                pdf_path, verbose=verbose
+            )
+
+            if verbose:
+                ic(text, anonymized_text, report_meta)
+                
+            self.stdout.write(self.style.SUCCESS("Processing file..."))
+            report_file_obj.process_file(text, anonymized_text, report_meta, verbose=verbose)
+            
+            sensitive_meta = report_file_obj.sensitive_meta
+            if verbose:
+                ic(report_file_obj.sensitive_meta)
+                
+            
+            self.stdout.write(self.style.SUCCESS("Saving..."))
+            sensitive_meta.save()
+            if verbose:
+                ic(sensitive_meta)
+                
+            report_file_obj.anonymized=True
+        finally:
+            # Clean up Ollama process if we started it
+            if ollama_proc is not None:
+                import signal
+                self.stdout.write(self.style.SUCCESS("Cleaning up Ollama server process..."))
+                try:
+                    ollama_proc.terminate()
+                    ollama_proc.wait(timeout=10)
+                    self.stdout.write(self.style.SUCCESS("Ollama server process terminated."))
+                except Exception as e:
+                    self.stdout.write(self.style.WARNING(f"Failed to terminate Ollama server process: {e}"))