endoreg-db 0.8.8.9__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

endoreg_db/tasks/upload_tasks.py

@@ -1,216 +0,0 @@
-import logging
-import os
-from typing import Optional
-
-from django.core.files.storage import default_storage
-
-from celery_app import shared_task
-from endoreg_db.models import SensitiveMeta, VideoFile
-from endoreg_db.models.upload_job import UploadJob
-
-logger = logging.getLogger(__name__)
-
-
-@shared_task(bind=True)
-def process_upload_job(self, job_id: str) -> dict:
-    """
-    Process an uploaded file (report or video) asynchronously.
-
-    This task:
-    1. Updates job status to 'processing'
-    2. Determines file type and routes to appropriate processor
-    3. For PDFs: Uses ReportReader to extract metadata and create SensitiveMeta
-    4. For videos: Uses VideoFile.create_from_file_initialized() and processing pipeline
-    5. Links the created SensitiveMeta to the job and marks as 'anonymized'
-    6. On error: marks job as 'error' with details
-
-    Args:
-        job_id: UUID string of the UploadJob to process
-
-    Returns:
-        dict: Processing result with status and details
-    """
-    try:
-        # Get the upload job
-        try:
-            upload_job = UploadJob.objects.get(id=job_id)
-        except UploadJob.DoesNotExist:
-            error_msg = f"UploadJob with id {job_id} not found"
-            logger.error(error_msg)
-            return {"status": "error", "error": error_msg}
-
-        # Mark as processing
-        upload_job.mark_processing()
-        logger.info(f"Started processing upload job {job_id}")
-
-        # Get file path
-        file_path = upload_job.file.path
-        if not os.path.exists(file_path):
-            error_msg = f"Uploaded file not found at {file_path}"
-            upload_job.mark_error(error_msg)
-            logger.error(error_msg)
-            return {"status": "error", "error": error_msg}
-
-        # Route to appropriate processor based on content type
-        sensitive_meta = None
-
-        if upload_job.content_type == "application/pdf":
-            sensitive_meta = _process_pdf_file(upload_job, file_path)
-        elif upload_job.content_type.startswith("video/"):
-            sensitive_meta = _process_video_file(upload_job, file_path)
-        else:
-            error_msg = f"Unsupported content type: {upload_job.content_type}"
-            upload_job.mark_error(error_msg)
-            logger.error(error_msg)
-            return {"status": "error", "error": error_msg}
-
-        if sensitive_meta:
-            # Mark job as completed with the created SensitiveMeta
-            upload_job.mark_completed(sensitive_meta)
-            logger.info(
-                f"Successfully processed upload job {job_id}, created SensitiveMeta {sensitive_meta.id}"
-            )
-            return {
-                "status": "anonymized",
-                "sensitive_meta_id": sensitive_meta.id,
-                "job_id": job_id,
-            }
-        else:
-            error_msg = "Processing completed but no SensitiveMeta was created"
-            upload_job.mark_error(error_msg)
-            logger.error(f"Upload job {job_id}: {error_msg}")
-            return {"status": "error", "error": error_msg}
-
-    except Exception as e:
-        error_msg = f"Unexpected error processing upload job {job_id}: {str(e)}"
-        logger.exception(error_msg)
-
-        try:
-            upload_job = UploadJob.objects.get(id=job_id)
-            upload_job.mark_error(error_msg)
-        except Exception as inner_e:
-            logger.error(f"Failed to update job status after error: {inner_e}")
-
-        return {"status": "error", "error": error_msg}
-
-
-def _process_pdf_file(upload_job: UploadJob, file_path: str) -> Optional[SensitiveMeta]:
-    """
-    Process a report file using ReportReader.
-
-    Args:
-        upload_job: The UploadJob instance
-        file_path: Path to the uploaded report file
-
-    Returns:
-        SensitiveMeta instance if successful, None otherwise
-    """
-    try:
-        # Import ReportReader (lazy import to avoid startup issues)
-        try:
-            from lx_anonymizer.report_reader import ReportReader
-        except ImportError as e:
-            raise ImportError(
-                f"Could not import ReportReader: {e}. Make sure lx-anonymizer is available."
-            )
-
-        logger.info(f"Processing report file: {file_path}")
-
-        # Create ReportReader instance
-        reader = ReportReader(locale="de_DE", text_date_format="%d.%m.%Y")
-
-        # Process the report
-        original_text, anonymized_text, metadata = reader.process_report(
-            pdf_path=file_path,
-            use_ensemble=False,  # Can be made configurable
-            verbose=False,
-            use_llm_extractor=None,  # Use default extractor
-        )
-
-        if not metadata:
-            raise ValueError("No metadata extracted from report")
-
-        # Create SensitiveMeta from extracted metadata
-        sensitive_meta = SensitiveMeta.create_from_dict(metadata)
-        sensitive_meta.save()
-
-        logger.info(f"Created SensitiveMeta {sensitive_meta.id} from report processing")
-        return sensitive_meta
-
-    except Exception as e:
-        logger.exception(f"Error processing report file {file_path}: {e}")
-        raise
-
-
-def _process_video_file(
-    upload_job: UploadJob, file_path: str
-) -> Optional[SensitiveMeta]:
-    """
-    Process a video file using VideoFile creation and processing pipeline.
-
-    Args:
-        upload_job: The UploadJob instance
-        file_path: Path to the uploaded video file
-
-    Returns:
-        SensitiveMeta instance if successful, None otherwise
-    """
-    try:
-        logger.info(f"Processing video file: {file_path}")
-
-        # Get the relative path for storage
-        relative_path = os.path.relpath(file_path, default_storage.location)
-
-        # Create VideoFile instance using the existing method
-        video_file = VideoFile.create_from_file_initialized(
-            file_path=relative_path,
-            skip_frame_extraction=False,  # Extract frames for processing
-            skip_meta_extraction=False,  # Extract metadata
-        )
-
-        if not video_file:
-            raise ValueError("Failed to create VideoFile instance")
-
-        # Run the video processing pipeline (pipe_1)
-        try:
-            # Import the processing function
-            from endoreg_db.management.commands.import_video import pipe_1
-
-            # Run processing pipeline
-            pipe_1(
-                video_file=video_file,
-                skip_frame_extraction=False,
-                skip_meta_extraction=False,
-                skip_anonymization=False,
-            )
-
-        except Exception as e:
-            logger.warning(
-                f"Video processing pipeline failed, but VideoFile was created: {e}"
-            )
-            # Continue anyway - we might still have usable metadata
-
-        # Get the associated SensitiveMeta
-        if hasattr(video_file, "sensitive_meta") and video_file.sensitive_meta:
-            sensitive_meta = video_file.sensitive_meta
-            logger.info(
-                f"Found SensitiveMeta {sensitive_meta.id} from video processing"
-            )
-            return sensitive_meta
-        else:
-            # Create a basic SensitiveMeta if none exists
-            sensitive_meta = SensitiveMeta.objects.create()
-
-            # Link it to the video if possible
-            if hasattr(video_file, "sensitive_meta"):
-                video_file.sensitive_meta = sensitive_meta
-                video_file.save(update_fields=["sensitive_meta"])
-
-            logger.info(
-                f"Created basic SensitiveMeta {sensitive_meta.id} for video file"
-            )
-            return sensitive_meta
-
-    except Exception as e:
-        logger.exception(f"Error processing video file {file_path}: {e}")
-        raise

endoreg_db/tasks/video_ingest.py

@@ -1,161 +0,0 @@
-"""
-Video ingestion tasks using RQ (Redis Queue).
-
-This module provides background task processing for video file imports,
-allowing asynchronous processing of video files dropped into the inbox directory.
-"""
-
-import logging
-import os
-from pathlib import Path
-from typing import Optional
-
-import rq
-import django
-from django.conf import settings
-from django.core.management import call_command
-from redis import Redis
-
-logger = logging.getLogger(__name__)
-
-# Redis connection configuration
-REDIS_URL = os.getenv('RQ_REDIS_URL', 'redis://localhost:6379/0')
-
-def get_redis_connection():
-    """Get Redis connection for RQ."""
-    try:
-        return Redis.from_url(REDIS_URL)
-    except Exception as e:
-        logger.error(f"Failed to connect to Redis at {REDIS_URL}: {e}")
-        raise
-
-def get_queue():
-    """Get the 'anonymize' RQ queue."""
-    redis_conn = get_redis_connection()
-    return rq.Queue('anonymize', connection=redis_conn)
-
-def run_import_video(file_path: str, center_name: Optional[str] = None, 
-                    processor_name: Optional[str] = None) -> dict:
-    """
-    RQ task to import a video file using the video import service.
-    
-    Args:
-        file_path: Path to the video file to import
-        center_name: Optional center name (defaults to settings.DEFAULT_CENTER)
-        processor_name: Optional processor name (defaults to settings.DEFAULT_PROCESSOR)
-        
-    Returns:
-        Dict containing task result status and metadata
-    """
-    try:
-        # Ensure Django is properly set up
-        if not django.apps.apps.ready:
-            django.setup()
-        
-        file_path = Path(file_path)
-        if not file_path.exists():
-            error_msg = f"Video file not found: {file_path}"
-            logger.error(error_msg)
-            return {
-                'status': 'error',
-                'message': error_msg,
-                'file_path': str(file_path)
-            }
-        
-        # Use settings defaults if not provided
-        if not isinstance(center_name, str):
-            getattr(settings, 'DEFAULT_CENTER', 'university_hospital_wuerzburg')
-            assert(center_name is str)
-        if not isinstance(processor_name, str):
-            assert(processor_name is str)
-            getattr(settings, 'DEFAULT_PROCESSOR', 'olympus_cv_1500')
-
-        logger.info(f"Starting import for video: {file_path}")
-        logger.info(f"Using center: {center_name}, processor: {processor_name}")
-        
-        # Try to use the service first
-        try:
-            from endoreg_db.services.__old import video_import
-            
-            video_file_obj = video_import.import_and_anonymize(
-                file_path=file_path,
-                center_name=center_name,
-                processor_name=processor_name,
-                save_video=True,
-                delete_source=True
-            )
-            
-            success_msg = f"Successfully imported video: {file_path.name} (UUID: {video_file_obj.uuid})"
-            logger.info(success_msg)
-            
-            return {
-                'status': 'success',
-                'message': success_msg,
-                'file_path': str(file_path),
-                'video_uuid': str(video_file_obj.uuid),
-                'center_name': center_name,
-                'processor_name': processor_name
-            }
-            
-        except Exception as service_error:
-            logger.warning(f"Service import failed: {service_error}, falling back to management command")
-            
-            # Fallback to Django management command
-            call_command(
-                "import_video", 
-                str(file_path),
-                "--center_name", center_name,
-                "--processor_name", processor_name,
-                "--save_video_file",
-                "--delete_source"
-            )
-            
-            success_msg = f"Successfully imported video via fallback: {file_path.name}"
-            logger.info(success_msg)
-            
-            return {
-                'status': 'success',
-                'message': success_msg,
-                'file_path': str(file_path),
-                'center_name': center_name,
-                'processor_name': processor_name,
-                'method': 'fallback_command'
-            }
-        
-    except Exception as e:
-        error_msg = f"Failed to import video {file_path}: {str(e)}"
-        logger.error(error_msg, exc_info=True)
-        
-        return {
-            'status': 'error',
-            'message': error_msg,
-            'file_path': str(file_path),
-            'exception': str(e)
-        }
-
-def enqueue_video_import(file_path: str, center_name: Optional[str] = None,
-                        processor_name: Optional[str] = None) -> rq.Job:
-    """
-    Enqueue a video import task.
-    
-    Args:
-        file_path: Path to the video file to import
-        center_name: Optional center name
-        processor_name: Optional processor name
-        
-    Returns:
-        RQ Job instance
-    """
-    queue = get_queue()
-    
-    job = queue.enqueue(
-        run_import_video,
-        file_path,
-        center_name,
-        processor_name,
-        job_timeout='30m',  # 30 minute timeout for video processing
-        result_ttl=86400    # Keep results for 24 hours
-    )
-    
-    logger.info(f"Enqueued video import job {job.id} for file: {file_path}")
-    return job