endoreg-db 0.6.4__py3-none-any.whl → 0.8.1__py3-none-any.whl

endoreg_db/utils/paths.py

@@ -1,86 +1,104 @@
+"""
+Centralizes path management for the application.
+
+This module sets up all necessary directories for storage and import operations.
+It provides a unified dictionary 'data_paths' for accessing all path objects.
+"""
+
+from logging import getLogger
+logger = getLogger(__name__)
+
 import os
 from pathlib import Path
+from typing import Dict
+import dotenv
+
+# Only load .env in non-pytest contexts to avoid leaking dev settings into tests
+if not os.environ.get("PYTEST_CURRENT_TEST"):
+    dotenv.load_dotenv()
+else:
+    logger.debug("Skipping .env load under pytest")
+
+# Define BASE_DIR as the project root (endoreg_db/utils -> endoreg_db -> repo root)
+STORAGE_DIR = Path(os.getenv("STORAGE_DIR"))
+
+# Resolve STORAGE_DIR from env or default under BASE_DIR
+#def _resolve_storage_dir() -> Path:
+#     env_val = os.getenv("STORAGE_DIR")
+#     if env_val:
+#         p = Path(env_val)
+#         return p if p.is_absolute() else (BASE_DIR / p).resolve()
+#     # Do not import django.conf.settings here to avoid early settings configuration.
+#     # Fall back to a local storage directory under the repo.
+#     return (BASE_DIR / "storage").resolve()
+
+# STORAGE_DIR = _resolve_storage_dir()
+STORAGE_DIR.mkdir(parents=True, exist_ok=True)
+
+PREFIX_RAW = "raw_"
+STORAGE_DIR_NAME = "data"
+IMPORT_DIR_NAME = "import"
+EXPORT_DIR_NAME = "export"
+
+VIDEO_DIR_NAME = "videos"
+ANONYM_VIDEO_DIR_NAME = "anonym_videos" # Added for processed videos
+FRAME_DIR_NAME = "frames"
+PDF_DIR_NAME = "pdfs" # Changed from reports
+WEIGHTS_DIR_NAME = "model_weights"
+EXAMINATION_DIR_NAME = "examinations"
+
+RAW_VIDEO_DIR_NAME = f"{PREFIX_RAW}videos"
+RAW_FRAME_DIR_NAME = f"{PREFIX_RAW}frames"
+RAW_PDF_DIR_NAME = f"{PREFIX_RAW}pdfs" # Changed from reports
+
+# Define data subdirectories under STORAGE_DIR
+VIDEO_DIR = STORAGE_DIR / VIDEO_DIR_NAME
+ANONYM_VIDEO_DIR = STORAGE_DIR / ANONYM_VIDEO_DIR_NAME # Added
+FRAME_DIR = STORAGE_DIR / FRAME_DIR_NAME
+PDF_DIR = STORAGE_DIR / PDF_DIR_NAME # Changed
+WEIGHTS_DIR = STORAGE_DIR / WEIGHTS_DIR_NAME
+RAW_VIDEO_DIR = STORAGE_DIR / RAW_VIDEO_DIR_NAME
+RAW_FRAME_DIR = STORAGE_DIR / RAW_FRAME_DIR_NAME
+RAW_PDF_DIR = STORAGE_DIR / RAW_PDF_DIR_NAME # Changed
+
+IMPORT_DIR = STORAGE_DIR / IMPORT_DIR_NAME
+VIDEO_IMPORT_DIR = IMPORT_DIR / VIDEO_DIR_NAME
+FRAME_IMPORT_DIR = IMPORT_DIR / FRAME_DIR_NAME
+PDF_IMPORT_DIR = IMPORT_DIR / PDF_DIR_NAME # Changed
+WEIGHTS_IMPORT_DIR = IMPORT_DIR / WEIGHTS_DIR_NAME
+
+EXPORT_DIR = STORAGE_DIR / EXPORT_DIR_NAME
+
+data_paths:Dict[str,Path] = {
+    "storage": STORAGE_DIR,
+    "video": VIDEO_DIR,
+    "anonym_video": ANONYM_VIDEO_DIR, # Added
+    "frame": FRAME_DIR,
+    "pdf": PDF_DIR, # Changed
+    "import": IMPORT_DIR,
+    "video_import": VIDEO_IMPORT_DIR,
+    "frame_import": FRAME_IMPORT_DIR,
+    "pdf_import": PDF_IMPORT_DIR, # Changed
+    "raw_video": RAW_VIDEO_DIR,
+    "raw_frame": RAW_FRAME_DIR,
+    "raw_pdf": RAW_PDF_DIR, # Changed
+    "weights": WEIGHTS_DIR,
+    "weights_import": WEIGHTS_IMPORT_DIR,
+    "export": EXPORT_DIR,
+    "video_export": EXPORT_DIR / VIDEO_DIR_NAME,
+    "anonym_video_export": EXPORT_DIR / ANONYM_VIDEO_DIR_NAME, # Added
+    "frame_export": EXPORT_DIR / FRAME_DIR_NAME,
+    "pdf_export": EXPORT_DIR / PDF_DIR_NAME, # Changed
+    "weights_export": EXPORT_DIR / WEIGHTS_DIR_NAME,
+    "examination_export": EXPORT_DIR / EXAMINATION_DIR_NAME,
+    "raw_video_export": EXPORT_DIR / RAW_VIDEO_DIR_NAME,
+    "raw_frame_export": EXPORT_DIR / RAW_FRAME_DIR_NAME,
+    "raw_pdf_export": EXPORT_DIR / RAW_PDF_DIR_NAME, # Changed
+}
 
-STORAGE_DIR_NAME = os.environ.get("DJANGO_STORAGE_DIR_NAME", "data")
-FRAME_DIR_NAME = os.environ.get("DJANGO_FRAME_DIR_NAME", "db_frames")
-VIDEO_DIR_NAME = os.environ.get("DJANGO_VIDEO_DIR_NAME", "db_videos")
-WEIGHTS_DIR_NAME = os.environ.get("DJANGO_WEIGHTS_DIR_NAME", "db_model_weights")
-PDF_DIR_NAME = os.environ.get("DJANGO_PDF_DIR_NAME", "pdfs")
-
-def get_storage_dir(raw: bool = False):
-    """
-    Get the storage directory from the environment variable or settings.
-    """
-    storage_dir = os.environ.get("DJANGO_STORAGE_DIR", "data")
-    storage_dir = Path(storage_dir).expanduser()
-
-    if raw:
-        name = storage_dir.name
-        storage_dir = storage_dir.parent / f"raw_{name}"
-
-
-    if not storage_dir.exists():
-        storage_dir.mkdir(parents=True, exist_ok=True)
-    return storage_dir
-
-def get_frame_dir(raw: bool = False):
-    """
-    Get the frame directory from the environment variable or settings.
-    """
-    frame_dir_name = os.environ.get("DJANGO_FRAME_DIR_NAME", FRAME_DIR_NAME)
-    storage_dir = get_storage_dir(raw)
-    frame_dir = storage_dir / frame_dir_name
-    if not frame_dir.exists():
-        frame_dir.mkdir(parents=True, exist_ok=True)
-    return frame_dir
-
-def get_video_dir(raw: bool = False):
-    """
-    Get the video directory from the environment variable or settings.
-    """
-    video_dir_name = os.environ.get("DJANGO_VIDEO_DIR_NAME", VIDEO_DIR_NAME)
-    storage_dir = get_storage_dir(raw)
-    video_dir = storage_dir / video_dir_name
-    if not video_dir.exists():
-        video_dir.mkdir(parents=True, exist_ok=True)
-    return video_dir
-
-def get_weights_dir():
-    """
-    Get the weights directory from the environment variable or settings.
-    """
-    weights_dir_name = os.environ.get("DJANGO_WEIGHTS_DIR_NAME", WEIGHTS_DIR_NAME)
-    storage_dir = get_storage_dir()
-    weights_dir = storage_dir / weights_dir_name
-    if not weights_dir.exists():
-        weights_dir.mkdir(parents=True, exist_ok=True)
-    return weights_dir
-
-def get_pdf_dir(raw: bool = False):
-    """
-    Get the pdf directory from the environment variable or settings.
-    """
-    pdf_dir_name = os.environ.get("DJANGO_PDF_DIR_NAME", PDF_DIR_NAME)
-    storage_dir = get_storage_dir(raw)
-    pdf_dir = storage_dir / pdf_dir_name
-    if not pdf_dir.exists():
-        pdf_dir.mkdir(parents=True, exist_ok=True)
-    return pdf_dir
-
-STORAGE_DIR = get_storage_dir()
-FRAME_DIR = get_frame_dir()
-VIDEO_DIR = get_video_dir()
-PDF_DIR = get_pdf_dir()
-PDF_DIR_NAME = PDF_DIR.name
-RAW_VIDEO_DIR = get_video_dir(raw=True)
-RAW_VIDEO_DIR_NAME = RAW_VIDEO_DIR.name
-RAW_FRAME_DIR = get_frame_dir(raw=True)
-RAW_FRAME_DIR_NAME = RAW_FRAME_DIR.name
-RAW_PDF_DIR = get_pdf_dir(raw=True)
-RAW_PDF_DIR_NAME = RAW_PDF_DIR.name
-WEIGHTS_DIR = get_weights_dir()
-TEST_RUN = os.environ.get("TEST_RUN", False)
-TEST_RUN_FRAME_NUMBER = os.environ.get("TEST_RUN_FRAME_NUMBER", 1000)
-# AI Stuff
-FRAME_PROCESSING_BATCH_SIZE = os.environ.get("DJANGO_FRAME_PROCESSING_BATCH_SIZE", 10)
+logger.info(f"Storage directory: {STORAGE_DIR.resolve()}")
+logger.info(f"Export directory: {EXPORT_DIR.resolve()}")
 
+for key, path in data_paths.items():
+    path.mkdir(parents=True, exist_ok=True)
+    logger.info(f"{key.capitalize()} directory: {path.resolve()}")

endoreg_db/utils/permissions.py

@@ -0,0 +1,143 @@
+"""
+Dynamic permission utilities that adapt to environment settings.
+
+This module provides permission classes that automatically adjust based on
+the DEBUG setting and other environment configurations.
+"""
+
+from django.conf import settings
+from rest_framework.permissions import IsAuthenticated, AllowAny, BasePermission
+from rest_framework.decorators import permission_classes as drf_permission_classes
+from functools import wraps
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+class DynamicAuthPermission:
+    """
+    Permission class that adapts based on environment settings.
+    
+    - In DEBUG mode: Allows access without authentication
+    - In production (DEBUG=False): Requires authentication
+    """
+    
+    @staticmethod
+    def get_permission_classes():
+        """
+        Returns appropriate permission classes based on current settings.
+        
+        Returns:
+            list: Permission classes to use
+        """
+        if getattr(settings, 'DEBUG', False):
+            logger.info("DEBUG mode detected - allowing unauthenticated access")
+            return [AllowAny]
+        else:
+            logger.info("Production mode detected - requiring authentication")
+            return [IsAuthenticated]
+
+
+def dynamic_permission_classes(force_auth=None):
+    """
+    Decorator that applies permission classes based on environment settings.
+    
+    Args:
+        force_auth (bool, optional): 
+            - True: Always require authentication regardless of DEBUG
+            - False: Always allow access regardless of DEBUG  
+            - None: Use environment-based logic (default)
+    """
+    def decorator(view_func):
+        @wraps(view_func)
+        def wrapper(*args, **kwargs):
+            return view_func(*args, **kwargs)
+        
+        # Determine permission classes
+        if force_auth is True:
+            permission_cls = [IsAuthenticated]
+            logger.info(f"View {view_func.__name__} - forced authentication required")
+        elif force_auth is False:
+            permission_cls = [AllowAny]
+            logger.info(f"View {view_func.__name__} - forced public access")
+        else:
+            permission_cls = DynamicAuthPermission.get_permission_classes()
+            logger.info(f"View {view_func.__name__} - dynamic permissions: {permission_cls}")
+        
+        # Apply the permission classes decorator
+        return drf_permission_classes(permission_cls)(wrapper)
+    
+    return decorator
+
+
+def get_auth_required():
+    """
+    Simple function to check if authentication is required in current environment.
+    
+    Returns:
+        bool: True if authentication is required, False otherwise
+    """
+    return not getattr(settings, 'DEBUG', False)
+
+
+def is_debug_mode():
+    """
+    Robustly determine if debug mode is enabled, checking both Django settings and environment variable.
+    Also treats active pytest sessions as debug to simplify API tests.
+    """
+    truthy = {"1", "true", "yes", "on"}
+    env_debug = str(os.environ.get("DJANGO_DEBUG", "false")).lower() in truthy
+    settings_debug = bool(getattr(settings, 'DEBUG', False))
+    pytest_active = "PYTEST_CURRENT_TEST" in os.environ
+    result = settings_debug or env_debug or pytest_active
+    logger.info(f"is_debug_mode: env={env_debug}, settings={settings_debug}, pytest={pytest_active}, result={result}")
+    return result
+
+# Compute default permission classes each call to avoid stale values during tests
+
+def get_debug_permissions():
+    return [AllowAny] if is_debug_mode() else [IsAuthenticated]
+
+# Export a name for convenience, but prefer calling get_debug_permissions() in views
+DEBUG_PERMISSIONS = get_debug_permissions()
+ALWAYS_AUTH_PERMISSIONS = [IsAuthenticated]
+ALWAYS_PUBLIC_PERMISSIONS = [AllowAny]
+
+# Log the current permission mode
+if is_debug_mode():
+    logger.info("🔓 Authentication disabled for DEBUG mode (robust check)")
+else:
+    logger.info("🔒 Authentication required for production mode (robust check)")
+
+
+class EnvironmentAwarePermission(BasePermission):
+    """
+    Custom permission class that can be used directly in DRF views.
+    Honors both Django settings.DEBUG and DJANGO_DEBUG env var.
+    """
+    
+    def has_permission(self, request, view):
+        """
+        Check if the request has permission.
+        
+        Args:
+            request: The Django request object
+            view: The DRF view
+            
+        Returns:
+            bool: True if permission granted, False otherwise
+        """
+        if is_debug_mode():
+            logger.debug(f"DEBUG mode - granting access to {view.__class__.__name__}")
+            return True
+        # In production, require authentication
+        is_authenticated = bool(getattr(request, 'user', None) and request.user.is_authenticated)
+        logger.debug(f"Production mode - authentication check for {view.__class__.__name__}: {is_authenticated}")
+        return is_authenticated
+    
+    def has_object_permission(self, request, view, obj):
+        """
+        Object-level permission check.
+        """
+        return self.has_permission(request, view)

endoreg_db/utils/pipelines/Readme.md

@@ -0,0 +1,235 @@
+# Pipeline Documentation for `process_video_dir.py`
+
+**File Path:** `endoreg_db/utils/pipelines/process_video_dir.py`
+
+---
+
+> This document outlines the multi-stage video processing pipeline implemented in the `process_video_dir.py` module, covering registration, validation, anonymization, and final state outcomes.
+
+
+
+## 0. Start: Registering a New Video (before any pipeline)
+
+- A new `VideoFile` record is created to register the video file.
+- Linked to a `Center` and an `EndoscopyProcessor`.
+- A `VideoMeta` object is created (or updated) by reading the technical information from the raw video (e.g., **fps**, **resolution**, **frame count**, **duration**).
+- `Frame` objects are initialized in the database — one for each expected frame, but they are not yet extracted.
+- A `VideoState` object is created to track the processing progress for this video.
+- **No frames are extracted yet. No sensitive information or AI predictions exist yet.**
+### Tables Involved
+
+| Table Name   |
+|--------------|
+| VideoFile    |
+| VideoMeta    |
+| Frame        |
+| VideoState   |
+
+## Pipeline 1 Steps
+
+---
+
+### Step 1: Ensure VideoState Exists (`_pipe_1_step_ensure_state`)
+- Ensures a tracking object (`VideoState`) exists for this video.
+
+---
+
+### Step 2: Update or Create VideoMeta (`_pipe_1_step_update_meta`)
+- Extract technical metadata from the raw file.
+- Update or create the `VideoMeta` object.
+
+---
+
+### Step 3: Extract Frames (`_pipe_1_step_extract_frames`)
+- Using `ffmpeg`, the raw video is split into frames.
+- Frames are saved to disk.
+- `Frame` database entries are updated: `is_extracted=True`.
+- `VideoState.frames_extracted` is set to `True`.
+
+---
+
+### Step 4: Extract Text (OCR) (`_pipe_1_step_extract_text`)
+- A random sample of frames is passed through OCR.
+- Text such as **Patient Name**, **Date of Birth (DOB)**, **IDs** is extracted.
+- A `SensitiveMeta` object is created and filled with this information.
+- `VideoState.text_meta_extracted` is set to `True`.
+
+---
+
+### Step 5: Predict Labels (AI Prediction) (`_pipe_1_step_predict`)
+- An AI model is run on frames to predict medical labels (e.g., **polyp**, **normal**, **outside**).
+- Smoothed predictions are generated over frames.
+- A `VideoPredictionMeta` is created to store AI model info and predictions.
+- `VideoState.initial_prediction_completed` is set to `True`.
+
+---
+
+### Step 6: Create Label Segments (`_pipe_1_step_create_segments`)
+- Predictions are converted into labeled segments (`LabelVideoSegment`).
+- Each segment is linked to frame ranges and specific labels.
+- A `LabelVideoSegmentState` is created for each segment to track validation.
+- `VideoState.lvs_created` is set to `True`.
+
+---
+## End of PIPE 1
+
+At this point in the pipeline:
+
+- Frames are **extracted** and **saved**.
+- OCR **text is extracted** and **stored**.
+- AI **predictions are made**.
+- **Label segments are created** based on predictions.
+- **Sensitive information** (e.g. patient identifiers) is stored.
+- `VideoState` is updated to reflect all completed steps.
+
+---
+
+### Tables Involved
+
+| Table Name               |
+|--------------------------|
+| `VideoFile`              |
+| `VideoMeta`              |
+| `Frame`                  |
+| `VideoState`             |
+| `SensitiveMeta`          |
+| `VideoPredictionMeta`    |
+| `LabelVideoSegment`      |
+| `LabelVideoSegmentState` |
+
+---
+## 2. VALIDATION Phase: Review and Approval
+
+**Goal:** Confirm that automated results are correct before anonymization.
+
+### Steps
+
+- A human reviewer (or automated test) opens each `LabelVideoSegment` and reviews it.
+- Each segment’s state is updated:  
+  `LabelVideoSegmentState.is_validated = True`
+- The extracted sensitive metadata (e.g., names, DOB) in `SensitiveMeta` is reviewed.
+- Sensitive metadata state fields are set:  
+  `dob_verified = True`  
+  `names_verified = True`
+- If **any** of these validations fail, processing **must stop** and be corrected.
+- **Only** when **all** validations are successfully marked, the video is allowed to proceed to **PIPE 2**.
+
+---
+
+### Tables Involved
+
+| Table Name               |
+|--------------------------|
+| `LabelVideoSegmentState` |
+| `SensitiveMeta`          |
+
+---
+## 3. PIPELINE 2: Anonymization and Cleanup
+
+**Goal:** Prepare a safe, anonymized version of the video.
+
+---
+
+### Pipeline 2 Major Steps
+
+---
+
+#### Step 1: Ensure VideoState Exists Again (`_pipe_2_step_ensure_state`)
+- Confirm that the `VideoState` tracking object is still present.
+
+---
+
+#### Step 2: Ensure Frames Available (`_pipe_2_step_extract_frames`)
+- Check if extracted frames are available.
+- If frames were deleted or are missing, **re-extract** them from the raw video file.
+
+---
+
+#### Step 3: Anonymize Video (`_pipe_2_step_anonymize`)
+- Frames labeled as `"outside"` are blacked out.
+- Remaining frames are masked using **endoscopy ROI** (Region of Interest).
+- New anonymized frames are created and saved temporarily.
+- A new anonymized video is assembled from these frames.
+- New video file path is saved in:  
+  `VideoFile.processed_file`
+- Video hash is calculated and saved in:  
+  `VideoFile.processed_video_hash`
+- `VideoState.anonymized = True`
+
+---
+
+#### Step 4: Delete Raw Files and Frames
+- Raw video file and extracted frames are **permanently deleted**.
+- `VideoState.frames_extracted = False`
+
+---
+
+#### Step 5: Delete SensitiveMeta (`_pipe_2_step_delete_sensitive_meta`)
+- The `SensitiveMeta` object is deleted from the database.
+- `VideoFile.sensitive_meta = None`
+
+---
+
+### At the End of PIPE 2:
+
+- Only the **anonymized processed video** remains.
+- **All sensitive data** and **raw video/frames** are permanently deleted.
+- **Technical metadata** and **AI prediction audit data** are retained.
+
+---
+
+### Tables Involved
+
+| Table Name      | Notes                            |
+|------------------|----------------------------------|
+| `VideoFile`      | Updated with processed file info |
+| `Frame`          | Frames used, then deleted        |
+| `VideoState`     | Tracks anonymization status      |
+| `SensitiveMeta`  | **Deleted after anonymization**  |
+
+---
+## 4. FINAL STATE
+
+### Database Contains:
+
+- The **processed, safe, anonymized video**:  
+  `VideoFile.processed_file`
+- **Technical information** about the video:  
+  `VideoMeta`
+- **Medical label segments** for audit:  
+  `LabelVideoSegment` and `LabelVideoSegmentState`
+- **AI prediction information** for audit:  
+  `VideoPredictionMeta`
+- **Sensitive patient information** has been **permanently deleted**
+- **Raw extracted frames** and the **original raw video file** have been **permanently deleted**
+
+---
+
+### `VideoState` Summary:
+
+- `frames_extracted = False`  *(raw frames are gone)*
+- `anonymized = True`  *(anonymized video is ready)*
+
+---
+
+### Tables Overview
+
+| Table Name              | Purpose                                                                 |
+|--------------------------|-------------------------------------------------------------------------|
+| `VideoFile`              | Main entry linking video paths, processed file, and metadata references |
+| `VideoMeta`              | Technical information (fps, duration, frame count, etc.)                |
+| `Frame`                  | Individual frame images from the video *(deleted in PIPE 2)*            |
+| `VideoState`             | Tracks pipeline processing steps and their completion                   |
+| `SensitiveMeta`          | Patient-sensitive data from OCR *(deleted in PIPE 2)*                   |
+| `VideoPredictionMeta`    | Information about the AI model and predictions                          |
+| `LabelVideoSegment`      | Video segments created from AI predictions                              |
+| `LabelVideoSegmentState` | Tracks human validation status for each segment                         |
+
+---
+### Note: on Frame Deletion and Data Integrity
+
+Although the frames are permanently deleted after the entire execution of Pipeline 1 and Pipeline 2, the necessary data and information about these frames are still populated in the database. Therefore, a script is required to utilize the correct imported data and **re-extract potential frames** from the original video.
+
+This process ensures that the required frame data can be retrieved and used even after deletion.
+
+

endoreg_db/utils/pipelines/process_video_dir.py

@@ -0,0 +1,120 @@
+from endoreg_db.models import (
+    VideoFile, Center, EndoscopyProcessor, AiModel, ModelMeta,
+    LabelVideoSegment, SensitiveMeta
+)
+from pathlib import Path
+from icecream import ic
+from tqdm import tqdm
+import os
+from django.core.management import call_command
+import logging
+from django.core.exceptions import ObjectDoesNotExist
+
+logger = logging.getLogger(__name__)
+# Usage
+# from endoreg_db.utils.pipelines.process_video_dir import process_video_dir
+# (video_dir=Path("/path/to/your/videos"))
+
+# DEFAULT_DIR = Path(__file__).resolve().parent.parent.parent.parent / "data/raw_videos"
+DEFAULT_DIR = Path(__file__).resolve().parent.parent.parent.parent / "data/study_data"
+CENTER_NAME = os.environ.get("CENTER_NAME", "university_hospital_wuerzburg")
+ENDOSCOPY_PROCESSOR_NAME = os.environ.get("ENDOSCOPY_PROCESSOR_NAME", "olympus_cv_1500")
+MODEL_NAME = "image_multilabel_classification_colonoscopy_default"
+MODEL_WEIGHTS_PATH = "./tests/assets/colo_segmentation_RegNetX800MF_6.ckpt"
+ic(DEFAULT_DIR)
+
+
+
+def process_video_dir(video_dir: Path=DEFAULT_DIR, center_name: str = CENTER_NAME, endoscopy_processor_name: str = ENDOSCOPY_PROCESSOR_NAME):
+    """
+    Process a directory of video files and create VideoFile objects in the database.
+    
+    Args:
+        video_dir (Path): Path to the directory containing video files.
+        center_name (str): Name of the center to associate with the video files.
+        endoscopy_processor_name (str): Name of the endoscopy processor to associate with the video files.
+    """
+    # Make sure the ai model exists:
+    ai_model = AiModel.objects.filter(name=MODEL_NAME)
+    if ai_model:
+        ai_model = ai_model.first()
+    try:
+        model_meta = ai_model.get_latest_version()
+        assert isinstance(model_meta, ModelMeta), "No ModelMeta found in the database."
+    except Exception:
+        logger.warning(f"ModelMeta not found for {MODEL_NAME}. Creating a new one.")
+        # Create a new ModelMeta instance
+        call_command(
+            "create_multilabel_model_meta",
+            "--model_path",
+            MODEL_WEIGHTS_PATH,
+        )
+
+
+
+    # Get the center and endoscopy processor objects
+
+    # Iterate through all files in the directory
+    for file_path in tqdm(video_dir.iterdir()):
+        logger.info(f"Processing file: {file_path}")
+        if file_path.is_file() and file_path.suffix in ['.mp4', '.avi', '.mov']:
+            # Create a VideoFile object
+            video_file = VideoFile.create_from_file_initialized(
+                file_path=file_path,
+                center_name=center_name,
+                processor_name=endoscopy_processor_name,
+            )
+            logger.warning(f"Processing video file: {video_file}")
+            try:
+                success_pipe_1 = video_file.pipe_1(
+                    model_name = MODEL_NAME,
+                )
+                assert success_pipe_1, f"Pipe 1 failed for video {video_file.uuid}"
+            except Exception as e:
+                logger.error(f"Pipe 1 failed for video {video_file.uuid}: {e}", exc_info=True)
+                raise e
+
+            video_file.refresh_from_db()
+
+            ####### SIMULATION OF VALIDATION #######
+            # Simulate the validation process for video segments
+            outside_segments = video_file.get_outside_segments()
+            logger.warning(f"Outside segments found for simulation: {outside_segments}")
+            if outside_segments:
+                for segment in outside_segments:
+                    try:
+                        seg_state = segment.state
+                        seg_state.is_validated = True
+                        seg_state.save()
+                        logger.info(f"Simulated validation for segment {segment.id}")
+                    except LabelVideoSegment.state.RelatedObjectDoesNotExist:
+                        logger.warning(f"Cannot simulate validation for segment {segment.id}: Related state object does not exist.")
+                    except Exception as e:
+                        logger.error(f"Error during segment state simulation for segment {segment.id}: {e}", exc_info=True)
+
+
+            # Simulate the validation process for the video files sensitive meta obj
+            if video_file.sensitive_meta:
+                try:
+                    sm_state = video_file.sensitive_meta.state
+                    sm_state.dob_verified = True
+                    sm_state.names_verified = True
+                    sm_state.save()
+                    logger.info(f"Simulated validation for sensitive meta of video {video_file.uuid}")
+                except Exception as e:
+                    logger.error(f"Error during sensitive meta state simulation for video {video_file.uuid}: {e}", exc_info=True)
+
+            else:
+                logger.warning(f"Cannot simulate validation for sensitive meta of video {video_file.uuid}: SensitiveMeta object does not exist (likely due to pipe_1 failure or incomplete text extraction).")
+
+            ######## SIMULATION OF VALIDATION ########
+
+            video_file.refresh_from_db()
+            if success_pipe_1:
+                try:
+                    video_file.pipe_2()
+                except Exception as e:
+                    logger.error(f"Pipe 2 failed for video {video_file.uuid}: {e}", exc_info=True)
+            else:
+                logger.warning(f"Skipping Pipe 2 for video {video_file.uuid} because Pipe 1 did not complete successfully.")
+