endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/utils/paths.py

@@ -0,0 +1,159 @@
+"""
+Centralizes path management for the application.
+
+This module sets up all necessary directories for storage and import operations.
+It provides a unified dictionary 'data_paths' for accessing all path objects.
+"""
+
+from logging import getLogger
+
+logger = getLogger(__name__)
+
+import os
+from pathlib import Path
+from typing import Dict, Union
+
+# Alternative approach using env_path helper, deprecated since monorepo setup. Alright for single install, env is always preferred.
+# from endoreg_db.config.env import env_path
+
+STORAGE_DIR = Path(os.getenv("STORAGE_DIR", "internal_storage"))
+
+storage_dir_env = os.getenv("STORAGE_DIR")
+if storage_dir_env is None:
+    raise RuntimeError("STORAGE_DIR environment variable is not set.")
+storage_dir = Path(storage_dir_env)
+STORAGE_DIR = storage_dir
+if not STORAGE_DIR.exists():
+    STORAGE_DIR.mkdir(parents=True, exist_ok=True)
+
+
+PREFIX_RAW = "raw_"
+io_dir_env = Path(os.getenv("IO_DIR", "data"))
+io_dir = Path(io_dir_env)
+IO_DIR = io_dir
+if not STORAGE_DIR.exists():
+    STORAGE_DIR.mkdir(parents=True, exist_ok=True)
+
+
+# Data dropoff folders - These can be external, determined by IO_DIR (Default: set to desktop root folder of OS)
+
+IMPORT_DIR_NAME = "import"
+EXPORT_DIR_NAME = "export"
+
+IMPORT_DIR = IO_DIR / IMPORT_DIR_NAME  # data/import
+EXPORT_DIR = IO_DIR / EXPORT_DIR_NAME  # data/export
+
+
+IMPORT_VIDEO_DIR_NAME = "video_import"
+REPORT_IMPORT_DIR_NAME = "report_import"
+
+VIDEO_EXPORT_DIR_NAME = "video_export"
+REPORT_EXPORT_DIR_NAME = "report_export"
+
+
+IMPORT_VIDEO_DIR = IMPORT_DIR / IMPORT_VIDEO_DIR_NAME
+IMPORT_REPORT_DIR = IMPORT_DIR / REPORT_IMPORT_DIR_NAME
+
+
+VIDEO_EXPORT_DIR = EXPORT_DIR / VIDEO_EXPORT_DIR_NAME
+REPORT_EXPORT_DIR = EXPORT_DIR / REPORT_EXPORT_DIR_NAME
+
+# Document Dir
+
+DOCUMENT_DIR = STORAGE_DIR / "documents"
+
+# After initial import, files will remain here.
+
+TRANSCODING_DIR = STORAGE_DIR / "temp"
+
+SENSITIVE_VIDEO_DIR_NAME = "sensitive_videos"
+SENSITIVE_REPORT_DIR_NAME = "sensitive_reports"
+ANONYM_VIDEO_DIR_NAME = "processed_videos_final"
+ANONYM_REPORT_DIR_NAME = "processed_reports_final"
+
+RAW_FRAME_DIR_NAME = f"{PREFIX_RAW}frames"
+FRAME_DIR_NAME = "frames"
+WEIGHTS_DIR_NAME = "model_weights"
+EXAMINATION_DIR_NAME = "examinations"
+
+# Define data subdirectories under STORAGE_DIR
+ANONYM_VIDEO_DIR = STORAGE_DIR / ANONYM_VIDEO_DIR_NAME
+SENSITIVE_VIDEO_DIR = STORAGE_DIR / SENSITIVE_VIDEO_DIR_NAME
+ANONYM_REPORT_DIR = STORAGE_DIR / ANONYM_REPORT_DIR_NAME
+SENSITIVE_REPORT_DIR = STORAGE_DIR / SENSITIVE_REPORT_DIR_NAME
+
+FRAME_DIR = STORAGE_DIR / FRAME_DIR_NAME
+
+
+WEIGHTS_DIR = STORAGE_DIR / WEIGHTS_DIR_NAME
+RAW_FRAME_DIR = STORAGE_DIR / RAW_FRAME_DIR_NAME
+
+WEIGHTS_IMPORT_DIR = IMPORT_DIR / WEIGHTS_DIR_NAME
+WEIGHTS_EXPORT_DIR = EXPORT_DIR / WEIGHTS_DIR_NAME
+
+FRAME_IMPORT_DIR = IMPORT_DIR / FRAME_DIR_NAME
+
+FRAME_EXPORT_DIR = EXPORT_DIR / FRAME_DIR_NAME
+
+
+data_paths: Dict[str, Path] = {
+    "storage": STORAGE_DIR,
+    "import": IMPORT_DIR,
+    "import_video": IMPORT_VIDEO_DIR,
+    "sensitive_video": SENSITIVE_VIDEO_DIR,
+    "sensitive_report": SENSITIVE_REPORT_DIR,
+    "anonym_video": ANONYM_VIDEO_DIR,
+    "anonym_report": ANONYM_REPORT_DIR,
+    "import_frame": FRAME_IMPORT_DIR,
+    "import_report": IMPORT_REPORT_DIR,
+    "raw_frame": RAW_FRAME_DIR,
+    "weights": WEIGHTS_DIR,
+    "weights_import": WEIGHTS_IMPORT_DIR,
+    "export": EXPORT_DIR,
+    "report_export": REPORT_EXPORT_DIR,
+    "video_export": VIDEO_EXPORT_DIR,
+    "frame_export": FRAME_EXPORT_DIR,
+    "weights_export": EXPORT_DIR / WEIGHTS_DIR_NAME,
+    "transcoding": TRANSCODING_DIR,
+    "frame": FRAME_DIR,
+    "documents": DOCUMENT_DIR,
+}
+
+logger.info(f"Storage directory: {STORAGE_DIR.resolve()}")
+logger.info(f"Export directory: {EXPORT_DIR.resolve()}")
+
+for key, path in data_paths.items():
+    path.mkdir(parents=True, exist_ok=True)
+
+    logger.info(f"{key.capitalize()} directory: {path.resolve()}")
+
+
+def to_storage_relative(path: Union[str, Path]) -> str:
+    """
+    Return a path string relative to STORAGE_DIR, suitable for Django FileField.name.
+
+    - If `path` is inside STORAGE_DIR (absolute or contains STORAGE_DIR as prefix),
+      we strip the STORAGE_DIR prefix and return the relative part.
+    - If `path` is outside STORAGE_DIR, we return it as a string unchanged
+      (caller can decide what to do).
+    """
+    # Normalize input to Path
+    p = Path(path)
+
+    # Resolve absolute path for comparison
+    storage_root = STORAGE_DIR.resolve()
+
+    if not p.is_absolute():
+        # Resolve relative path against current working directory
+        # (in tests, cwd should be project root, so this still lands under STORAGE_DIR)
+        p = p.resolve()
+
+    try:
+        rel = p.relative_to(storage_root)
+    except ValueError:
+        # Not under STORAGE_DIR – probably already a relative name or external.
+        # In that case, just return the string representation as-is.
+        return str(path)
+
+    # Use POSIX-style separators for Django FileField
+    return rel.as_posix()

endoreg_db/utils/permissions.py

@@ -0,0 +1,160 @@
+"""
+Dynamic permission utilities that adapt to environment settings.
+
+This module provides permission classes that automatically adjust based on
+the DEBUG setting and other environment configurations.
+"""
+
+from django.conf import settings
+from rest_framework.permissions import IsAuthenticated, AllowAny, BasePermission
+
+# from rest_framework.decorators import permission_classes as drf_permission_classes
+from functools import wraps
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+class DynamicAuthPermission:
+    """
+    Permission class that adapts based on environment settings.
+
+    - In DEBUG mode: Allows access without authentication
+    - In production (DEBUG=False): Requires authentication
+    """
+
+    @staticmethod
+    def get_permission_classes():
+        """
+        Returns appropriate permission classes based on current settings.
+
+        Returns:
+            list: Permission classes to use
+        """
+        if getattr(settings, "DEBUG", False):
+            logger.info("DEBUG mode detected - allowing unauthenticated access")
+            return [AllowAny]
+        else:
+            logger.info("Production mode detected - requiring authentication")
+            return [IsAuthenticated]
+
+
+def dynamic_permission_classes(force_auth=None):
+    """
+    Decorator that applies permission classes based on environment settings.
+
+    Args:
+        force_auth (bool, optional):
+            - True: Always require authentication regardless of DEBUG
+            - False: Always allow access regardless of DEBUG
+            - None: Use environment-based logic (default)
+    """
+
+    def decorator(view_func):
+        from rest_framework.decorators import (
+            permission_classes as drf_permission_classes,
+        )
+
+        @wraps(view_func)
+        def wrapper(*args, **kwargs):
+            return view_func(*args, **kwargs)
+
+        # Determine permission classes
+        if force_auth is True:
+            permission_cls = [IsAuthenticated]
+            logger.info(f"View {view_func.__name__} - forced authentication required")
+        elif force_auth is False:
+            permission_cls = [AllowAny]
+            logger.info(f"View {view_func.__name__} - forced public access")
+        else:
+            permission_cls = DynamicAuthPermission.get_permission_classes()
+            logger.info(
+                f"View {view_func.__name__} - dynamic permissions: {permission_cls}"
+            )
+
+        # Apply the permission classes decorator
+        return drf_permission_classes(permission_cls)(wrapper)
+
+    return decorator
+
+
+def get_auth_required():
+    """
+    Simple function to check if authentication is required in current environment.
+
+    Returns:
+        bool: True if authentication is required, False otherwise
+    """
+    return not getattr(settings, "DEBUG", False)
+
+
+def is_debug_mode():
+    """
+    Robustly determine if debug mode is enabled, checking both Django settings and environment variable.
+    Also treats active pytest sessions as debug to simplify API tests.
+    """
+    truthy = {"1", "true", "yes", "on"}
+    env_debug = str(os.environ.get("DJANGO_DEBUG", "false")).lower() in truthy
+    settings_debug = bool(getattr(settings, "DEBUG", False))
+    pytest_active = "PYTEST_CURRENT_TEST" in os.environ
+    result = settings_debug or env_debug or pytest_active
+    logger.info(
+        f"is_debug_mode: env={env_debug}, settings={settings_debug}, pytest={pytest_active}, result={result}"
+    )
+    return result
+
+
+# Compute default permission classes each call to avoid stale values during tests
+
+
+def get_debug_permissions():
+    return [AllowAny] if is_debug_mode() else [IsAuthenticated]
+
+
+# Export a name for convenience, but prefer calling get_debug_permissions() in views
+DEBUG_PERMISSIONS = get_debug_permissions()
+ALWAYS_AUTH_PERMISSIONS = [IsAuthenticated]
+ALWAYS_PUBLIC_PERMISSIONS = [AllowAny]
+
+# Log the current permission mode
+if is_debug_mode():
+    logger.info("🔓 Authentication disabled for DEBUG mode (robust check)")
+else:
+    logger.info("🔒 Authentication required for production mode (robust check)")
+
+
+class EnvironmentAwarePermission(BasePermission):
+    """
+    Custom permission class that can be used directly in DRF views.
+    Honors both Django settings.DEBUG and DJANGO_DEBUG env var.
+    """
+
+    def has_permission(self, request, view):
+        """
+        Check if the request has permission.
+
+        Args:
+            request: The Django request object
+            view: The DRF view
+
+        Returns:
+            bool: True if permission granted, False otherwise
+        """
+        if is_debug_mode():
+            logger.debug(f"DEBUG mode - granting access to {view.__class__.__name__}")
+            return True
+        # In production, require authentication
+        is_authenticated = bool(
+            getattr(request, "user", None) and request.user.is_authenticated
+        )
+        logger.debug(
+            f"Production mode - authentication check for {view.__class__.__name__}: {is_authenticated}"
+        )
+        return is_authenticated
+
+    def has_object_permission(self, request, view, obj):
+        """
+        Object-level permission check.
+        """
+        return self.has_permission(request, view)

endoreg_db/utils/pipelines/Readme.md

@@ -0,0 +1,235 @@
+# Pipeline Documentation for `process_video_dir.py`
+
+**File Path:** `endoreg_db/utils/pipelines/process_video_dir.py`
+
+---
+
+> This document outlines the multi-stage video processing pipeline implemented in the `process_video_dir.py` module, covering registration, validation, anonymization, and final state outcomes.
+
+
+
+## 0. Start: Registering a New Video (before any pipeline)
+
+- A new `VideoFile` record is created to register the video file.
+- Linked to a `Center` and an `EndoscopyProcessor`.
+- A `VideoMeta` object is created (or updated) by reading the technical information from the raw video (e.g., **fps**, **resolution**, **frame count**, **duration**).
+- `Frame` objects are initialized in the database — one for each expected frame, but they are not yet extracted.
+- A `VideoState` object is created to track the processing progress for this video.
+- **No frames are extracted yet. No sensitive information or AI predictions exist yet.**
+### Tables Involved
+
+| Table Name   |
+|--------------|
+| VideoFile    |
+| VideoMeta    |
+| Frame        |
+| VideoState   |
+
+## Pipeline 1 Steps
+
+---
+
+### Step 1: Ensure VideoState Exists (`_pipe_1_step_ensure_state`)
+- Ensures a tracking object (`VideoState`) exists for this video.
+
+---
+
+### Step 2: Update or Create VideoMeta (`_pipe_1_step_update_meta`)
+- Extract technical metadata from the raw file.
+- Update or create the `VideoMeta` object.
+
+---
+
+### Step 3: Extract Frames (`_pipe_1_step_extract_frames`)
+- Using `ffmpeg`, the raw video is split into frames.
+- Frames are saved to disk.
+- `Frame` database entries are updated: `is_extracted=True`.
+- `VideoState.frames_extracted` is set to `True`.
+
+---
+
+### Step 4: Extract Text (OCR) (`_pipe_1_step_extract_text`)
+- A random sample of frames is passed through OCR.
+- Text such as **Patient Name**, **Date of Birth (DOB)**, **IDs** is extracted.
+- A `SensitiveMeta` object is created and filled with this information.
+- `VideoState.text_meta_extracted` is set to `True`.
+
+---
+
+### Step 5: Predict Labels (AI Prediction) (`_pipe_1_step_predict`)
+- An AI model is run on frames to predict medical labels (e.g., **polyp**, **normal**, **outside**).
+- Smoothed predictions are generated over frames.
+- A `VideoPredictionMeta` is created to store AI model info and predictions.
+- `VideoState.initial_prediction_completed` is set to `True`.
+
+---
+
+### Step 6: Create Label Segments (`_pipe_1_step_create_segments`)
+- Predictions are converted into labeled segments (`LabelVideoSegment`).
+- Each segment is linked to frame ranges and specific labels.
+- A `LabelVideoSegmentState` is created for each segment to track validation.
+- `VideoState.lvs_created` is set to `True`.
+
+---
+## End of PIPE 1
+
+At this point in the pipeline:
+
+- Frames are **extracted** and **saved**.
+- OCR **text is extracted** and **stored**.
+- AI **predictions are made**.
+- **Label segments are created** based on predictions.
+- **Sensitive information** (e.g. patient identifiers) is stored.
+- `VideoState` is updated to reflect all completed steps.
+
+---
+
+### Tables Involved
+
+| Table Name               |
+|--------------------------|
+| `VideoFile`              |
+| `VideoMeta`              |
+| `Frame`                  |
+| `VideoState`             |
+| `SensitiveMeta`          |
+| `VideoPredictionMeta`    |
+| `LabelVideoSegment`      |
+| `LabelVideoSegmentState` |
+
+---
+## 2. VALIDATION Phase: Review and Approval
+
+**Goal:** Confirm that automated results are correct before anonymization.
+
+### Steps
+
+- A human reviewer (or automated test) opens each `LabelVideoSegment` and reviews it.
+- Each segment’s state is updated:  
+  `LabelVideoSegmentState.is_validated = True`
+- The extracted sensitive metadata (e.g., names, DOB) in `SensitiveMeta` is reviewed.
+- Sensitive metadata state fields are set:  
+  `dob_verified = True`  
+  `names_verified = True`
+- If **any** of these validations fail, processing **must stop** and be corrected.
+- **Only** when **all** validations are successfully marked, the video is allowed to proceed to **PIPE 2**.
+
+---
+
+### Tables Involved
+
+| Table Name               |
+|--------------------------|
+| `LabelVideoSegmentState` |
+| `SensitiveMeta`          |
+
+---
+## 3. PIPELINE 2: Anonymization and Cleanup
+
+**Goal:** Prepare a safe, anonymized version of the video.
+
+---
+
+### Pipeline 2 Major Steps
+
+---
+
+#### Step 1: Ensure VideoState Exists Again (`_pipe_2_step_ensure_state`)
+- Confirm that the `VideoState` tracking object is still present.
+
+---
+
+#### Step 2: Ensure Frames Available (`_pipe_2_step_extract_frames`)
+- Check if extracted frames are available.
+- If frames were deleted or are missing, **re-extract** them from the raw video file.
+
+---
+
+#### Step 3: Anonymize Video (`_pipe_2_step_anonymize`)
+- Frames labeled as `"outside"` are blacked out.
+- Remaining frames are masked using **endoscopy ROI** (Region of Interest).
+- New anonymized frames are created and saved temporarily.
+- A new anonymized video is assembled from these frames.
+- New video file path is saved in:  
+  `VideoFile.processed_file`
+- Video hash is calculated and saved in:  
+  `VideoFile.processed_video_hash`
+- `VideoState.state.anonymization_status.mark_anonymized()`
+
+---
+
+#### Step 4: Delete Raw Files and Frames
+- Raw video file and extracted frames are **permanently deleted**.
+- `VideoState.frames_extracted = False`
+
+---
+
+#### Step 5: Delete SensitiveMeta (`_pipe_2_step_delete_sensitive_meta`)
+- The `SensitiveMeta` object is deleted from the database.
+- `VideoFile.sensitive_meta = None`
+
+---
+
+### At the End of PIPE 2:
+
+- Only the **anonymized processed video** remains.
+- **All sensitive data** and **raw video/frames** are permanently deleted.
+- **Technical metadata** and **AI prediction audit data** are retained.
+
+---
+
+### Tables Involved
+
+| Table Name      | Notes                            |
+|------------------|----------------------------------|
+| `VideoFile`      | Updated with processed file info |
+| `Frame`          | Frames used, then deleted        |
+| `VideoState`     | Tracks anonymization status      |
+| `SensitiveMeta`  | **Deleted after anonymization**  |
+
+---
+## 4. FINAL STATE
+
+### Database Contains:
+
+- The **processed, safe, anonymized video**:  
+  `VideoFile.processed_file`
+- **Technical information** about the video:  
+  `VideoMeta`
+- **Medical label segments** for audit:  
+  `LabelVideoSegment` and `LabelVideoSegmentState`
+- **AI prediction information** for audit:  
+  `VideoPredictionMeta`
+- **Sensitive patient information** has been **permanently deleted**
+- **Raw extracted frames** and the **original raw video file** have been **permanently deleted**
+
+---
+
+### `VideoState` Summary:
+
+- `frames_extracted = False`  *(raw frames are gone)*
+- `anonymized = True`  *(anonymized video is ready)*
+
+---
+
+### Tables Overview
+
+| Table Name              | Purpose                                                                 |
+|--------------------------|-------------------------------------------------------------------------|
+| `VideoFile`              | Main entry linking video paths, processed file, and metadata references |
+| `VideoMeta`              | Technical information (fps, duration, frame count, etc.)                |
+| `Frame`                  | Individual frame images from the video *(deleted in PIPE 2)*            |
+| `VideoState`             | Tracks pipeline processing steps and their completion                   |
+| `SensitiveMeta`          | Patient-sensitive data from OCR *(deleted in PIPE 2)*                   |
+| `VideoPredictionMeta`    | Information about the AI model and predictions                          |
+| `LabelVideoSegment`      | Video segments created from AI predictions                              |
+| `LabelVideoSegmentState` | Tracks human validation status for each segment                         |
+
+---
+### Note: on Frame Deletion and Data Integrity
+
+Although the frames are permanently deleted after the entire execution of Pipeline 1 and Pipeline 2, the necessary data and information about these frames are still populated in the database. Therefore, a script is required to utilize the correct imported data and **re-extract potential frames** from the original video.
+
+This process ensures that the required frame data can be retrieved and used even after deletion.
+
+

endoreg_db/utils/pipelines/process_video_dir.py

@@ -0,0 +1,144 @@
+import logging
+import os
+from pathlib import Path
+from django.core.management import call_command
+from icecream import ic
+from tqdm import tqdm
+from endoreg_db.utils.file_operations import sha256_file
+from endoreg_db.models import (
+    AiModel,
+    LabelVideoSegment,
+    ModelMeta,
+    VideoFile,
+)
+
+logger = logging.getLogger(__name__)
+# Usage
+# from endoreg_db.utils.pipelines.process_video_dir import process_video_dir
+# (video_dir=Path("/path/to/your/videos"))
+
+# DEFAULT_DIR = Path(__file__).resolve().parent.parent.parent.parent / "data/raw_videos"
+DEFAULT_DIR = Path(__file__).resolve().parent.parent.parent.parent / "data/study_data"
+CENTER_NAME = os.environ.get("CENTER_NAME", "university_hospital_wuerzburg")
+ENDOSCOPY_PROCESSOR_NAME = os.environ.get("ENDOSCOPY_PROCESSOR_NAME", "olympus_cv_1500")
+MODEL_NAME = "image_multilabel_classification_colonoscopy_default"
+MODEL_WEIGHTS_PATH = "./tests/assets/colo_segmentation_RegNetX800MF_6.safetensors"
+ic(DEFAULT_DIR)
+
+
+def process_video_dir(
+    video_dir: Path = DEFAULT_DIR,
+    center_name: str = CENTER_NAME,
+    endoscopy_processor_name: str = ENDOSCOPY_PROCESSOR_NAME,
+):
+    """
+    Process a directory of video files and create VideoFile objects in the database.
+
+    Args:
+        video_dir (Path): Path to the directory containing video files.
+        center_name (str): Name of the center to associate with the video files.
+        endoscopy_processor_name (str): Name of the endoscopy processor to associate with the video files.
+    """
+    # Make sure the ai model exists:
+    ai_model = AiModel.objects.filter(name=MODEL_NAME)
+    if ai_model:
+        ai_model = ai_model.first()
+    try:
+        model_meta = ai_model.get_latest_version()
+        assert isinstance(model_meta, ModelMeta), "No ModelMeta found in the database."
+    except Exception:
+        logger.warning(f"ModelMeta not found for {MODEL_NAME}. Creating a new one.")
+        # Create a new ModelMeta instance
+        call_command(
+            "create_multilabel_model_meta",
+            "--model_path",
+            MODEL_WEIGHTS_PATH,
+        )
+
+    # Get the center and endoscopy processor objects
+
+    # Iterate through all files in the directory
+    for file_path in tqdm(video_dir.iterdir()):
+        logger.info(f"Processing file: {file_path}")
+        if file_path.is_file() and file_path.suffix in [".mp4", ".avi", ".mov"]:
+            # Create a VideoFile object
+            video_file = VideoFile.create_from_file_initialized(
+                file_path=file_path,
+                center_name=center_name,
+                processor_name=endoscopy_processor_name,
+                video_hash=sha256_file(file_path),
+            )
+            logger.warning(f"Processing video file: {video_file}")
+            try:
+                success_pipe_1 = video_file.pipe_1(
+                    model_name=MODEL_NAME,
+                )
+                assert success_pipe_1, (
+                    f"Pipe 1 failed for video {video_file.video_hash}"
+                )
+            except Exception as e:
+                logger.error(
+                    f"Pipe 1 failed for video {video_file.video_hash}: {e}",
+                    exc_info=True,
+                )
+                raise e
+
+            video_file.refresh_from_db()
+
+            ####### SIMULATION OF VALIDATION #######
+            # Simulate the validation process for video segments
+            outside_segments = video_file.get_outside_segments()
+            logger.warning(f"Outside segments found for simulation: {outside_segments}")
+            if outside_segments:
+                for segment in outside_segments:
+                    try:
+                        seg_state = segment.state
+                        seg_state.is_validated = True
+                        seg_state.save()
+                        logger.info(f"Simulated validation for segment {segment.id}")
+                    except LabelVideoSegment.state.RelatedObjectDoesNotExist:
+                        logger.warning(
+                            f"Cannot simulate validation for segment {segment.id}: Related state object does not exist."
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Error during segment state simulation for segment {segment.id}: {e}",
+                            exc_info=True,
+                        )
+
+            # Simulate the validation process for the video files sensitive meta obj
+            if video_file.sensitive_meta:
+                try:
+                    sm_state = video_file.sensitive_meta.state
+                    sm_state.dob_verified = True
+                    sm_state.names_verified = True
+                    sm_state.save()
+                    logger.info(
+                        f"Simulated validation for sensitive meta of video {video_file.video_hash}"
+                    )
+                except Exception as e:
+                    logger.error(
+                        f"Error during sensitive meta state simulation for video {video_file.video_hash}: {e}",
+                        exc_info=True,
+                    )
+
+            else:
+                logger.warning(
+                    f"Cannot simulate validation for sensitive meta of video {video_file.video_hash}: SensitiveMeta object does not exist (likely due to pipe_1 failure or incomplete text extraction)."
+                )
+
+            ######## SIMULATION OF VALIDATION ########
+
+            video_file.refresh_from_db()
+            if success_pipe_1:
+                try:
+                    video_file.pipe_2()
+                except Exception as e:
+                    logger.error(
+                        f"Pipe 2 failed for video {video_file.video_hash}: {e}",
+                        exc_info=True,
+                    )
+            else:
+                logger.warning(
+                    f"Skipping Pipe 2 for video {video_file.video_hash} because Pipe 1 did not complete successfully."
+                )