endoreg-db 0.8.5.2__tar.gz → 0.8.5.4__tar.gz

{endoreg_db-0.8.5.2 → endoreg_db-0.8.5.4}/HOW_WE_KEEP_DATA_PRIVATE.md

@@ -192,11 +192,17 @@ def get_or_create_sensitive_meta(self) -> "SensitiveMeta":
             "patient_last_name": "Unknown",
             "patient_dob": date(1990, 1, 1),
             "examination_date": date.today(),
-            "center": self.center,
+            "center": self.center,  # ← Can pass Center object OR "center_name": "hospital_name" string
         }
         self.sensitive_meta = SensitiveMeta.create_from_dict(default_data)
 ```
 
+**Note:** `create_from_dict()` accepts **both** formats:
+- `"center": center_obj` (Center object)
+- `"center_name": "hospital_name"` (String)
+
+This flexibility allows different parts of the codebase to use whichever format is more convenient.
+
 ### 3. Update with Real Data (sensitive_meta_logic.py)
 
 ```python
@@ -314,14 +320,34 @@ self.sensitive_meta = SensitiveMeta.objects.create(center=self.center)
 # → ERROR: "First name is required to calculate patient hash"
 ```
 
-### ✅ **Right: `create_from_dict()` with defaults**
+### ❌ **Wrong: `create_from_dict()` without center parameter**
+```python
+# BROKEN - Missing center information
+default_data = {
+    "patient_first_name": "Patient",
+    "patient_last_name": "Unknown",
+}
+self.sensitive_meta = SensitiveMeta.create_from_dict(default_data)
+# → ERROR: "center_name is required in data dictionary"
+```
+
+### ✅ **Right: `create_from_dict()` with defaults and center**
 ```python
-# NEW CODE (WORKING)
+# NEW CODE (WORKING) - Option 1: Center object
 default_data = {
     "patient_first_name": "Patient",
     "patient_last_name": "Unknown",
     "patient_dob": date(1990, 1, 1),
-    "center": self.center,
+    "center": self.center,  # ← Center object
+}
+self.sensitive_meta = SensitiveMeta.create_from_dict(default_data)
+
+# NEW CODE (WORKING) - Option 2: Center name string
+default_data = {
+    "patient_first_name": "Patient",
+    "patient_last_name": "Unknown",
+    "patient_dob": date(1990, 1, 1),
+    "center_name": "university_hospital_wuerzburg",  # ← String
 }
 self.sensitive_meta = SensitiveMeta.create_from_dict(default_data)
 # → Hash calculation succeeds
@@ -329,6 +355,11 @@ self.sensitive_meta = SensitiveMeta.create_from_dict(default_data)
 # → Real data can be updated later
 ```
 
+**Why both formats are supported:**
+- `video_file.py` has access to the Center **object** (`self.center`)
+- `video_import.py` may only have the center **name** string
+- Both are valid and will work correctly
+
 ---
 
 ## Related Documentation

{endoreg_db-0.8.5.2 → endoreg_db-0.8.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: endoreg-db
-Version: 0.8.5.2
+Version: 0.8.5.4
 Summary: EndoReg Db Django App
 Project-URL: Homepage, https://info.coloreg.de
 Project-URL: Repository, https://github.com/wg-lux/endoreg-db

{endoreg_db-0.8.5.2 → endoreg_db-0.8.5.4}/endoreg_db/models/metadata/sensitive_meta_logic.py

@@ -215,7 +215,7 @@ def create_pseudo_examiner_logic(instance: "SensitiveMeta") -> "Examiner":
         )
         # Ensure default center exists or handle appropriately
         try:
-            default_center = Center.objects.get_by_natural_key("endoreg_db_demo")
+            default_center = Center.objects.get(name="endoreg_db_demo")
         except Center.DoesNotExist:
             logger.error(
                 "Default center 'endoreg_db_demo' not found. Cannot create default examiner."
@@ -448,7 +448,50 @@ def perform_save_logic(instance: "SensitiveMeta") -> "Examiner":
 def create_sensitive_meta_from_dict(
     cls: Type["SensitiveMeta"], data: Dict[str, Any]
 ) -> "SensitiveMeta":
-    """Logic to create a SensitiveMeta instance from a dictionary."""
+    """
+    Create a SensitiveMeta instance from a dictionary.
+
+    **Center handling:**
+    This function accepts TWO ways to specify the center:
+    1. `center` (Center object) - Directly pass a Center instance
+    2. `center_name` (string) - Pass the center name as a string (will be resolved to Center object)
+
+    At least ONE of these must be provided.
+
+    **Example usage:**
+    ```python
+    # Option 1: With Center object
+    data = {
+        "patient_first_name": "Patient",
+        "patient_last_name": "Unknown",
+        "patient_dob": date(1990, 1, 1),
+        "examination_date": date.today(),
+        "center": center_obj,  # ← Center object
+    }
+    sm = SensitiveMeta.create_from_dict(data)
+
+    # Option 2: With center name string
+    data = {
+        "patient_first_name": "Patient",
+        "patient_last_name": "Unknown",
+        "patient_dob": date(1990, 1, 1),
+        "examination_date": date.today(),
+        "center_name": "university_hospital_wuerzburg",  # ← String
+    }
+    sm = SensitiveMeta.create_from_dict(data)
+    ```
+
+    Args:
+        cls: The SensitiveMeta class
+        data: Dictionary containing field values
+
+    Returns:
+        SensitiveMeta: The created instance
+
+    Raises:
+        ValueError: If neither center nor center_name is provided
+        ValueError: If center_name does not match any Center in database
+    """
 
     field_names = {
         f.name
@@ -585,15 +628,29 @@ def create_sensitive_meta_from_dict(
                 )
                 selected_data.pop("examination_date", None)
 
-    # Handle Center
+    # Handle Center - accept both center_name (string) and center (object)
+    from ..administration import Center
+
+    center = data.get("center")  # First try direct Center object
     center_name = data.get("center_name")
-    if not center_name:
-        raise ValueError("center_name is required in data dictionary.")
-    try:
-        center = Center.objects.get_by_natural_key(center_name)
+
+    if center is not None:
+        # Center object provided directly - validate it's a Center instance
+        if not isinstance(center, Center):
+            raise ValueError(f"'center' must be a Center instance, got {type(center)}")
         selected_data["center"] = center
-    except Center.DoesNotExist as exc:
-        raise ValueError(f"Center with name '{center_name}' does not exist.") from exc
+    elif center_name:
+        # center_name string provided - resolve to Center object
+        try:
+            center = Center.objects.get(name=center_name)
+            selected_data["center"] = center
+        except Center.DoesNotExist:
+            raise ValueError(f"Center with name '{center_name}' does not exist.")
+    else:
+        # Neither center nor center_name provided
+        raise ValueError(
+            "Either 'center' (Center object) or 'center_name' (string) is required in data dictionary."
+        )
 
     # Handle Names and Gender
     first_name = selected_data.get("patient_first_name") or DEFAULT_UNKNOWN_NAME
@@ -708,17 +765,37 @@ def update_sensitive_meta_from_dict(
         k: v for k, v in data.items() if k in field_names and k not in excluded_fields
     }
 
-    # Handle potential Center update
+    # Handle potential Center update - accept both center_name (string) and center (object)
+    from ..administration import Center
+
+    center = data.get("center")  # First try direct Center object
     center_name = data.get("center_name")
-    if center_name:
+
+    if center is not None:
+        # Center object provided directly - validate and update
+        if isinstance(center, Center):
+            instance.center = center
+            logger.debug(f"Updated center from Center object: {center.name}")
+        else:
+            logger.warning(
+                f"Invalid center type {type(center)}, expected Center instance. Ignoring."
+            )
+        # Remove from selected_data to prevent override
+        selected_data.pop("center", None)
+    elif center_name:
+        # center_name string provided - resolve to Center object
         try:
-            center = Center.objects.get_by_natural_key(center_name)
-            instance.center = center  # Update center directly
-        except Center.DoesNotExist as exc:
+            center_obj = Center.objects.get(name=center_name)
+            instance.center = center_obj
+            logger.debug(f"Updated center from center_name string: {center_name}")
+        except Center.DoesNotExist:
             logger.warning(
                 f"Center '{center_name}' not found during update. Keeping existing center."
             )
-            selected_data.pop("center", None)  # Remove from dict if not found
+    else:
+        # Both are None/missing - remove 'center' from selected_data to preserve existing value
+        selected_data.pop("center", None)
+    # If both are None/missing, keep existing center (no update needed)
 
     # Set examiner names if provided, before calling save
     examiner_first_name = data.get("examiner_first_name")
@@ -801,6 +878,11 @@ def update_sensitive_meta_from_dict(
     # Update other attributes from selected_data
     patient_name_changed = False
     for k, v in selected_data.items():
+        # Skip None values to avoid overwriting existing data
+        if v is None:
+            logger.debug(f"Skipping field '{k}' during update because value is None")
+            continue
+
         # Avoid overwriting examiner names if they were just explicitly set
         if (
             k not in ["examiner_first_name", "examiner_last_name"]

{endoreg_db-0.8.5.2 → endoreg_db-0.8.5.4}/pyproject.toml

@@ -14,7 +14,7 @@ maintainers = [
 ]
 name = "endoreg-db"
 readme = "README.md"
-version = "0.8.5.2"
+version = "0.8.5.4"
 
 classifiers = [
   "Development Status :: 3 - Alpha",

{endoreg_db-0.8.5.2 → endoreg_db-0.8.5.4}/uv.lock

@@ -771,7 +771,7 @@ sdist = { url = "https://files.pythonhosted.org/packages/62/59/e47bbd0542d0e6f4c
 
 [[package]]
 name = "endoreg-db"
-version = "0.8.5.0"
+version = "0.8.5.3"
 source = { editable = "." }
 dependencies = [
     { name = "agl-report-reader" },

endoreg_db-0.8.5.2/docs/TWO_PHASE_PATIENT_DATA_PATTERN.md

@@ -1,352 +0,0 @@
-# Two-Phase Patient Data Pattern
-
-## Overview
-
-The endoreg-db system implements a **two-phase pattern** for handling patient data during video import and anonymization. This pattern allows the system to:
-
-1. **Create valid SensitiveMeta instances** even when patient data is not yet available
-2. **Update with real patient data** once it becomes available (e.g., from OCR extraction)
-3. **Maintain referential integrity** while recalculating hashes and pseudo-entities
-
-This document explains how this pattern works and why it's necessary.
-
----
-
-## The Problem
-
-During video import, patient data is not immediately available:
-
-```python
-# Video import starts
-video = VideoFile.create_from_file_initialized(file_path, center_name, processor_name)
-video.initialize()  # ← Needs SensitiveMeta, but no patient data yet!
-```
-
-However, `SensitiveMeta` requires:
-- `patient_first_name` (for hash calculation)
-- `patient_last_name` (for hash calculation)  
-- `patient_dob` (for hash calculation)
-- `center` (for anonymization)
-
-**Without these fields, hash calculation fails:**
-```
-AssertionError: First name is required to calculate patient hash.
-```
-
-Patient data is only extracted **later** during frame cleaning:
-```python
-# lx_anonymizer extracts patient data from video frames
-extracted_metadata = {
-    "patient_first_name": "Max",
-    "patient_last_name": "Mustermann",
-    "patient_dob": date(1985, 3, 15),
-}
-```
-
----
-
-## The Solution: Two-Phase Pattern
-
-### Phase 1: Initial Creation (with Defaults)
-
-When `SensitiveMeta` is first created, **default values** are set to allow hash calculation:
-
-```python
-# video_file.py - get_or_create_sensitive_meta()
-default_data = {
-    "patient_first_name": "Patient",      # Default
-    "patient_last_name": "Unknown",       # Default
-    "patient_dob": date(1990, 1, 1),      # Default
-    "examination_date": date.today(),
-    "center": video.center,
-}
-sensitive_meta = SensitiveMeta.create_from_dict(default_data)
-```
-
-**What happens internally:**
-1. `perform_save_logic()` is called
-2. Default names are already set (no replacement needed)
-3. Hash is calculated: `sha256("Patient Unknown 1990-01-01 ...")`
-4. Temporary pseudo-entities are created based on this hash
-
-**Result:** Video import can proceed without errors ✅
-
----
-
-### Phase 2: Update (with Extracted Data)
-
-Later, when real patient data is extracted, it **overwrites** the defaults:
-
-```python
-# video_import.py - _update_sensitive_metadata()
-extracted_metadata = {
-    "patient_first_name": "Max",
-    "patient_last_name": "Mustermann",
-    "patient_dob": date(1985, 3, 15),
-}
-
-sm.update_from_dict(extracted_metadata)
-sm.save()  # ← Triggers perform_save_logic() AGAIN
-```
-
-**What happens internally:**
-1. `update_from_dict()` sets new values:
-   - `instance.patient_first_name = "Max"` (overwrites "Patient")
-   - `instance.patient_last_name = "Mustermann"` (overwrites "Unknown")
-2. `save()` is called → triggers `perform_save_logic()`
-3. Default-setting is **skipped** (names already exist):
-   ```python
-   if not instance.patient_first_name:  # ← FALSE (already "Max")
-       instance.patient_first_name = DEFAULT_UNKNOWN_NAME  # ← NOT EXECUTED
-   ```
-4. Hash is **RECALCULATED** with real data:
-   - `sha256("Max Mustermann 1985-03-15 ...")`
-5. **New pseudo-entities** are created/retrieved based on new hash
-
-**Result:** Correct anonymization with real patient data ✅
-
----
-
-## Code Flow Diagram
-
-```mermaid
-graph TD
-    A[VideoFile.create_from_file_initialized] --> B[video.initialize]
-    B --> C[get_or_create_sensitive_meta]
-    C --> D[SensitiveMeta.create_from_dict default_data]
-    
-    D --> E[perform_save_logic]
-    E --> F{patient_first_name exists?}
-    F -->|NO| G[Set: patient_first_name = 'Patient']
-    F -->|YES| H[Skip default setting]
-    G --> I[Calculate Hash: sha256 'Patient Unknown...']
-    H --> I
-    I --> J[Create Pseudo-Patient A]
-    
-    J --> K[Frame Cleaning Extracts Data]
-    K --> L[extracted_metadata = 'Max Mustermann...']
-    L --> M[sm.update_from_dict extracted_metadata]
-    
-    M --> N[Set: patient_first_name = 'Max']
-    N --> O[sm.save]
-    O --> P[perform_save_logic AGAIN]
-    
-    P --> Q{patient_first_name exists?}
-    Q -->|YES Already 'Max'| R[Skip default setting]
-    R --> S[RECALCULATE Hash: sha256 'Max Mustermann...']
-    S --> T[Create/Get Pseudo-Patient B]
-    
-    T --> U[Correct Anonymization Complete]
-    
-    style D fill:#ffcccc
-    style E fill:#ffcccc
-    style I fill:#ffcccc
-    style J fill:#ffcccc
-    style M fill:#ccffcc
-    style P fill:#ccffcc
-    style S fill:#ccffcc
-    style T fill:#ccffcc
-```
-
----
-
-## Key Implementation Details
-
-### 1. Default Name Setting (sensitive_meta_logic.py)
-
-```python
-@transaction.atomic
-def perform_save_logic(instance: "SensitiveMeta") -> "Examiner":
-    """
-    CRITICAL: Set default patient names BEFORE hash calculation.
-    
-    Two-phase approach:
-    - Phase 1: Set defaults if names are missing
-    - Phase 2: Skip if names already exist (real data)
-    """
-    
-    if not instance.patient_first_name:
-        instance.patient_first_name = DEFAULT_UNKNOWN_NAME  # "unknown"
-        
-    if not instance.patient_last_name:
-        instance.patient_last_name = DEFAULT_UNKNOWN_NAME  # "unknown"
-    
-    # Hash is ALWAYS recalculated (uses current field values)
-    instance.patient_hash = calculate_patient_hash(instance)
-    instance.examination_hash = calculate_examination_hash(instance)
-    
-    # Pseudo-entities created based on current hash
-    pseudo_patient = get_or_create_pseudo_patient_logic(instance)
-    # ...
-```
-
-### 2. Initial Creation (video_file.py)
-
-```python
-def get_or_create_sensitive_meta(self) -> "SensitiveMeta":
-    """Creates SensitiveMeta with default patient data."""
-    
-    if self.sensitive_meta is None:
-        default_data = {
-            "patient_first_name": "Patient",
-            "patient_last_name": "Unknown",
-            "patient_dob": date(1990, 1, 1),
-            "examination_date": date.today(),
-            "center": self.center,
-        }
-        self.sensitive_meta = SensitiveMeta.create_from_dict(default_data)
-```
-
-### 3. Update with Real Data (sensitive_meta_logic.py)
-
-```python
-def update_sensitive_meta_from_dict(instance: "SensitiveMeta", data: Dict[str, Any]) -> "SensitiveMeta":
-    """Updates instance fields and triggers save (hash recalculation)."""
-    
-    # Update instance attributes
-    for k, v in selected_data.items():
-        setattr(instance, k, v)  # Overwrites defaults
-    
-    # Trigger save → perform_save_logic() → hash recalculation
-    instance.save()
-    return instance
-```
-
-### 4. Integration in Video Import (video_import.py)
-
-```python
-def _update_sensitive_metadata(self, extracted_metadata: Dict[str, Any]):
-    """Updates SensitiveMeta with extracted patient data."""
-    
-    sm = self.current_video.sensitive_meta
-    sm.update_from_dict(extracted_metadata)
-    # → Overwrites "Patient Unknown" with "Max Mustermann"
-    # → save() triggers hash recalculation
-    # → New pseudo-entities created
-```
-
----
-
-## Example Timeline
-
-| Step | Action | patient_first_name | patient_hash | Pseudo-Patient |
-|------|--------|-------------------|--------------|----------------|
-| 1 | `video.initialize()` | - | - | - |
-| 2 | `get_or_create_sensitive_meta()` | "Patient" | `hash("Patient Unknown...")` | Pseudo-Patient A (temp) |
-| 3 | Frame cleaning extraction | "Patient" | `hash("Patient Unknown...")` | Pseudo-Patient A |
-| 4 | `update_from_dict({"patient_first_name": "Max"})` | "Max" | - | - |
-| 5 | `save()` → `perform_save_logic()` | "Max" | `hash("Max Mustermann...")` | Pseudo-Patient B (real) |
-
-**Final state:** Correctly anonymized with real patient data and proper pseudo-entities ✅
-
----
-
-## Why This Pattern Works
-
-### 1. **Conditional Default Setting**
-```python
-if not instance.patient_first_name:  # Only sets if missing
-    instance.patient_first_name = DEFAULT_UNKNOWN_NAME
-```
-- **First save:** Field is empty → default is set
-- **Second save:** Field has "Max" → condition is False → skipped
-
-### 2. **Hash Always Recalculated**
-```python
-# This runs on EVERY save, regardless of phase
-instance.patient_hash = calculate_patient_hash(instance)
-```
-- **First save:** Hash of "Patient Unknown"
-- **Second save:** Hash of "Max Mustermann"
-
-### 3. **Pseudo-Entities Auto-Update**
-```python
-# get_or_create based on current hash
-pseudo_patient = get_or_create_pseudo_patient_logic(instance)
-```
-- **First save:** Creates/retrieves pseudo-patient with temp hash
-- **Second save:** Creates/retrieves **different** pseudo-patient with real hash
-
----
-
-## Testing the Pattern
-
-### Test Initial Creation
-```python
-def test_initial_creation_with_defaults():
-    video = VideoFile.create_from_file_initialized(...)
-    sm = video.sensitive_meta
-    
-    assert sm.patient_first_name == "Patient"
-    assert sm.patient_last_name == "Unknown"
-    assert sm.patient_hash is not None  # Hash calculated successfully
-```
-
-### Test Update with Real Data
-```python
-def test_update_with_extracted_data():
-    video = VideoFile.create_from_file_initialized(...)
-    sm = video.sensitive_meta
-    initial_hash = sm.patient_hash
-    
-    # Extract real data
-    sm.update_from_dict({
-        "patient_first_name": "Max",
-        "patient_last_name": "Mustermann",
-        "patient_dob": date(1985, 3, 15),
-    })
-    
-    assert sm.patient_first_name == "Max"  # Updated
-    assert sm.patient_last_name == "Mustermann"  # Updated
-    assert sm.patient_hash != initial_hash  # Hash recalculated
-```
-
----
-
-## Common Pitfalls (Avoided by This Pattern)
-
-### ❌ **Wrong: Direct `objects.create()` without defaults**
-```python
-# OLD CODE (BROKEN)
-self.sensitive_meta = SensitiveMeta.objects.create(center=self.center)
-# → save() called immediately
-# → perform_save_logic() tries to calculate hash
-# → ERROR: "First name is required to calculate patient hash"
-```
-
-### ✅ **Right: `create_from_dict()` with defaults**
-```python
-# NEW CODE (WORKING)
-default_data = {
-    "patient_first_name": "Patient",
-    "patient_last_name": "Unknown",
-    "patient_dob": date(1990, 1, 1),
-    "center": self.center,
-}
-self.sensitive_meta = SensitiveMeta.create_from_dict(default_data)
-# → Hash calculation succeeds
-# → Video import proceeds
-# → Real data can be updated later
-```
-
----
-
-## Related Documentation
-
-- **Implementation:** `endoreg_db/models/metadata/sensitive_meta_logic.py`
-- **Video Integration:** `endoreg_db/models/media/video/video_file.py`
-- **Import Service:** `endoreg_db/services/video_import.py`
-- **Hash Calculation:** `endoreg_db/utils/hashs.py`
-
----
-
-## Summary
-
-The two-phase pattern allows endoreg-db to:
-1. ✅ Create valid SensitiveMeta instances without patient data
-2. ✅ Import videos without blocking on missing information
-3. ✅ Update with real data once available
-4. ✅ Maintain proper anonymization with correct hashes
-5. ✅ Avoid "First name required" errors during hash calculation
-
-**Key principle:** Defaults are temporary placeholders that get replaced by real data, with hash recalculation ensuring correct anonymization.