endoreg-db 0.8.3.8__tar.gz → 0.8.4.0__tar.gz

{endoreg_db-0.8.3.8 → endoreg_db-0.8.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: endoreg-db
-Version: 0.8.3.8
+Version: 0.8.4.0
 Summary: EndoReg Db Django App
 Project-URL: Homepage, https://info.coloreg.de
 Project-URL: Repository, https://github.com/wg-lux/endoreg-db

{endoreg_db-0.8.3.8 → endoreg_db-0.8.4.0}/endoreg_db/management/commands/create_model_meta_from_huggingface.py

@@ -50,11 +50,17 @@ class Command(BaseCommand):
 
         try:
             # Download the model weights
-            weights_path = hf_hub_download(repo_id=model_id, filename="pytorch_model.bin", local_dir="/tmp")
+            weights_path = hf_hub_download(
+                repo_id=model_id,
+                filename="colo_segmentation_RegNetX800MF_base.ckpt",
+                local_dir="/tmp",
+            )
             self.stdout.write(f"Downloaded weights to: {weights_path}")
 
             # Get or create AI model
-            ai_model, created = AiModel.objects.get_or_create(name=model_name, defaults={"description": f"Model from {model_id}"})
+            ai_model, created = AiModel.objects.get_or_create(
+                name=model_name, defaults={"description": f"Model from {model_id}"}
+            )
             if created:
                 self.stdout.write(f"Created AI model: {ai_model.name}")
 
@@ -62,7 +68,9 @@ class Command(BaseCommand):
             try:
                 labelset = LabelSet.objects.get(name=labelset_name)
             except LabelSet.DoesNotExist:
-                self.stdout.write(self.style.ERROR(f"LabelSet '{labelset_name}' not found"))
+                self.stdout.write(
+                    self.style.ERROR(f"LabelSet '{labelset_name}' not found")
+                )
                 return
 
             # Create ModelMeta
@@ -86,13 +94,19 @@ class Command(BaseCommand):
 
             # Save the weights file to the model
             with open(weights_path, "rb") as f:
-                model_meta.weights.save(f"{model_name}_v{version}_pytorch_model.bin", ContentFile(f.read()))
+                model_meta.weights.save(
+                    f"{model_name}_v{version}_pytorch_model.bin", ContentFile(f.read())
+                )
 
             # Set as active meta
             ai_model.active_meta = model_meta
             ai_model.save()
 
-            self.stdout.write(self.style.SUCCESS(f"Successfully {'created' if created else 'updated'} ModelMeta: {model_meta}"))
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"Successfully {'created' if created else 'updated'} ModelMeta: {model_meta}"
+                )
+            )
 
         except Exception as e:
             self.stdout.write(self.style.ERROR(f"Error creating ModelMeta: {e}"))

{endoreg_db-0.8.3.8 → endoreg_db-0.8.4.0}/endoreg_db/models/metadata/model_meta_logic.py

@@ -1,21 +1,21 @@
 import shutil
+from logging import getLogger
 from pathlib import Path
-from typing import Optional, TYPE_CHECKING, Any, Type
-from huggingface_hub import hf_hub_download
+from typing import TYPE_CHECKING, Any, Optional, Type
+
 from django.db import transaction
+from huggingface_hub import hf_hub_download
 
 # Assuming ModelMeta, AiModel, LabelSet are importable from the correct locations
 # Adjust imports based on your project structure if necessary
 from ..administration.ai.ai_model import AiModel
 from ..label.label_set import LabelSet
-from ..utils import WEIGHTS_DIR, STORAGE_DIR
-
-from logging import getLogger
+from ..utils import STORAGE_DIR, WEIGHTS_DIR
 
 logger = getLogger("ai_model")
 
 if TYPE_CHECKING:
-    from .model_meta import ModelMeta # Import ModelMeta for type hinting
+    from .model_meta import ModelMeta  # Import ModelMeta for type hinting
 
 
 def get_latest_version_number_logic(
@@ -29,13 +29,13 @@ def get_latest_version_number_logic(
     """
     versions_qs = cls.objects.filter(
         name=meta_name, model__name=model_name
-    ).values_list('version', flat=True)
+    ).values_list("version", flat=True)
 
     max_v = 0
     found_numeric_version = False
 
     for v_str in versions_qs:
-        if v_str is None: # Skip None versions
+        if v_str is None:  # Skip None versions
             continue
         try:
             v_int = int(v_str)
@@ -47,13 +47,13 @@ def get_latest_version_number_logic(
                 f"Warning: Could not parse version string '{v_str}' as an integer for "
                 f"meta_name='{meta_name}', model_name='{model_name}' while determining the max version."
             )
-    
+
     return max_v if found_numeric_version else 0
 
 
 @transaction.atomic
 def create_from_file_logic(
-    cls: Type["ModelMeta"], # cls is ModelMeta
+    cls: Type["ModelMeta"],  # cls is ModelMeta
     meta_name: str,
     model_name: str,
     labelset_name: str,
@@ -94,11 +94,14 @@ def create_from_file_logic(
             )
         elif existing and bump_if_exists:
             target_version = str(latest_version_num + 1)
-            logger.info(f"Bumping version for {meta_name}/{model_name} to {target_version}")
+            logger.info(
+                f"Bumping version for {meta_name}/{model_name} to {target_version}"
+            )
     else:
         target_version = str(latest_version_num + 1)
-        logger.info(f"Setting next version for {meta_name}/{model_name} to {target_version}")
-
+        logger.info(
+            f"Setting next version for {meta_name}/{model_name} to {target_version}"
+        )
 
     # --- Prepare Weights File ---
     source_weights_path = Path(weights_file).resolve()
@@ -108,7 +111,10 @@ def create_from_file_logic(
     # Construct destination path within MEDIA_ROOT/WEIGHTS_DIR
     weights_filename = source_weights_path.name
     # Relative path for the FileField upload_to
-    relative_dest_path = Path(WEIGHTS_DIR.relative_to(STORAGE_DIR)) / f"{meta_name}_v{target_version}_{weights_filename}"
+    relative_dest_path = (
+        Path(WEIGHTS_DIR.relative_to(STORAGE_DIR))
+        / f"{meta_name}_v{target_version}_{weights_filename}"
+    )
     # Full path for shutil.copy
     full_dest_path = STORAGE_DIR / relative_dest_path
 
@@ -125,8 +131,8 @@ def create_from_file_logic(
     # --- Create/Update ModelMeta Instance ---
     defaults = {
         "labelset": label_set,
-        "weights": relative_dest_path.as_posix(), # Store relative path for FileField
-        **kwargs, # Pass through other fields like activation, mean, std, etc.
+        "weights": relative_dest_path.as_posix(),  # Store relative path for FileField
+        **kwargs,  # Pass through other fields like activation, mean, std, etc.
     }
 
     # Remove None values from defaults to avoid overriding model defaults unnecessarily
@@ -152,35 +158,39 @@ def create_from_file_logic(
 
     return model_meta
 
+
 # --- Add other logic functions referenced by ModelMeta here ---
 # (get_latest_version_number_logic, get_activation_function_logic, etc.)
 # Placeholder for get_activation_function_logic
 def get_activation_function_logic(activation_name: str):
-    import torch.nn as nn # Import locally as it's specific to this function
+    import torch.nn as nn  # Import locally as it's specific to this function
+
     if activation_name.lower() == "sigmoid":
         return nn.Sigmoid()
     elif activation_name.lower() == "softmax":
         # Note: Softmax usually requires specifying the dimension
-        return nn.Softmax(dim=1) # Assuming dim=1 (channels) is common
+        return nn.Softmax(dim=1)  # Assuming dim=1 (channels) is common
     elif activation_name.lower() == "none":
         return nn.Identity()
     else:
         # Consider adding more activations or raising an error
         raise ValueError(f"Unsupported activation function: {activation_name}")
 
+
 # Placeholder for get_inference_dataset_config_logic
 def get_inference_dataset_config_logic(model_meta: "ModelMeta") -> dict:
     # This would typically extract relevant fields from model_meta
     # for configuring a dataset during inference
     return {
-        "mean": [float(x) for x in model_meta.mean.split(',')],
-        "std": [float(x) for x in model_meta.std.split(',')],
-        "size_y": model_meta.size_y, # Add size_y key
-        "size_x": model_meta.size_x, # Add size_x key
-        "axes": [int(x) for x in model_meta.axes.split(',')],
+        "mean": [float(x) for x in model_meta.mean.split(",")],
+        "std": [float(x) for x in model_meta.std.split(",")],
+        "size_y": model_meta.size_y,  # Add size_y key
+        "size_x": model_meta.size_x,  # Add size_x key
+        "axes": [int(x) for x in model_meta.axes.split(",")],
         # Add other relevant config like normalization type, etc.
     }
 
+
 # Placeholder for get_config_dict_logic
 def get_config_dict_logic(model_meta: "ModelMeta") -> dict:
     # Returns a dictionary representation of the model's configuration
@@ -202,6 +212,7 @@ def get_config_dict_logic(model_meta: "ModelMeta") -> dict:
         # Add any other relevant fields
     }
 
+
 # Placeholder for get_model_meta_by_name_version_logic
 def get_model_meta_by_name_version_logic(
     cls: Type["ModelMeta"],
@@ -227,17 +238,24 @@ def get_model_meta_by_name_version_logic(
             ) from exc
     else:
         # Get latest version
-        latest = cls.objects.filter(name=meta_name, model=ai_model).order_by("-date_created").first()
+        latest = (
+            cls.objects.filter(name=meta_name, model=ai_model)
+            .order_by("-date_created")
+            .first()
+        )
         if latest:
             return latest
         else:
             raise cls.DoesNotExist(
                 f"No ModelMeta found for '{meta_name}' and model '{model_name}'."
             )
-            
-from huggingface_hub import model_info
+
+
 import re
 
+from huggingface_hub import model_info
+
+
 def infer_default_model_meta_from_hf(model_id: str) -> dict[str, Any]:
     """
     Infers default model metadata (activation, normalization, input size)
@@ -248,7 +266,9 @@ def infer_default_model_meta_from_hf(model_id: str) -> dict[str, Any]:
     """
 
     if not (info := model_info(model_id)):
-        logger.info(f"Could not retrieve model info for {model_id}, using ColoReg segmentation defaults.")
+        logger.info(
+            f"Could not retrieve model info for {model_id}, using ColoReg segmentation defaults."
+        )
         return {
             "name": "wg-lux/colo_segmentation_RegNetX800MF_base",
             "activation": "sigmoid",
@@ -295,18 +315,29 @@ def infer_default_model_meta_from_hf(model_id: str) -> dict[str, Any]:
         "size_y": size_y,
         "description": f"Inferred defaults for {model_id}",
     }
-    
-def setup_default_from_huggingface_logic(cls, model_id: str, labelset_name: str | None = None):
+
+
+def setup_default_from_huggingface_logic(
+    cls, model_id: str, labelset_name: str | None = None
+):
     """
     Downloads model weights from Hugging Face and auto-fills ModelMeta fields.
     """
     meta = infer_default_model_meta_from_hf(model_id)
 
     # Download weights
-    weights_path = hf_hub_download(repo_id=model_id, filename="pytorch_model.bin", local_dir=WEIGHTS_DIR)
+    weights_path = hf_hub_download(
+        repo_id=model_id,
+        filename="colo_segmentation_RegNetX800MF_base.ckpt",
+        local_dir=WEIGHTS_DIR,
+    )
 
     ai_model, _ = AiModel.objects.get_or_create(name=meta["name"])
-    labelset = LabelSet.objects.first() if not labelset_name else LabelSet.objects.get(name=labelset_name)
+    labelset = (
+        LabelSet.objects.first()
+        if not labelset_name
+        else LabelSet.objects.get(name=labelset_name)
+    )
 
     return create_from_file_logic(
         cls,

{endoreg_db-0.8.3.8 → endoreg_db-0.8.4.0}/endoreg_db/services/video_import.py

@@ -55,7 +55,18 @@ class VideoImportService():
             self.project_root = Path(__file__).parent.parent.parent.parent
         
         # Track processed files to prevent duplicates
-        self.processed_files = set(str(Path(ANONYM_VIDEO_DIR) / file) for file in os.listdir(ANONYM_VIDEO_DIR))
+        try:
+            # Ensure anonym_video directory exists before listing files
+            anonym_video_dir = Path(ANONYM_VIDEO_DIR)
+            if anonym_video_dir.exists():
+                self.processed_files = set(str(anonym_video_dir / file) for file in os.listdir(ANONYM_VIDEO_DIR))
+            else:
+                logger.info(f"Creating anonym_videos directory: {anonym_video_dir}")
+                anonym_video_dir.mkdir(parents=True, exist_ok=True)
+                self.processed_files = set()
+        except Exception as e:
+            logger.warning(f"Failed to initialize processed files tracking: {e}")
+            self.processed_files = set()
                     
         # Central video instance and processing context
         self.current_video: Optional[VideoFile] = None
@@ -146,6 +157,9 @@ class VideoImportService():
         High-level helper that orchestrates the complete video import and anonymization process.
         Uses the central video instance pattern for improved state management.
         """
+        # DEFENSIVE: Initialize processing_context immediately to prevent KeyError crashes
+        self.processing_context = {'file_path': Path(file_path)}
+        
         try:
             # Initialize processing context
             self._initialize_processing_context(file_path, center_name, processor_name, 
@@ -182,7 +196,12 @@ class VideoImportService():
             return self.current_video
             
         except Exception as e:
-            self.logger.error(f"Video import and anonymization failed for {file_path}: {e}")
+            # Safe file path access - handles cases where processing_context wasn't initialized
+            safe_file_path = getattr(self, 'processing_context', {}).get('file_path', file_path)
+            # Debug: Log context state for troubleshooting
+            context_keys = list(getattr(self, 'processing_context', {}).keys())
+            self.logger.debug(f"Context keys during error: {context_keys}")
+            self.logger.error(f"Video import and anonymization failed for {safe_file_path}: {e}")
             self._cleanup_on_error()
             raise
         finally:
@@ -969,6 +988,10 @@ class VideoImportService():
         This method is always called in the finally block of import_and_anonymize()
         to ensure the file lock is released even if processing fails.
         """
+        # DEFENSIVE: Ensure processing_context exists before accessing it
+        if not hasattr(self, 'processing_context'):
+            self.processing_context = {}
+            
         try:
             # Release file lock if it was acquired
             lock_context = self.processing_context.get('_lock_context')

{endoreg_db-0.8.3.8 → endoreg_db-0.8.4.0}/pyproject.toml

@@ -14,7 +14,7 @@ maintainers = [
 ]
 name = "endoreg-db"
 readme = "README.md"
-version = "0.8.3.8"
+version = "0.8.4.0"
 
 classifiers = [
   "Development Status :: 3 - Alpha",

{endoreg_db-0.8.3.8 → endoreg_db-0.8.4.0}/uv.lock

@@ -771,7 +771,7 @@ sdist = { url = "https://files.pythonhosted.org/packages/62/59/e47bbd0542d0e6f4c
 
 [[package]]
 name = "endoreg-db"
-version = "0.8.3.8"
+version = "0.8.4.0"
 source = { editable = "." }
 dependencies = [
     { name = "agl-report-reader" },

endoreg_db-0.8.3.8/lx-anonymizer/.envrc

@@ -1,3 +0,0 @@
-source_url "https://raw.githubusercontent.com/cachix/devenv/82c0147677e510b247d8b9165c54f73d32dfd899/direnvrc" "sha256-7u4iDd1nZpxL4tCzmPG0dQgC5V+/44Ba+tHkPob1v2k="
-
-use devenv

endoreg_db-0.8.3.8/lx-anonymizer/.git

@@ -1 +0,0 @@
-gitdir: ../.git/modules/lx-anonymizer

endoreg_db-0.8.3.8/lx-anonymizer/.github/workflows/publish.yml

@@ -1,39 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches: [main]
-    tags:  # Optional: only publish on tagged releases
-      - 'v*'  
-  pull_request:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repository
-        uses: actions/checkout@v4
-
-      # (Optional) run tests, lint, etc. in preceding steps
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-
-      - name: Install build and twine
-        run: |
-          pip install --upgrade pip
-          pip install build twine
-
-      - name: Build Distribution
-        run: |
-          python -m build
-
-      - name: Publish to PyPI
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
-        run: |
-          python -m twine upload dist/*
-        env:
-          PYPI_USERNAME: __token__
-          PYPI_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

endoreg_db-0.8.3.8/lx-anonymizer/.github/workflows/python-app.yml

@@ -1,39 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Check out the repository
-        uses: actions/checkout@v4
-
-      - name: Ensure flake8 is installed
-        run: |
-          which flake8 || pip install flake8  # Fallback if flake8 isn't found
-      - name: Lint with flake8
-        run: |
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-      - name: Install uv
-        uses: astral-sh/setup-uv@v5
-        with:
-          enable-cache: true
-
-      - name: Set up Python
-        run: uv python install
-
-      - name: Install dependencies
-        run: uv sync --all-extras --dev
-
-      - name: Run tests
-        run: 
-          echo "PYTHONPATH=$PYTHONPATH:$(pwd)/lx_anonymizer" >> $GITHUB_ENV
-          uv run pytest tests
-      

endoreg_db-0.8.3.8/lx-anonymizer/.gitignore

@@ -1,183 +0,0 @@
-.devenv/
-.direnv/
-uv.lock
-lx_anonymizer/agl-anonymizer/*
-lx_anonymizer/agl-anonymizer-temp/*
-
-agl-anonymizer/*
-agl-anonymizer-temp/*
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# data
-**~/etc
-**/etc/
-**/data/
-**/temp/
-**/results/
-**/images/
-**/logs/
-./logs/
-
-
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
-.pdm.toml
-.pdm-python
-.pdm-build/
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/

endoreg_db-0.8.3.8/lx-anonymizer/.python-version

@@ -1 +0,0 @@
-3.11

endoreg_db-0.8.3.8/lx-anonymizer/Abstract.md

@@ -1,11 +0,0 @@
-Background:
-Ensuring the privacy of patient data is critical for clinical research compliance with GDPR and international standards. Existing anonymization methods often face trade-offs between data utility and re-identification risk.
-
-Aims and Methods:
-We developed a novel anonymization framework specifically tailored for gastroenterological datasets. The approach incorporates k-anonymity, differential privacy techniques, and syntactic generalization strategies. Validation was performed on a dataset of 10,000 patient records extracted from the [Dataset/Registry Name], evaluating data utility (preservation of analytical outcomes) and disclosure risk according to [ISO standard or specific metric].
-
-Results:
-Application of the framework achieved a 92% reduction in re-identification risk compared to baseline pseudonymization methods (p<0.001), while maintaining 89% of original variable correlations (R² preservation). Processing time per 1,000 records was reduced by 34%. The anonymized datasets passed [external validation tool or procedure] benchmarks, ensuring compliance with GDPR data minimization and integrity principles.
-
-Conclusion:
-This anonymization framework enables high-utility, GDPR-compliant sharing of patient data for gastroenterological research. It offers a scalable and validated approach to balance patient privacy with research needs, facilitating multicenter study collaboration under evolving legal standards.