endoreg-db 0.5.3__tar.gz → 0.6.1__tar.gz

endoreg_db-0.6.1/.envrc

@@ -0,0 +1,3 @@
+source_url "https://raw.githubusercontent.com/cachix/devenv/82c0147677e510b247d8b9165c54f73d32dfd899/direnvrc" "sha256-7u4iDd1nZpxL4tCzmPG0dQgC5V+/44Ba+tHkPob1v2k="
+
+use devenv

{endoreg_db-0.5.3 → endoreg_db-0.6.1}/.gitignore

@@ -2,6 +2,12 @@ result
 result/
 *result
 .ruff_cache/
+.devenv
+untracked_notes.md
+tests/sensitive/
+erc_data/
+./data/
+test_reports/*
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -10,9 +16,11 @@ __pycache__/
 _build
 .direnv
 
-###### Do not Remove ######
+.devenv.flake.nix
+
+###### Do not Remove ###### # Deprecated?
 # Protects temporary files from being comitted
-endoreg_client_manager/data/*
+endoreg_client_manager/data/* 
 
 # C extensions
 *.so

endoreg_db-0.6.1/.pylintrc

@@ -0,0 +1,14 @@
+[MESSAGES CONTROL]
+disable=
+    E1101,
+    W0611,
+    C0114, ; Missing module docstring
+    C0115, ; Missing class docstring
+    C0116, ; Missing function or method docstring
+    C0301, ; Line too long
+    C0411,
+    C0412,
+    C0413, 
+    C0415, ; Import outside toplevel
+    W0613, ; Unused argument
+    W0719, ; Broad exception raised

endoreg_db-0.6.1/.python-version

@@ -0,0 +1 @@
+3.13

endoreg_db-0.6.1/PKG-INFO

@@ -0,0 +1,151 @@
+Metadata-Version: 2.4
+Name: endoreg-db
+Version: 0.6.1
+Summary: EndoReg Db Django App
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: agl-report-reader>=0.4.0
+Requires-Dist: django-bootstrap5>=24.3
+Requires-Dist: django-cors-headers>=4.7.0
+Requires-Dist: django-extensions>=3.2.3
+Requires-Dist: django>=5.1.3
+Requires-Dist: djangorestframework>=3.15.2
+Requires-Dist: faker>=32.1.0
+Requires-Dist: icecream>=2.1.4
+Requires-Dist: numpy>=2.1.3
+Requires-Dist: opencv-python>=4.10.0.84
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: pillow>=11.0.0
+Requires-Dist: pydantic>=2.10.6
+Requires-Dist: pytesseract>=0.3.13
+Requires-Dist: python-ffmpeg>=2.0.12
+Requires-Dist: pytorch-lightning>=2.4.0
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: scikit-learn>=1.5.2
+Requires-Dist: scipy>=1.14.1
+Requires-Dist: tesseract>=0.1.3
+Requires-Dist: tqdm>=4.67.0
+Requires-Dist: urllib3>=2.2.3
+Description-Content-Type: text/markdown
+
+# EndoregDB - Professional Data Infrastructure for Clinical Research
+
+EndoregDB is a comprehensive database framework designed to manage medical and research-related data for clinical trials. This repository focuses on efficient data processing, automated deployment, security, and reproducibility, offering a flexible setup for local development environments as well as distributed systems. It supports the integration of AI/ML tools and advanced image and report processing.
+
+This infrastructure was originally designed for clinical research studies and is optimized for handling large data volumes, including:
+
+- Medical reports,
+- Patient imaging and video data,
+- Clinical product and treatment data,
+  and more.
+
+## 🚀 Key Features
+
+### System Architecture
+
+- **Modular Design**: Built on scalable and reusable components to simplify integration into various environments.
+- **Multi-System Support**: Manages configurations for local workstations and production servers.
+- **Role-Specific Configuration**: Predefined roles for common use cases:
+  - Medical data processing systems
+  - AI/ML model deployment
+  - Research workstation configuration
+
+### Security & Data Management
+
+- **Data Encryption**: All sensitive data is encrypted, and privacy policies are enforced.
+- **Impermanence**: Stateless system configuration with persistence for critical data.
+- **Access Control**: Role-based access and identity management integration.
+
+### Data and Processing Environment
+
+- **Data Processing**: Optimized for processing medical datasets with preprocessing tools.
+- **AI/ML Support**:
+  - Integration of machine learning tools for predictive analysis.
+  - TensorFlow, PyTorch, and other frameworks supported for model training.
+- **Image/Video Processing**: Support for analyzing patient images and clinical videos.
+
+### Development Tools & Infrastructure
+
+- **Data Science Toolchains**: Pre-configured environments for data processing, analysis, and visualization.
+- **Monitoring & Logging**: Setup for continuous monitoring and logging to ensure system stability and performance.
+
+---
+
+## 🛠 Getting Started
+
+### Prerequisites
+
+- A Linux-based system (Ubuntu/Debian recommended) or NixOS
+- Hardware with sufficient storage for data processing (at least 1 TB recommended)
+
+### Quick Start
+
+1. Clone the repository:
+
+   ```bash
+   git clone https://github.com/wg-lux/endoreg-db.git
+   cd endoreg-db
+   ```
+
+2. Set up your Python environment:
+   _TODO_: Explain Devenv / point to other docs
+
+   ```bash
+   direnv allow
+   ```
+
+3. Run tests:
+   Call Devenv Script to run tests
+
+   ```bash
+   runtests
+   ```
+
+---
+
+## 📁 Repository Structure
+
+```
+endoreg-db/
+├── endoreg_db/                # Main Django app for medical data
+│   ├── case_generator/        # Sample case generator
+│   ├── data/                  # Medical knowledge base
+│   ├── management/            # Data wrangling operations
+│   ├── models/                # Data models
+│   ├── migrations/            # Database migrations
+│   └── serializers/           # Serializers for data
+├── .gitignore                 # Git ignore file for unnecessary files
+└── README.md                  # Project description and setup instructions
+```
+
+---
+
+## 🔒 Security Features
+
+- **Data Encryption**: All sensitive patient data is encrypted.
+- **Role-Based Access Control**: Configurable roles for managing access to various parts of the system.
+- **Logging & Auditing**: Comprehensive logging system that tracks user activities and data changes.
+
+---
+
+## 🖥️ Supported Systems
+
+- **Workstations**: Local development or research workstations with low data processing demands.
+- **Servers**: Scalable server infrastructure for processing large data volumes, integrated with cloud services for scalability.
+
+---
+
+## 🛟 Support
+
+For issues and questions:
+
+- Create an issue in the repository
+- Review the Deployment Guide for common issues
+
+---
+
+## 📜 License
+
+MIT - see LICENSE
+
+---

endoreg_db-0.6.1/README.md

@@ -0,0 +1,121 @@
+# EndoregDB - Professional Data Infrastructure for Clinical Research
+
+EndoregDB is a comprehensive database framework designed to manage medical and research-related data for clinical trials. This repository focuses on efficient data processing, automated deployment, security, and reproducibility, offering a flexible setup for local development environments as well as distributed systems. It supports the integration of AI/ML tools and advanced image and report processing.
+
+This infrastructure was originally designed for clinical research studies and is optimized for handling large data volumes, including:
+
+- Medical reports,
+- Patient imaging and video data,
+- Clinical product and treatment data,
+  and more.
+
+## 🚀 Key Features
+
+### System Architecture
+
+- **Modular Design**: Built on scalable and reusable components to simplify integration into various environments.
+- **Multi-System Support**: Manages configurations for local workstations and production servers.
+- **Role-Specific Configuration**: Predefined roles for common use cases:
+  - Medical data processing systems
+  - AI/ML model deployment
+  - Research workstation configuration
+
+### Security & Data Management
+
+- **Data Encryption**: All sensitive data is encrypted, and privacy policies are enforced.
+- **Impermanence**: Stateless system configuration with persistence for critical data.
+- **Access Control**: Role-based access and identity management integration.
+
+### Data and Processing Environment
+
+- **Data Processing**: Optimized for processing medical datasets with preprocessing tools.
+- **AI/ML Support**:
+  - Integration of machine learning tools for predictive analysis.
+  - TensorFlow, PyTorch, and other frameworks supported for model training.
+- **Image/Video Processing**: Support for analyzing patient images and clinical videos.
+
+### Development Tools & Infrastructure
+
+- **Data Science Toolchains**: Pre-configured environments for data processing, analysis, and visualization.
+- **Monitoring & Logging**: Setup for continuous monitoring and logging to ensure system stability and performance.
+
+---
+
+## 🛠 Getting Started
+
+### Prerequisites
+
+- A Linux-based system (Ubuntu/Debian recommended) or NixOS
+- Hardware with sufficient storage for data processing (at least 1 TB recommended)
+
+### Quick Start
+
+1. Clone the repository:
+
+   ```bash
+   git clone https://github.com/wg-lux/endoreg-db.git
+   cd endoreg-db
+   ```
+
+2. Set up your Python environment:
+   _TODO_: Explain Devenv / point to other docs
+
+   ```bash
+   direnv allow
+   ```
+
+3. Run tests:
+   Call Devenv Script to run tests
+
+   ```bash
+   runtests
+   ```
+
+---
+
+## 📁 Repository Structure
+
+```
+endoreg-db/
+├── endoreg_db/                # Main Django app for medical data
+│   ├── case_generator/        # Sample case generator
+│   ├── data/                  # Medical knowledge base
+│   ├── management/            # Data wrangling operations
+│   ├── models/                # Data models
+│   ├── migrations/            # Database migrations
+│   └── serializers/           # Serializers for data
+├── .gitignore                 # Git ignore file for unnecessary files
+└── README.md                  # Project description and setup instructions
+```
+
+---
+
+## 🔒 Security Features
+
+- **Data Encryption**: All sensitive patient data is encrypted.
+- **Role-Based Access Control**: Configurable roles for managing access to various parts of the system.
+- **Logging & Auditing**: Comprehensive logging system that tracks user activities and data changes.
+
+---
+
+## 🖥️ Supported Systems
+
+- **Workstations**: Local development or research workstations with low data processing demands.
+- **Servers**: Scalable server infrastructure for processing large data volumes, integrated with cloud services for scalability.
+
+---
+
+## 🛟 Support
+
+For issues and questions:
+
+- Create an issue in the repository
+- Review the Deployment Guide for common issues
+
+---
+
+## 📜 License
+
+MIT - see LICENSE
+
+---

endoreg_db-0.6.1/dev/cg.py

@@ -0,0 +1,9 @@
+from endoreg_db.case_generator.utils import get_case_generator
+from endoreg_db.case_generator.lab_sample_factory import LabSampleFactory
+
+
+TEMPLATE_NAME = "pre_default_screening_colonoscopy"
+
+cg = get_case_generator(TEMPLATE_NAME)
+lab_factory = LabSampleFactory()
+

endoreg_db-0.6.1/dev/explore_legacy_frames.py

@@ -0,0 +1,108 @@
+from tqdm import tqdm
+import os
+import json
+import shutil
+import random
+from collections import defaultdict
+
+# Define the directory paths
+base_dir = "/run/media/setup-user/8f1cd448-7aef-4eb8-9acb-d05869ea1923"
+jsons_dir = os.path.join(base_dir, "jsons")
+output_json_dir = "./jsons"
+output_image_dir = "./images"
+
+# Remove existing output directories
+shutil.rmtree(output_json_dir, ignore_errors=True)
+shutil.rmtree(output_image_dir, ignore_errors=True)
+
+# Ensure the output directories exist
+os.makedirs(output_json_dir, exist_ok=True)
+os.makedirs(output_image_dir, exist_ok=True)
+
+# Initialize dictionaries to track examples and counts
+key_stats = defaultdict(lambda: defaultdict(int))
+examples = defaultdict(list)
+
+def normalize_value(value):
+    """
+    Converts complex data types into a hashable string representation.
+    """
+    if isinstance(value, (list, dict)):
+        return str(value)
+    return value
+
+def explore_json_file(json_file):
+    """
+    Reads a JSON file and updates key_stats and examples for specific keys.
+    Also copies the corresponding image if the "path" key exists.
+    """
+    try:
+        with open(json_file, 'r', encoding='utf-8') as file:
+            data = json.loads(file.read())
+    except json.JSONDecodeError as e:
+        print(f"Error reading {json_file}: {e}")
+        return False  # Skip processing this file
+    
+    # Update key_stats and gather examples
+    extracted_keys = False
+    for key, value in data.items():
+        if key in ["predictions_smooth", "labels_new"]:
+            for sub_key, sub_value in value.items():
+                sub_value_normalized = normalize_value(sub_value)
+                key_stats[f"{key}.{sub_key}"][sub_value_normalized] += 1
+
+                # Add example if fewer than 50 have been collected
+                if len(examples[f"{key}.{sub_key}"]) < 50:
+                    examples[f"{key}.{sub_key}"].append(data)
+                    extracted_keys = True
+        else:
+            value_normalized = normalize_value(value)
+            key_stats[key][value_normalized] += 1
+    
+    # Copy the corresponding image if "path" key exists
+    if "path" in data:
+        image_path = os.path.join(base_dir, data["path"])
+        if os.path.exists(image_path):
+            try:
+                shutil.copy(image_path, output_image_dir)
+            except Exception as e:
+                print(f"Error copying image {image_path}: {e}")
+    
+    return extracted_keys
+
+def explore_directory(jsons_dir):
+    """
+    Iterates through the JSON files in a randomized order and processes them.
+    """
+    # Gather all JSON files
+    json_files = []
+    for root, _, files in os.walk(jsons_dir):
+        for file in tqdm(files):
+            if file.endswith(".json"):
+                json_files.append(os.path.join(root, file))
+    
+    # Randomize the order of JSON files
+    random.shuffle(json_files)
+
+    # Process each JSON file
+    for json_file in tqdm(json_files):
+        extracted = explore_json_file(json_file)
+
+        # Copy file if it contained relevant keys
+        if extracted:
+            shutil.copy(json_file, output_json_dir)
+
+# Start the exploration process
+explore_directory(jsons_dir)
+
+# Save key_stats and examples to separate summary files
+with open(os.path.join(output_json_dir, "key_stats.json"), "w", encoding="utf-8") as stats_file:
+    json.dump({k: dict(v) for k, v in key_stats.items()}, stats_file, indent=4)
+
+with open(os.path.join(output_json_dir, "examples.json"), "w", encoding="utf-8") as examples_file:
+    json.dump({k: [e["_id"] for e in v] for k, v in examples.items()}, examples_file, indent=4)
+
+# Display summary
+print("Exploration complete!")
+print(f"Total keys analyzed: {len(key_stats)}")
+print(f"Total examples collected: {sum(len(v) for v in examples.values())}")

endoreg_db-0.6.1/dev/test_model.py

@@ -0,0 +1,3 @@
+# run in django shell
+
+

endoreg_db-0.6.1/dev/video_utils.py

@@ -0,0 +1,46 @@
+import subprocess
+import os
+
+# /home/setup-user/dev/endoreg-db/tests/sensitive/raw_videos/001.mp4
+# /home/setup-user/dev/endoreg-db/tests/sensitive/raw_videos/001_short.mp4
+
+def copy_first_minute(input_file, output_file):
+    """
+    Copies the first minute (60 seconds) of a given MP4 file to a new file.
+    
+    Parameters:
+        input_file (str): Path to the input MP4 file.
+        output_file (str): Path to the output MP4 file.
+    """
+    # Check if the input file exists
+    if not os.path.isfile(input_file):
+        raise FileNotFoundError(f"Input file '{input_file}' does not exist.")
+    
+    # Construct the ffmpeg command
+    command = [
+        "ffmpeg",
+        "-i", input_file,          # Input file
+        "-t", "60",                # Duration to copy (60 seconds)
+        "-c", "copy",              # Copy codec (no re-encoding)
+        output_file                # Output file
+    ]
+
+    try:
+        # Run the command
+        subprocess.run(command, check=True)
+        print(f"The first minute of '{input_file}' has been saved to '{output_file}'.")
+    except subprocess.CalledProcessError as e:
+        print(f"Error during file processing: {e}")
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+
+# Example usage
+if __name__ == "__main__":
+    # Input and output file paths
+    input_path = "input.mp4"
+    output_path = "output_first_minute.mp4"
+
+    try:
+        copy_first_minute(input_path, output_path)
+    except Exception as error:
+        print(f"Failed: {error}")

endoreg_db-0.6.1/devenv.lock

@@ -0,0 +1,183 @@
+{
+  "nodes": {
+    "devenv": {
+      "locked": {
+        "dir": "src/modules",
+        "lastModified": 1734605856,
+        "owner": "cachix",
+        "repo": "devenv",
+        "rev": "f81cf7dc4cbfa46de11618af94e5983c5e600d8c",
+        "type": "github"
+      },
+      "original": {
+        "dir": "src/modules",
+        "owner": "cachix",
+        "repo": "devenv",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1733328505,
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "mk-shell-bin": {
+      "locked": {
+        "lastModified": 1677004959,
+        "owner": "rrbutani",
+        "repo": "nix-mk-shell-bin",
+        "rev": "ff5d8bd4d68a347be5042e2f16caee391cd75887",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rrbutani",
+        "repo": "nix-mk-shell-bin",
+        "type": "github"
+      }
+    },
+    "nix2container": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1730479402,
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "rev": "5fb215a1564baa74ce04ad7f903d94ad6617e17a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1734435836,
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "4989a246d7a390a859852baddb1013f825435cee",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1734529975,
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "72d11d40b9878a67c38f003c240c2d2e1811e72a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-24.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1734425854,
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "0ddd26d0925f618c3a5d85a4fa5eb1e23a09491d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "devenv": "devenv",
+        "mk-shell-bin": "mk-shell-bin",
+        "nix2container": "nix2container",
+        "nixpkgs": "nixpkgs",
+        "pre-commit-hooks": "pre-commit-hooks"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}