compox 2.0.0.dev2__tar.gz → 2.1.0.dev1__tar.gz

compox-2.1.0.dev1/PKG-INFO

@@ -0,0 +1,154 @@
+Metadata-Version: 2.4
+Name: compox
+Version: 2.1.0.dev1
+Author-Email: Jan Matula <jan.matula@tescan.com>, =?utf-8?q?Old=C5=99ich_Kodym?= <oldrich.kodym@tescan.com>
+License-Expression: Apache-2.0 AND LicenseRef-Commons-Clause-1.0
+License-File: LICENSE
+License-File: NOTICE
+License-File: THIRD_PARTY_LICENSES.txt
+Requires-Python: >=3.10.0
+Requires-Dist: fastapi>=0.100.1
+Requires-Dist: boto3>=1.36.0
+Requires-Dist: celery==5.3.1
+Requires-Dist: uvicorn>=0.25.0
+Requires-Dist: h5py>=3.9.0
+Requires-Dist: numpy<2.0.0
+Requires-Dist: natsort==8.4.0
+Requires-Dist: pytest>=7.4.2
+Requires-Dist: pydantic-settings>=2.0.3
+Requires-Dist: python-minifier>=2.9.0
+Requires-Dist: pystray>=0.19.5
+Requires-Dist: pywin32>=306; platform_system == "Windows"
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: toml>=0.10.2
+Requires-Dist: flower>=2.0.1
+Requires-Dist: python-multipart>=0.0.20
+Requires-Dist: mangum>=0.19.0
+Requires-Dist: requests==2.32.3
+Requires-Dist: pycurl>=7.45.4
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: typer>=0.16.0
+Requires-Dist: pip>=25.2
+Requires-Dist: build>=1.3.0
+Requires-Dist: pdm-backend>=2.4.5
+Description-Content-Type: text/markdown
+
+# Compox
+Compox is a simple Python execution engine for Tescan applications. It's purpose is to run data processing algorithms written in Python on the server and report the results to the client application such as Tescan 3D Viewer or Picannto.
+
+See the [GitHub pages](https://tescan-3dim.github.io/compox) for full python API reference, docs and tutorials!
+
+## Installation
+Compox can be installed through PyPI: 
+```bash
+pip install compox
+```
+
+You can also install the server by cloning this repository and installing it from source.
+
+Note that some additional dependencies, such as MinIO executable for your operating system, will be downloaded on the first run of the server.
+
+## Running the server
+Once installed into your Python environment, Compox can be run using the `compox` command. This command is available in the virtual environment 
+created during the setup process. Run the following command to list the available commands:
+
+```bash
+compox --help
+```
+
+It's recommended to start with generating an initial configuration file with `compox generate-config`. See the configuration options section below.
+To see the usage of the `generate-config` command, you can run:
+
+```bash
+compox generate-config --help
+```
+
+To run the Compox, use the following command:
+
+```bash
+compox run --config /path/to/config.yaml
+```
+
+### Deploying algorithms
+See Algorithm development tutorial for information about target structure of the deployable algorithms.
+
+Once your algorithm is ready, you can use the `compox deploy-algorithms` command to deploy it to your server. 
+To see the usage of the `deploy-algorithms` command, you can run:
+
+```bash
+compox deploy-algorithms --help
+```
+
+This command will read the algorithm definitions from the folder specified in your configuration file with the `deploy_algorithms_from` key
+and deploy them to the server. For example:
+
+```bash
+compox deploy-algorithms --config /path/to/config.yaml
+```
+
+Note that the server does not need to be running in order to deploy the algorithms.
+
+
+## Configuration
+The server uses pydantic settings for configuration. The options can be either set as runtime arguments, in a yaml file, or using the `COMPOX__` environment variables, in corresponding order of precedence.
+
+- Ensure all paths and URLs are correctly set before running the application.
+- Adjust CUDA settings based on hardware capabilities.
+- Logging paths should be accessible by the application to prevent errors.
+- See the [configuration reference](#configuration-reference) below for detailed configuration options.
+
+<details><summary>Configuration reference</summary>
+
+<br>
+
+### Compox Configuration Reference
+
+| Section                         | Field                      | Default                            | Description                                                                 |
+|----------------------------------|-----------------------------|------------------------------------|-----------------------------------------------------------------------------|
+|                                  | `port`                    | `5461`                             | The main server port used to make requests                                 |
+|                                  | `deploy_algorithms_from`  | `"./algorithms"`                   | Directory for algorithm deployment sources.                                |
+|                                  | `log_path`                | `"LOG_DEFAULT:compox.log"`     | Path to the main log file (supports dynamic prefixes).                     |
+|                                  | `config`                  | `None`                             | Optional config path override.                                             |
+| `info`                           | `product_name`              | `"TESCAN 3D Backend"`              | Product display name.                                                      |
+| `info`                           | `server_tags`               | `[]` → auto appends `"compox"` | Tags attached to the server. `"compox"` is added automatically.        |
+| `info`                           | `group_name`                | `"TESCAN GROUP, a.s."`             | Name of the corporate group.                                               |
+| `info`                           | `organization_name`         | `"TESCAN 3DIM, s.r.o."`            | Full name of the organization.                                             |
+| `info`                           | `organization_domain`       | `"tescan3dim.com"`                 | Domain used in server configuration.                                       |
+| `gui`                            | `algorithm_add_remove_in_menus` | `False`                        | Enables/disables GUI menu for algorithm management.                        |
+| `gui`                            | `use_systray`               | `False`                            | Enables/disables systray GUI integration.                                  |
+| `gui`                            | `icon_path`                 | Path to the installed package resource | Path to the systray icon (supports dynamic prefixes)               |
+| `inference`                       | `device`                   | `"cuda"`                           | Device used for model inference (`"cpu"`, `"cuda"`, `"mps"`).              |
+| `inference`                       | `cuda_visible_devices`     | `"0"`                              | Comma-separated list of visible CUDA GPUs.                                 |
+| `storage`                        | `collection_prefix`         | `""`                               | Prefix applied to object store collections. (useful for AWS s3 store, where unique bucket names are needed)|
+| `storage`                        | `data_store_expire_days`    | `1`                                | Number of days until stored datasets expire.                               |
+| `storage`                        | `access_key_id`             | generated with `UUIDv4`            | Generated access key for storage backend. If `null` is provided, random UUIDv4 is generated. |
+| `storage`                        | `secret_access_key`         | generated with `UUIDv4`            | Generated secret key for storage backend. If `null` is provided, random UUIDv4 is generated. |S
+| `storage.backend_settings` (minio) | `provider`                | `"minio"`                          | Selected backend provider.                                                 |
+| `storage.backend_settings` (minio) | `start_instance`         | `True`                              | Whether to start a local MinIO server.                                     |
+| `storage.backend_settings` (minio) | `port`                   | `9091`                              | MinIO service port.                                                        |
+| `storage.backend_settings` (minio) | `console_port`           | `9090`                              | MinIO admin console port.                                                  |
+| `storage.backend_settings` (minio) | `executable_path`        | `"minio/minio_bin"`                 | Path to the MinIO binary (accepts dynamic prefixes).                |
+| `storage.backend_settings` (minio) | `storage_path`           | `"minio/compox_store"`              | Storage directory used by MinIO (accepts dynamic prefixes).         |
+| `storage.backend_settings` (minio) | `aws_region`             | `None`                              | Optional AWS compatibility region.                                         |
+| `storage.backend_settings` (minio) | `s3_domain_name`         | `None`                              | Optional domain override for S3 compatibility.                             |
+| `storage.backend_settings` (minio) | `s3_endpoint_url`        | Derived from `port`                 | Computed as `http://localhost:{port}`.                                     |
+| `storage.backend_settings` (aws)   | `provider`                | `"aws"`                            | AWS backend selection.                                                     |
+| `storage.backend_settings` (aws)   | `s3_endpoint_url`        | `None`                             | Optional override for S3 endpoint URL.                                     |
+| `storage.backend_settings` (aws)   | `aws_region`             | `None`                             | AWS region (e.g. `us-east-1`).                                             |
+| `storage.backend_settings` (aws)   | `s3_domain_name`         | `None`                             | Domain used for S3-style URLs.                                             |
+
+
+Some fields in the Compox configuration (such as `log_path`, `icon_path`, etc.) support **dynamic prefixes** that resolve to OS-specific or runtime-specific paths. This allows for portability across platforms (e.g., Windows, Linux) and between development and production environments.
+
+#### Supported Prefixes
+
+| Prefix                   | Meaning (Resolved To...)                                                                 |
+|--------------------------|------------------------------------------------------------------------------------------|
+| `LOG_DEFAULT:`           | A platform-dependent log directory:                                                     |
+|                          | - Windows: `%TEMP%/<organization>/<product>`                                            |
+|                          | - Linux/macOS: `/var/log/<organization>/<product>`                                      |
+| `PROGRAMDATA_DEFAULT:`   | A system-wide data directory (Windows only):                                            |
+|                          | - e.g., `%PROGRAMDATA%/<organization>/<product>`                                        |
+|                          | - On Linux/macOS, defaults to `"."` (current dir)                                       |
+
+These are resolved **at runtime** in the `Settings.parse_paths()` validator method.

compox-2.1.0.dev1/README.md

@@ -0,0 +1,119 @@
+# Compox
+Compox is a simple Python execution engine for Tescan applications. It's purpose is to run data processing algorithms written in Python on the server and report the results to the client application such as Tescan 3D Viewer or Picannto.
+
+See the [GitHub pages](https://tescan-3dim.github.io/compox) for full python API reference, docs and tutorials!
+
+## Installation
+Compox can be installed through PyPI: 
+```bash
+pip install compox
+```
+
+You can also install the server by cloning this repository and installing it from source.
+
+Note that some additional dependencies, such as MinIO executable for your operating system, will be downloaded on the first run of the server.
+
+## Running the server
+Once installed into your Python environment, Compox can be run using the `compox` command. This command is available in the virtual environment 
+created during the setup process. Run the following command to list the available commands:
+
+```bash
+compox --help
+```
+
+It's recommended to start with generating an initial configuration file with `compox generate-config`. See the configuration options section below.
+To see the usage of the `generate-config` command, you can run:
+
+```bash
+compox generate-config --help
+```
+
+To run the Compox, use the following command:
+
+```bash
+compox run --config /path/to/config.yaml
+```
+
+### Deploying algorithms
+See Algorithm development tutorial for information about target structure of the deployable algorithms.
+
+Once your algorithm is ready, you can use the `compox deploy-algorithms` command to deploy it to your server. 
+To see the usage of the `deploy-algorithms` command, you can run:
+
+```bash
+compox deploy-algorithms --help
+```
+
+This command will read the algorithm definitions from the folder specified in your configuration file with the `deploy_algorithms_from` key
+and deploy them to the server. For example:
+
+```bash
+compox deploy-algorithms --config /path/to/config.yaml
+```
+
+Note that the server does not need to be running in order to deploy the algorithms.
+
+
+## Configuration
+The server uses pydantic settings for configuration. The options can be either set as runtime arguments, in a yaml file, or using the `COMPOX__` environment variables, in corresponding order of precedence.
+
+- Ensure all paths and URLs are correctly set before running the application.
+- Adjust CUDA settings based on hardware capabilities.
+- Logging paths should be accessible by the application to prevent errors.
+- See the [configuration reference](#configuration-reference) below for detailed configuration options.
+
+<details><summary>Configuration reference</summary>
+
+<br>
+
+### Compox Configuration Reference
+
+| Section                         | Field                      | Default                            | Description                                                                 |
+|----------------------------------|-----------------------------|------------------------------------|-----------------------------------------------------------------------------|
+|                                  | `port`                    | `5461`                             | The main server port used to make requests                                 |
+|                                  | `deploy_algorithms_from`  | `"./algorithms"`                   | Directory for algorithm deployment sources.                                |
+|                                  | `log_path`                | `"LOG_DEFAULT:compox.log"`     | Path to the main log file (supports dynamic prefixes).                     |
+|                                  | `config`                  | `None`                             | Optional config path override.                                             |
+| `info`                           | `product_name`              | `"TESCAN 3D Backend"`              | Product display name.                                                      |
+| `info`                           | `server_tags`               | `[]` → auto appends `"compox"` | Tags attached to the server. `"compox"` is added automatically.        |
+| `info`                           | `group_name`                | `"TESCAN GROUP, a.s."`             | Name of the corporate group.                                               |
+| `info`                           | `organization_name`         | `"TESCAN 3DIM, s.r.o."`            | Full name of the organization.                                             |
+| `info`                           | `organization_domain`       | `"tescan3dim.com"`                 | Domain used in server configuration.                                       |
+| `gui`                            | `algorithm_add_remove_in_menus` | `False`                        | Enables/disables GUI menu for algorithm management.                        |
+| `gui`                            | `use_systray`               | `False`                            | Enables/disables systray GUI integration.                                  |
+| `gui`                            | `icon_path`                 | Path to the installed package resource | Path to the systray icon (supports dynamic prefixes)               |
+| `inference`                       | `device`                   | `"cuda"`                           | Device used for model inference (`"cpu"`, `"cuda"`, `"mps"`).              |
+| `inference`                       | `cuda_visible_devices`     | `"0"`                              | Comma-separated list of visible CUDA GPUs.                                 |
+| `storage`                        | `collection_prefix`         | `""`                               | Prefix applied to object store collections. (useful for AWS s3 store, where unique bucket names are needed)|
+| `storage`                        | `data_store_expire_days`    | `1`                                | Number of days until stored datasets expire.                               |
+| `storage`                        | `access_key_id`             | generated with `UUIDv4`            | Generated access key for storage backend. If `null` is provided, random UUIDv4 is generated. |
+| `storage`                        | `secret_access_key`         | generated with `UUIDv4`            | Generated secret key for storage backend. If `null` is provided, random UUIDv4 is generated. |S
+| `storage.backend_settings` (minio) | `provider`                | `"minio"`                          | Selected backend provider.                                                 |
+| `storage.backend_settings` (minio) | `start_instance`         | `True`                              | Whether to start a local MinIO server.                                     |
+| `storage.backend_settings` (minio) | `port`                   | `9091`                              | MinIO service port.                                                        |
+| `storage.backend_settings` (minio) | `console_port`           | `9090`                              | MinIO admin console port.                                                  |
+| `storage.backend_settings` (minio) | `executable_path`        | `"minio/minio_bin"`                 | Path to the MinIO binary (accepts dynamic prefixes).                |
+| `storage.backend_settings` (minio) | `storage_path`           | `"minio/compox_store"`              | Storage directory used by MinIO (accepts dynamic prefixes).         |
+| `storage.backend_settings` (minio) | `aws_region`             | `None`                              | Optional AWS compatibility region.                                         |
+| `storage.backend_settings` (minio) | `s3_domain_name`         | `None`                              | Optional domain override for S3 compatibility.                             |
+| `storage.backend_settings` (minio) | `s3_endpoint_url`        | Derived from `port`                 | Computed as `http://localhost:{port}`.                                     |
+| `storage.backend_settings` (aws)   | `provider`                | `"aws"`                            | AWS backend selection.                                                     |
+| `storage.backend_settings` (aws)   | `s3_endpoint_url`        | `None`                             | Optional override for S3 endpoint URL.                                     |
+| `storage.backend_settings` (aws)   | `aws_region`             | `None`                             | AWS region (e.g. `us-east-1`).                                             |
+| `storage.backend_settings` (aws)   | `s3_domain_name`         | `None`                             | Domain used for S3-style URLs.                                             |
+
+
+Some fields in the Compox configuration (such as `log_path`, `icon_path`, etc.) support **dynamic prefixes** that resolve to OS-specific or runtime-specific paths. This allows for portability across platforms (e.g., Windows, Linux) and between development and production environments.
+
+#### Supported Prefixes
+
+| Prefix                   | Meaning (Resolved To...)                                                                 |
+|--------------------------|------------------------------------------------------------------------------------------|
+| `LOG_DEFAULT:`           | A platform-dependent log directory:                                                     |
+|                          | - Windows: `%TEMP%/<organization>/<product>`                                            |
+|                          | - Linux/macOS: `/var/log/<organization>/<product>`                                      |
+| `PROGRAMDATA_DEFAULT:`   | A system-wide data directory (Windows only):                                            |
+|                          | - e.g., `%PROGRAMDATA%/<organization>/<product>`                                        |
+|                          | - On Linux/macOS, defaults to `"."` (current dir)                                       |
+
+These are resolved **at runtime** in the `Settings.parse_paths()` validator method.

{compox-2.0.0.dev2 → compox-2.1.0.dev1}/pyproject.toml

@@ -46,7 +46,7 @@ license-files = [
     "NOTICE",
     "THIRD_PARTY_LICENSES.txt",
 ]
-version = "2.0.0.dev2"
+version = "2.1.0.dev1"
 
 [project.scripts]
 compox = "compox.cli:app"

compox-2.1.0.dev1/src/compox/algorithm_debug.py

@@ -0,0 +1,90 @@
+import json, typer
+from typing import Optional, Annotated
+
+from compox.algorithm_utils.LocalDebugSession import LocalDebugSession
+
+app = typer.Typer(help="Commands for local debugging of algorithms")
+
+def parse_params(value: Optional[str]) -> Optional[dict]:
+    """
+    Parse a JSON string of algorithm parameters from the command line.
+
+    Parameters
+    ----------
+    value : str | None
+        A JSON string containing algorithm parameters.
+
+    Returns
+    -------
+    dict | None
+        Parsed dictionary of parameters, or ``None`` if no input was provided.
+
+    Raises
+    ------
+    typer.BadParameter
+        If the string cannot be parsed as valid JSON.
+    """
+
+    if value is None or value == "":
+        return None
+    try:
+        return json.loads(value)
+    except json.JSONDecodeError as e:
+        raise typer.BadParameter(f"--params must be valid JSON: {e}")
+
+@app.command("run")
+def debug_run(
+    algo: str = typer.Option(".", help="Path to algorithm folder"),
+    data: str = typer.Option(..., help="Path to dataset folder (e.g. PNG/TIF stack)"),
+    device: str = typer.Option("cpu", help="Device to run on (cpu/cuda)"),
+    params: Annotated[Optional[str], typer.Option("--params", callback=parse_params, help="Algorithm params (JSON)")] = None,
+):
+    """
+    Run an algorithm locally in debug mode (without Compox backend).
+
+    Parameters
+    ----------
+    algo : str
+        Path to the algorithm directory.
+    data : str
+        Path to the folder containing image slices.
+    device : str
+        Target device to execute the algorithm on.
+    params : dict | None
+        Algorithm parameters.
+    """
+
+    debug(algo_dir=algo, data=data, params=params, device=device)
+
+
+def debug(algo_dir=".", data=".", params=None, device="cpu"):
+    """
+    Run a local debugging session. The function initializes a `LocalDebugSession`, 
+    loads an image stack from the specified folder, uploads it into the temporary 
+    database, and executes the algorithm in the given directory.
+
+    Parameters
+    ----------
+    algo_dir : str, optional
+        Path to the algorithm folder. Default is the current directory (".").
+    data : str, optional
+        Path to the dataset folder (e.g. a directory with PNG/TIF slices).
+    params : dict, optional
+        Dictionary of algorithm parameters.
+    device : str, optional
+        Target device for execution. Default is "cpu".
+
+    Returns
+    -------
+    Any
+        Output returned by the algorithm’s ``run()`` method.
+    """
+
+    sess = LocalDebugSession(device=device)
+    input_ids = sess.load_data(data)
+    out = sess.run(algo_dir, {"input_dataset_ids": input_ids}, args=params or {})
+    print(out)
+    return out
+
+if __name__ == "__main__":
+    app()

{compox-2.0.0.dev2 → compox-2.1.0.dev1}/src/compox/algorithm_utils/AlgorithmDeployer.py

@@ -11,6 +11,7 @@ import shutil
 import glob
 import tempfile
 import zipimport
+import importlib
 import sys
 import re
 import hashlib
@@ -99,12 +100,10 @@ class AlgorithmDeployer:
                 "additional_parameters"
             ]
 
-        self.check_importable = pyproject_toml["tool"]["compox"][
-            "check_importable"
-        ]
-        self.obfuscate = pyproject_toml["tool"]["compox"]["obfuscate"]
-        self.hash_module = pyproject_toml["tool"]["compox"]["hash_module"]
-        self.hash_assets = pyproject_toml["tool"]["compox"]["hash_assets"]
+        self.check_importable = pyproject_toml.get("tool", {}).get("compox", {}).get("check_importable", False)
+        self.obfuscate = pyproject_toml.get("tool", {}).get("compox", {}).get("obfuscate", True)
+        self.hash_module = pyproject_toml.get("tool", {}).get("compox", {}).get("hash_module", True)
+        self.hash_assets = pyproject_toml.get("tool", {}).get("compox", {}).get("hash_assets", True)
 
     def parse_pyproject_toml(self, path_to_algorithm_directory: str) -> dict:
         """
@@ -382,7 +381,9 @@ class AlgorithmDeployer:
                     module_path + ".zip"
                 )
                 if not importable:
-                    raise ValueError("The runner cannot be imported.")
+                    raise ValueError(
+                        "The current environment cannot import the the algorithm module. Check the above logs for more details about the ImportError cause. This check can be disabled by setting check_importable to False in the affected algorithm's pyproject.toml file."
+                    )
 
             zip_bytes = open(module_path + ".zip", "rb").read()
             # store the zip file in the module-store collection
@@ -767,10 +768,14 @@ class AlgorithmDeployer:
         """
         try:
             sys.path.insert(0, path_to_zip)
-            module = zipimport.zipimporter(path_to_zip)
-            module.load_module("Runner")
+            importer = zipimport.zipimporter(path_to_zip)
+            spec = importlib.util.spec_from_loader("Runner", importer)
+            if spec is not None:
+                module = importlib.util.module_from_spec(spec)
+                importer.exec_module(module)
             return True
-        except Exception as e:
+        except ImportError as e:
+            logger.error(f"ImportError: {e}")
             return False
 
     @staticmethod