compox 2.0.0.dev3__tar.gz → 2.1.0.dev1__tar.gz

{compox-2.0.0.dev3 → compox-2.1.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compox
-Version: 2.0.0.dev3
+Version: 2.1.0.dev1
 Author-Email: Jan Matula <jan.matula@tescan.com>, =?utf-8?q?Old=C5=99ich_Kodym?= <oldrich.kodym@tescan.com>
 License-Expression: Apache-2.0 AND LicenseRef-Commons-Clause-1.0
 License-File: LICENSE
@@ -35,6 +35,9 @@ Description-Content-Type: text/markdown
 
 # Compox
 Compox is a simple Python execution engine for Tescan applications. It's purpose is to run data processing algorithms written in Python on the server and report the results to the client application such as Tescan 3D Viewer or Picannto.
+
+See the [GitHub pages](https://tescan-3dim.github.io/compox) for full python API reference, docs and tutorials!
+
 ## Installation
 Compox can be installed through PyPI: 
 ```bash

{compox-2.0.0.dev3 → compox-2.1.0.dev1}/README.md

@@ -1,5 +1,8 @@
 # Compox
 Compox is a simple Python execution engine for Tescan applications. It's purpose is to run data processing algorithms written in Python on the server and report the results to the client application such as Tescan 3D Viewer or Picannto.
+
+See the [GitHub pages](https://tescan-3dim.github.io/compox) for full python API reference, docs and tutorials!
+
 ## Installation
 Compox can be installed through PyPI: 
 ```bash

{compox-2.0.0.dev3 → compox-2.1.0.dev1}/pyproject.toml

@@ -46,7 +46,7 @@ license-files = [
     "NOTICE",
     "THIRD_PARTY_LICENSES.txt",
 ]
-version = "2.0.0.dev3"
+version = "2.1.0.dev1"
 
 [project.scripts]
 compox = "compox.cli:app"

compox-2.1.0.dev1/src/compox/algorithm_debug.py

@@ -0,0 +1,90 @@
+import json, typer
+from typing import Optional, Annotated
+
+from compox.algorithm_utils.LocalDebugSession import LocalDebugSession
+
+app = typer.Typer(help="Commands for local debugging of algorithms")
+
+def parse_params(value: Optional[str]) -> Optional[dict]:
+    """
+    Parse a JSON string of algorithm parameters from the command line.
+
+    Parameters
+    ----------
+    value : str | None
+        A JSON string containing algorithm parameters.
+
+    Returns
+    -------
+    dict | None
+        Parsed dictionary of parameters, or ``None`` if no input was provided.
+
+    Raises
+    ------
+    typer.BadParameter
+        If the string cannot be parsed as valid JSON.
+    """
+
+    if value is None or value == "":
+        return None
+    try:
+        return json.loads(value)
+    except json.JSONDecodeError as e:
+        raise typer.BadParameter(f"--params must be valid JSON: {e}")
+
+@app.command("run")
+def debug_run(
+    algo: str = typer.Option(".", help="Path to algorithm folder"),
+    data: str = typer.Option(..., help="Path to dataset folder (e.g. PNG/TIF stack)"),
+    device: str = typer.Option("cpu", help="Device to run on (cpu/cuda)"),
+    params: Annotated[Optional[str], typer.Option("--params", callback=parse_params, help="Algorithm params (JSON)")] = None,
+):
+    """
+    Run an algorithm locally in debug mode (without Compox backend).
+
+    Parameters
+    ----------
+    algo : str
+        Path to the algorithm directory.
+    data : str
+        Path to the folder containing image slices.
+    device : str
+        Target device to execute the algorithm on.
+    params : dict | None
+        Algorithm parameters.
+    """
+
+    debug(algo_dir=algo, data=data, params=params, device=device)
+
+
+def debug(algo_dir=".", data=".", params=None, device="cpu"):
+    """
+    Run a local debugging session. The function initializes a `LocalDebugSession`, 
+    loads an image stack from the specified folder, uploads it into the temporary 
+    database, and executes the algorithm in the given directory.
+
+    Parameters
+    ----------
+    algo_dir : str, optional
+        Path to the algorithm folder. Default is the current directory (".").
+    data : str, optional
+        Path to the dataset folder (e.g. a directory with PNG/TIF slices).
+    params : dict, optional
+        Dictionary of algorithm parameters.
+    device : str, optional
+        Target device for execution. Default is "cpu".
+
+    Returns
+    -------
+    Any
+        Output returned by the algorithm’s ``run()`` method.
+    """
+
+    sess = LocalDebugSession(device=device)
+    input_ids = sess.load_data(data)
+    out = sess.run(algo_dir, {"input_dataset_ids": input_ids}, args=params or {})
+    print(out)
+    return out
+
+if __name__ == "__main__":
+    app()

{compox-2.0.0.dev3 → compox-2.1.0.dev1}/src/compox/algorithm_utils/AlgorithmDeployer.py

@@ -11,6 +11,7 @@ import shutil
 import glob
 import tempfile
 import zipimport
+import importlib
 import sys
 import re
 import hashlib
@@ -99,12 +100,10 @@ class AlgorithmDeployer:
                 "additional_parameters"
             ]
 
-        self.check_importable = pyproject_toml["tool"]["compox"][
-            "check_importable"
-        ]
-        self.obfuscate = pyproject_toml["tool"]["compox"]["obfuscate"]
-        self.hash_module = pyproject_toml["tool"]["compox"]["hash_module"]
-        self.hash_assets = pyproject_toml["tool"]["compox"]["hash_assets"]
+        self.check_importable = pyproject_toml.get("tool", {}).get("compox", {}).get("check_importable", False)
+        self.obfuscate = pyproject_toml.get("tool", {}).get("compox", {}).get("obfuscate", True)
+        self.hash_module = pyproject_toml.get("tool", {}).get("compox", {}).get("hash_module", True)
+        self.hash_assets = pyproject_toml.get("tool", {}).get("compox", {}).get("hash_assets", True)
 
     def parse_pyproject_toml(self, path_to_algorithm_directory: str) -> dict:
         """
@@ -382,7 +381,9 @@ class AlgorithmDeployer:
                     module_path + ".zip"
                 )
                 if not importable:
-                    raise ValueError("The runner cannot be imported.")
+                    raise ValueError(
+                        "The current environment cannot import the the algorithm module. Check the above logs for more details about the ImportError cause. This check can be disabled by setting check_importable to False in the affected algorithm's pyproject.toml file."
+                    )
 
             zip_bytes = open(module_path + ".zip", "rb").read()
             # store the zip file in the module-store collection
@@ -767,10 +768,14 @@ class AlgorithmDeployer:
         """
         try:
             sys.path.insert(0, path_to_zip)
-            module = zipimport.zipimporter(path_to_zip)
-            module.load_module("Runner")
+            importer = zipimport.zipimporter(path_to_zip)
+            spec = importlib.util.spec_from_loader("Runner", importer)
+            if spec is not None:
+                module = importlib.util.module_from_spec(spec)
+                importer.exec_module(module)
             return True
-        except Exception as e:
+        except ImportError as e:
+            logger.error(f"ImportError: {e}")
             return False
 
     @staticmethod

compox-2.1.0.dev1/src/compox/algorithm_utils/LocalDebugSession.py

@@ -0,0 +1,307 @@
+import os, glob, io, uuid, h5py
+import numpy as np
+from PIL import Image, ImageSequence
+
+from compox.tasks.DebuggingTaskHandler import DebuggingTaskHandler
+
+class LocalDebugSession:
+    """
+    Helper class for running Compox algorithms locally in a simplified
+    "debug" environment.
+
+    It sets up a fake `TaskHandler`, uploads image data to an in-memory database, 
+    and runs the specified algorithm. It allows developers to test and debug 
+    algorithms without deploying them to a full Compox server.
+
+    Parameters
+    ----------
+    task_id : str, optional
+        Unique ID for the debugging task. Default is `"local-debug"`.
+    device : str, optional
+        Target device to run the algorithm on. Default is `"cpu"`.
+    tmp_bucket : str, optional
+        Temporary in-memory storage bucket name for the mock database.
+        Default is `"data-store"`.
+    """
+
+    def __init__(self, task_id="local-debug", device="cpu", tmp_bucket="data-store"):
+        self.task = DebuggingTaskHandler(task_id)
+        self.task.set_as_current_task_handler()
+        self.bucket = tmp_bucket
+        self.device = device
+
+
+    def _to_h5_bytes(self, arr):
+        """
+        Convert a NumPy array into an in-memory HDF5 file. Each image slice 
+        is stored as a small binary HDF5 object (byte array), which mimics 
+        the format expected by the Compox database.
+
+        Parameters
+        ----------
+        arr : np.ndarray
+            Image array to be serialized into HDF5 format.
+
+        Returns
+        -------
+        bytes
+            The in-memory representation of the HDF5 file.
+        """
+
+        bio = io.BytesIO()
+        with h5py.File(bio, "w") as f:
+            f.create_dataset("image", data=arr)
+        return bio.getvalue()
+
+
+    def load_data(self, source):
+        """
+        Load image data from a specified source into the local debug database.
+        
+        Parameters
+        ----------
+        source : str
+            Path to the data source. Can be a folder containing image slices,
+            a single image file (PNG/JPEG/TIFF), a .npy file, or an HDF5 file.
+
+        Returns
+        -------
+        list[str]
+            A list of generated object IDs (UUIDs) corresponding to the stored
+            HDF5 image slices.
+        """
+        if os.path.isdir(source):
+            return self.load_from_folder(source)
+        else:
+            ext = os.path.splitext(source)[1].lower()
+            if ext in (".tif", ".tiff"):
+                return self.load_from_tiff(source)
+            elif ext in (".png", ".jpg", ".jpeg"):
+                return self.load_from_pil_image(source)
+            elif ext == ".npy":
+                return self.load_from_npy(source)
+            elif ext in (".h5", ".hdf5"):
+                return self.load_from_h5(source)
+            else:
+                raise ValueError(f"Unsupported file type: {ext}")
+            
+    
+    def load_from_npy(self, filepath: str) -> list[str]:
+        """
+        Load a .npy file and store it as individual 2D slices in the fake database.
+
+        Parameters
+        ----------
+        filepath : str
+            Path to the .npy file containing a 2D or 3D NumPy array.
+
+        Returns
+        -------
+        list[str]
+            A list of generated object IDs (UUIDs) corresponding to the stored
+            HDF5 image slices.
+        """
+        arr = np.load(filepath)
+        nd = arr.ndim
+
+        slices_bytes: list[bytes] = []
+
+        if nd == 3:
+            for z in range(arr.shape[0]):
+                slices_bytes.append(self._to_h5_bytes(arr[z]))
+        elif nd == 2:
+            slices_bytes.append(self._to_h5_bytes(arr))
+        else:
+            raise ValueError(
+                f"Unsupported numpy array ndim={nd} in '{filepath}'. "
+                "Expected 2D (H, W) or 3D (Z, H, W)."
+            )
+
+        ids = [str(uuid.uuid4()) for _ in slices_bytes]
+        self.task.database_connection.put_objects(self.bucket, ids, slices_bytes)
+        return ids
+    
+
+    def load_from_h5(self, filepath: str) -> list[str]:
+        """
+        Load a 2D or 3D dataset from an HDF5 file and store it as 2D slices.
+
+        Parameters
+        ----------
+        filepath : str
+            Path to the .h5/.hdf5 file.
+
+        Returns
+        -------
+        list[str]
+            List of object IDs corresponding to stored slices.
+
+        Raises
+        ------
+        KeyError
+            If no suitable 2D/3D dataset can be found in the file.
+        ValueError
+            If the selected dataset has unsupported number of dimensions.
+        """
+        slices_bytes: list[bytes] = []
+
+        def _find_datasets(f):
+            found = []
+            def visitor(name, obj):
+                if isinstance(obj, h5py.Dataset) and obj.ndim in (2, 3):
+                    found.append((name, obj))
+            f.visititems(visitor)
+            return found
+
+        with h5py.File(filepath, "r") as f:
+
+            datasets = _find_datasets(f)
+
+            if not datasets:
+                raise KeyError(
+                    f"No suitable 2D/3D dataset found anywhere in '{filepath}'. "
+                    f"Available top-level groups: {list(f.keys())}"
+                )
+
+            _, dset = datasets[0]
+
+            if dset.ndim == 2:
+                slices_bytes.append(self._to_h5_bytes(dset[()]))
+
+            elif dset.ndim == 3:
+                for z in range(dset.shape[0]):
+                    slices_bytes.append(self._to_h5_bytes(dset[z, ...]))
+            else:
+                raise ValueError("Internal error: filtered ndim != 2/3")
+
+        ids = [str(uuid.uuid4()) for _ in slices_bytes]
+        self.task.database_connection.put_objects(self.bucket, ids, slices_bytes)
+        return ids
+
+
+    def load_from_pil_image(self, filepath: str) -> list[str]:
+        """
+        Load a single image file (PNG/JPEG/...) and store it as one 2D slice
+        in the debug database.
+
+        Parameters
+        ----------
+        filepath : str
+            Path to the image file.
+
+        Returns
+        -------
+        list[str]
+            List with a single object ID corresponding to the stored slice.
+        """
+        with Image.open(filepath) as img:
+            arr = np.asarray(img)
+
+        slice_bytes = self._to_h5_bytes(arr)
+        obj_id = str(uuid.uuid4())
+
+        self.task.database_connection.put_objects(self.bucket, [obj_id], [slice_bytes])
+        return [obj_id]
+
+    def load_from_tiff(self, filepath: str) -> list[str]:
+        """
+        Load a TIFF file (single-page or multi-page) and store each page
+        as a separate 2D slice in the fake database.
+
+        Parameters
+        ----------
+        filepath : str
+            Path to the .tif/.tiff file.
+
+        Returns
+        -------
+        list[str]
+            List of object IDs corresponding to stored slices.
+        """
+        slices_bytes: list[bytes] = []
+
+        with Image.open(filepath) as im:
+            frames = list(ImageSequence.Iterator(im))
+
+            if not frames:
+                raise ValueError(f"TIFF file '{filepath}' contains no frames.")
+
+            for frame in frames:
+                arr = np.asarray(frame)
+                slices_bytes.append(self._to_h5_bytes(arr))
+
+        ids = [str(uuid.uuid4()) for _ in slices_bytes]
+        self.task.database_connection.put_objects(self.bucket, ids, slices_bytes)
+        return ids
+
+
+    def load_from_folder(self, folder, exts=(".tif", ".tiff", ".png", ".jpg", ".jpeg")):
+        """
+        Load a folder of image slices (e.g., PNG/TIFF) into the local database.
+        Each image is read from disk, converted to a NumPy array, serialized
+        into an HDF5 byte object, and uploaded into the fake database.
+
+        Parameters
+        ----------
+        folder : str
+            Path to the folder containing image slices.
+        exts : tuple of str, optional
+            Allowed image extensions. Default is `(".tif", ".tiff", ".png", ".jpg", ".jpeg")`.
+
+        Returns
+        -------
+        list[str]
+            A list of generated object IDs (UUIDs) corresponding to the stored
+            HDF5 image slices.
+
+        Raises
+        ------
+        ValueError
+            If no supported image slices are found in the folder.
+        """
+
+        paths = [p for p in sorted(glob.glob(os.path.join(folder, "*"))) if p.lower().endswith(exts)]
+        if not paths:
+            raise ValueError(
+                f"No supported image slices found in folder '{folder}'. "
+                f"Supported extensions: {exts}"
+            )
+        ext = os.path.splitext(paths[0])[1].lower()
+        if ext not in exts:
+            raise ValueError(
+                f"Folder loader expects slice image formats {exts}, "
+                f"but found '{ext}' in folder: {folder}. "
+                "If you want to load numpy/HDF5 volumes, pass the file path directly "
+                "instead of a folder."
+            )
+        files = []
+        for p in paths:
+            with Image.open(p) as img:
+                files.append(self._to_h5_bytes(np.asarray(img)))
+        ids = [str(uuid.uuid4()) for _ in files]
+        self.task.database_connection.put_objects(self.bucket, ids, files)
+        return ids
+
+
+    def run(self, algo_dir, inputs: dict, args: dict | None = None):
+        """
+        Execute a Compox algorithm locally. This method fetches and runs the algorithm 
+        from the specified folder.
+
+        Parameters
+        ----------
+        algo_dir : str
+            Path to the directory containing the algorithm implementation.
+        inputs : dict
+            Input metadata for the algorithm (e.g. dataset IDs).
+        args : dict, optional
+            Algorithm parameters passed as keyword arguments.
+
+        Returns
+        -------
+        Any
+            The algorithm's output object, as returned by its `run()` method.
+        """
+
+        runner = self.task.fetch_algorithm(algo_dir, device=self.device)
+        return runner.run(inputs, args=args or {})

{compox-2.0.0.dev3 → compox-2.1.0.dev1}/src/compox/algorithm_utils/io_schemas.py

@@ -12,6 +12,10 @@ class DataSchema(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
 
+class GenericSchema(DataSchema):
+    data: np.ndarray
+
+
 class SegmentationSchema(DataSchema):
     mask: np.ndarray
 
@@ -28,9 +32,9 @@ class ImageSchema(DataSchema):
             raise ValueError(
                 f"Image must be 2D (got {v.ndim} dimensions), grayscale or RGB (got {1 if v.ndim == 2 else v.shape[0]} channels)."
             )
-        if v.dtype not in [np.uint8, np.uint16, np.float32, np.float16]:
+        if v.dtype not in [np.uint8, np.uint16, np.float64, np.float32, np.float16]:
             raise ValueError(
-                f"Image must be uint8, uint16, float32 or float16 (got {v.dtype})."
+                f"Image must be uint8, uint16, float64, float32 or float16 (got {v.dtype})."
             )
         return v
     

{compox-2.0.0.dev3 → compox-2.1.0.dev1}/src/compox/cli.py

@@ -11,11 +11,13 @@ import subprocess
 import os
 from typing import Optional
 from compox.config.server_settings import Settings
+from compox.algorithm_debug import app as debug_app
 
 app = typer.Typer(
     help="This CLI tool contains commands for running the Compox."
 )
 
+app.add_typer(debug_app, name="debug")
 
 @app.command(
     "run",

{compox-2.0.0.dev3 → compox-2.1.0.dev1}/src/compox/config/server_settings.py

@@ -61,8 +61,8 @@ class MinioSettings(BaseModel):
     model_config = ConfigDict(extra='forbid')
     provider: Literal["minio"] = "minio"
     start_instance: bool = True
-    port: int = 9091
-    console_port: int = 9090
+    port: int = 5483
+    console_port: int = 5482
     executable_path: str = os.path.join("minio", "minio_bin") if os.name != "nt" else os.path.join("minio", "minio.exe")
     storage_path: str = os.path.join("minio", "compox_store")
     # derived attributes
@@ -186,7 +186,7 @@ class Settings(BaseSettings, cli_parse_args=_PARSE_CLI):
     
     model_config = ConfigDict(extra='forbid', env_nested_delimiter='__', env_prefix="COMPOX_")
     
-    port: int = 5461
+    port: int = 5481
     deploy_algorithms_from: str = "./algorithms"  # points to default foo/bar algorithms
     
     info: CompoxInfo = CompoxInfo()

{compox-2.0.0.dev3 → compox-2.1.0.dev1}/tests/algorithms/dummy/README.md

@@ -56,7 +56,7 @@ additional_parameters = [
 ]
 ```
 
-The `check_importable` field is used to check if the algorithm can be imported. If set to `true`, compox will check if the algorithm can be imported before deploying it as a service. (NOTE: THIS CURRENTLY DOES NOT WORK).
+The `check_importable` field is used to check if the algorithm can be imported. If set to `true`, compox will check if the algorithm can be imported before deploying it as a service.
 
 ```toml
 check_importable = false

{compox-2.0.0.dev3 → compox-2.1.0.dev1}/tests/test_algorithm_deployer.py

@@ -74,7 +74,7 @@ def invalid_alg_dir(tmp_path):
     d = tmp_path / "invalid"
     d.mkdir()
     (d / "pyproject.toml").write_text(
-        toml.dumps({"project": {"name": "foo", "version": "0.1"}})
+        toml.dumps({"some_invalid_section": {"foo": "bar"}})
     )
     (d / "Runner.py").write_text("class Runner: pass")
     return str(d)

{compox-2.0.0.dev3 → compox-2.1.0.dev1}/tests/test_server.yaml

@@ -1,4 +1,4 @@
-port: 5461
+port: 5484
 deploy_algorithms_from: "./algorithms"
 
 info:
@@ -17,8 +17,8 @@ storage:
   backend_settings:
     provider: "minio"
     start_instance: true
-    port: 9091
-    console_port: 9090
+    port: 5486
+    console_port: 5485
     executable_path: "minio/TESCAN3DBackendDB.exe"
     storage_path: "minio/compox_store"