careamics 0.1.0rc1__tar.gz → 0.1.0rc2__tar.gz

{careamics-0.1.0rc1 → careamics-0.1.0rc2}/.github/workflows/ci.yml

@@ -40,7 +40,7 @@ jobs:
             - uses: actions/checkout@v4
 
             - name: 🐍 Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v4
+              uses: actions/setup-python@v5
               with:
                   python-version: ${{ matrix.python-version }}
                   cache-dependency-path: pyproject.toml
@@ -91,7 +91,7 @@ jobs:
                   fetch-depth: 0
 
             - name: 🐍 Set up Python
-              uses: actions/setup-python@v4
+              uses: actions/setup-python@v5
               with:
                   python-version: 3.x
 

{careamics-0.1.0rc1 → careamics-0.1.0rc2}/.pre-commit-config.yaml

@@ -10,23 +10,23 @@ ci:
 
 repos:
     - repo: https://github.com/abravalheri/validate-pyproject
-      rev: v0.14
+      rev: v0.15
       hooks:
           - id: validate-pyproject
 
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.0.292
+      rev: v0.1.6
       hooks:
           - id: ruff
             args: [--fix, --target-version, py38]
 
     - repo: https://github.com/psf/black
-      rev: 23.9.1
+      rev: 23.11.0
       hooks:
           - id: black
 
     - repo: https://github.com/pre-commit/mirrors-mypy
-      rev: v1.5.1
+      rev: v1.7.1
       hooks:
           - id: mypy
             files: ^src/
@@ -42,7 +42,7 @@ repos:
 
   # jupyter linting and formatting
     - repo: https://github.com/nbQA-dev/nbQA
-      rev: 1.7.0
+      rev: 1.7.1
       hooks:
           - id: nbqa-ruff
             args: [--fix]

{careamics-0.1.0rc1 → careamics-0.1.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: careamics
-Version: 0.1.0rc1
+Version: 0.1.0rc2
 Summary: Toolbox for running N2V and friends.
 Project-URL: homepage, https://careamics.github.io/
 Project-URL: repository, https://github.com/CAREamics/careamics
@@ -14,6 +14,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Typing :: Typed
 Requires-Python: >=3.8
 Requires-Dist: bioimageio-core
@@ -26,6 +27,7 @@ Requires-Dist: torchvision
 Requires-Dist: zarr
 Provides-Extra: all
 Requires-Dist: careamics-portfolio; extra == 'all'
+Requires-Dist: ipython; extra == 'all'
 Requires-Dist: itkwidgets; extra == 'all'
 Requires-Dist: jupyter; extra == 'all'
 Requires-Dist: pre-commit; extra == 'all'
@@ -43,12 +45,11 @@ Requires-Dist: ipython; extra == 'notebooks'
 Requires-Dist: itkwidgets; extra == 'notebooks'
 Requires-Dist: jupyter; extra == 'notebooks'
 Requires-Dist: torchsummary; extra == 'notebooks'
+Requires-Dist: wandb; extra == 'notebooks'
 Provides-Extra: test
 Requires-Dist: pytest; extra == 'test'
 Requires-Dist: pytest-cov; extra == 'test'
 Requires-Dist: wandb; extra == 'test'
-Provides-Extra: wandb
-Requires-Dist: wandb; extra == 'wandb'
 Description-Content-Type: text/markdown
 
 <p align="center">

{careamics-0.1.0rc1 → careamics-0.1.0rc2}/pyproject.toml

@@ -1,15 +1,12 @@
-# https://peps.python.org/pep-0517/
-
 [build-system]
 requires = ["hatchling", "hatch-vcs"]
 build-backend = "hatchling.build"
-# read more about configuring hatch at:
-# https://hatch.pypa.io/latest/config/build/
 
 # https://hatch.pypa.io/latest/config/metadata/
 [tool.hatch.version]
 source = "vcs"
 
+# https://hatch.pypa.io/latest/config/build/
 [tool.hatch.build.targets.wheel]
 only-include = ["src"]
 sources = ["src"]
@@ -35,6 +32,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "License :: OSI Approved :: BSD License",
     "Typing :: Typed",
 ]
@@ -53,12 +51,9 @@ dependencies = [
 # development dependencies and tooling
 dev = ["pre-commit", "pytest", "pytest-cov"]
 
-# test for ci
+# for ci
 test = ["pytest", "pytest-cov", "wandb"]
 
-# use wandb for logging
-wandb = ["wandb"]
-
 # notebooks
 notebooks = [
     "jupyter",
@@ -66,6 +61,7 @@ notebooks = [
     "itkwidgets",
     "torchsummary",
     "ipython",
+    "wandb",
 ]
 
 # all
@@ -78,43 +74,32 @@ all = [
     "careamics-portfolio",
     "itkwidgets",
     "torchsummary",
+    "ipython",
 ]
 
 [project.urls]
 homepage = "https://careamics.github.io/"
 repository = "https://github.com/CAREamics/careamics"
 
-# https://beta.ruff.rs/docs
+# https://docs.astral.sh/ruff/
 [tool.ruff]
 line-length = 88
 target-version = "py38"
 src = ["src"]
 select = [
-    "E",  # style errors
-    "W",  # style warnings
-    "F",  # flakes
-    "D",  # pydocstyle
-    "I",  # isort
-    "UP", # pyupgrade
-    # "S",    # bandit
+    "E",    # style errors
+    "W",    # style warnings
+    "F",    # flakes
+    "I",    # isort
+    "UP",   # pyupgrade
     "C4",   # flake8-comprehensions
     "B",    # flake8-bugbear
     "A001", # flake8-builtins
     "RUF",  # ruff-specific rules
+    "TCH",  # flake8-type-checking
+    "TID",  # flake8-tidy-imports
 ]
 ignore = [
-    "D100", # Missing docstring in public module
-    "D107", # Missing docstring in __init__
-    "D203", # 1 blank line required before class docstring
-    "D212", # Multi-line docstring summary should start at the first line
-    "D213", # Multi-line docstring summary should start at the second line
-    "D401", # First line should be in imperative mood
-    "D413", # Missing blank line after last section
-    "D416", # Section name should end with a colon
-
-    # incompatibility with mypy
-    "RUF005", # collection-literal-concatenation, in prediction_utils.py:30
-
     # version specific
     "UP006", # Replace typing.List by list, mandatory for py3.8
     "UP007", # Replace Union by |, mandatory for py3.9
@@ -122,12 +107,8 @@ ignore = [
 ignore-init-module-imports = true
 show-fixes = true
 
-[tool.ruff.pydocstyle]
-convention = "numpy"
-
 [tool.ruff.per-file-ignores]
-"tests/*.py" = ["D", "S"]
-"setup.py" = ["D"]
+"tests/*.py" = ["S"]
 
 [tool.black]
 line-length = 88
@@ -141,10 +122,6 @@ allow_untyped_calls = false
 disallow_any_generics = false
 ignore_missing_imports = false
 
-# # module specific overrides
-# [[tool.mypy.overrides]]
-# module = ["numpy.*",]
-# ignore_errors = true
 # https://docs.pytest.org/en/6.2.x/customize.html
 [tool.pytest.ini_options]
 minversion = "6.0"
@@ -153,8 +130,7 @@ filterwarnings = [
     # "error",
     # "ignore::UserWarning",
 ]
-markers = ["gpu: marks tests as requiring gpu"]
-
+markers = ["gpu: mark tests as requiring gpu"]
 
 # https://coverage.readthedocs.io/en/6.4/config.html
 [tool.coverage.report]
@@ -162,9 +138,14 @@ exclude_lines = [
     "pragma: no cover",
     "if TYPE_CHECKING:",
     "@overload",
-    "except ImportError",
     "\\.\\.\\.",
+    "except ImportError:",
     "raise NotImplementedError()",
+    "except PackageNotFoundError:",
+    "if torch.cuda.is_available():",
+    "except UsageError as e:",
+    "except ModuleNotFoundError:",
+    "except KeyboardInterrupt:",
 ]
 
 [tool.coverage.run]
@@ -178,7 +159,6 @@ ignore = [
     ".github_changelog_generator",
     ".pre-commit-config.yaml",
     ".ruff_cache/**/*",
-    "setup.py",
     "tests/**/*",
 ]
 

{careamics-0.1.0rc1 → careamics-0.1.0rc2}/src/careamics/bioimage/__init__.py

@@ -1,7 +1,7 @@
 """Provide utilities for exporting models to BioImage model zoo."""
 
 __all__ = [
-    "build_zip_model",
+    "save_bioimage_model",
     "import_bioimage_model",
     "get_default_model_specs",
     "PYTORCH_STATE_DICT",
@@ -9,7 +9,7 @@ __all__ = [
 
 from .io import (
     PYTORCH_STATE_DICT,
-    build_zip_model,
-    get_default_model_specs,
     import_bioimage_model,
+    save_bioimage_model,
 )
+from .rdf import get_default_model_specs

careamics-0.1.0rc2/src/careamics/bioimage/io.py

@@ -0,0 +1,182 @@
+"""Export to bioimage.io format."""
+from pathlib import Path
+from typing import Union
+
+import torch
+from bioimageio.core import load_resource_description
+from bioimageio.core.build_spec import build_model
+
+from careamics.config.config import Configuration
+from careamics.utils.context import cwd
+
+PYTORCH_STATE_DICT = "pytorch_state_dict"
+
+
+def save_bioimage_model(
+    path: Union[str, Path],
+    config: Configuration,
+    specs: dict,
+) -> None:
+    """
+    Build bioimage model zip file from model RDF data.
+
+    Parameters
+    ----------
+    path : Union[str, Path]
+        Path to the model zip file.
+    config : Configuration
+        Configuration object.
+    specs : dict
+        Model RDF dict.
+    """
+    workdir = config.working_directory
+
+    # temporary folder
+    temp_folder = Path.home().joinpath(".careamics", "bmz_tmp")
+    temp_folder.mkdir(exist_ok=True, parents=True)
+
+    # change working directory to the temp folder
+    with cwd(temp_folder):
+        # load best checkpoint
+        checkpoint_path = workdir.joinpath(
+            f"{config.experiment_name}_best.pth"
+        ).absolute()
+        checkpoint = torch.load(checkpoint_path, map_location="cpu")
+
+        # save chekpoint entries in separate files
+        weight_path = Path("model_weights.pth")
+        torch.save(checkpoint["model_state_dict"], weight_path)
+
+        optim_path = Path("optim.pth")
+        torch.save(checkpoint["optimizer_state_dict"], optim_path)
+
+        scheduler_path = Path("scheduler.pth")
+        torch.save(checkpoint["scheduler_state_dict"], scheduler_path)
+
+        grad_path = Path("grad.pth")
+        torch.save(checkpoint["grad_scaler_state_dict"], grad_path)
+
+        config_path = Path("config.pth")
+        torch.save(config.model_dump(), config_path)
+
+        # create attachments
+        attachments = [
+            str(optim_path),
+            str(scheduler_path),
+            str(grad_path),
+            str(config_path),
+        ]
+
+        # create requirements file
+        requirements = Path("requirements.txt")
+        with open(requirements, "w") as f:
+            f.write("git+https://github.com/CAREamics/careamics.git")
+
+        algo_config = config.algorithm
+        specs.update(
+            {
+                "weight_type": PYTORCH_STATE_DICT,
+                "weight_uri": str(weight_path),
+                "architecture": "careamics.models.unet.UNet",
+                "pytorch_version": torch.__version__,
+                "model_kwargs": {
+                    "conv_dim": algo_config.get_conv_dim(),
+                    "depth": algo_config.model_parameters.depth,
+                    "num_channels_init": algo_config.model_parameters.num_channels_init,
+                },
+                "dependencies": "pip:" + str(requirements),
+                "attachments": {"files": attachments},
+            }
+        )
+
+        if config.algorithm.is_3D:
+            specs["tags"].append("3D")
+        else:
+            specs["tags"].append("2D")
+
+        # build model zip
+        build_model(
+            output_path=Path(path).absolute(),
+            **specs,
+        )
+
+        # remove temporary files
+        for file in temp_folder.glob("*"):
+            file.unlink()
+
+    # delete temporary folder
+    temp_folder.rmdir()
+
+
+def import_bioimage_model(model_path: Union[str, Path]) -> Path:
+    """
+    Load configuration and weights from a bioimage zip model.
+
+    Parameters
+    ----------
+    model_path : Union[str, Path]
+        Path to the bioimage.io archive.
+
+    Returns
+    -------
+    Path
+        Path to the checkpoint.
+
+    Raises
+    ------
+    ValueError
+        If the model format is invalid.
+    FileNotFoundError
+        If the checkpoint file was not found.
+    """
+    model_path = Path(model_path)
+
+    # check the model extension (should be a zip file).
+    if model_path.suffix != ".zip":
+        raise ValueError("Invalid model format. Expected bioimage model zip file.")
+
+    # load the model
+    rdf = load_resource_description(model_path)
+
+    # create a valid checkpoint file from weights and attached files
+    basedir = model_path.parent.joinpath("rdf_model")
+    basedir.mkdir(exist_ok=True)
+    optim_path = None
+    scheduler_path = None
+    grad_path = None
+    config_path = None
+    weight_path = None
+
+    if rdf.weights.get(PYTORCH_STATE_DICT) is not None:
+        weight_path = rdf.weights.get(PYTORCH_STATE_DICT).source
+
+    for file in rdf.attachments.files:
+        if file.name.endswith("optim.pth"):
+            optim_path = file
+        elif file.name.endswith("scheduler.pth"):
+            scheduler_path = file
+        elif file.name.endswith("grad.pth"):
+            grad_path = file
+        elif file.name.endswith("config.pth"):
+            config_path = file
+
+    if (
+        weight_path is None
+        or optim_path is None
+        or scheduler_path is None
+        or grad_path is None
+        or config_path is None
+    ):
+        raise FileNotFoundError(f"No valid checkpoint file was found in {model_path}.")
+
+    checkpoint = {
+        "model_state_dict": torch.load(weight_path, map_location="cpu"),
+        "optimizer_state_dict": torch.load(optim_path, map_location="cpu"),
+        "scheduler_state_dict": torch.load(scheduler_path, map_location="cpu"),
+        "grad_scaler_state_dict": torch.load(grad_path, map_location="cpu"),
+        "config": torch.load(config_path, map_location="cpu"),
+    }
+    checkpoint_path = basedir.joinpath("checkpoint.pth")
+    torch.save(checkpoint, checkpoint_path)
+
+    return checkpoint_path

careamics-0.1.0rc2/src/careamics/bioimage/rdf.py

@@ -0,0 +1,105 @@
+"""RDF related methods."""
+from pathlib import Path
+
+
+def _get_model_doc(name: str) -> str:
+    """
+    Return markdown documentation path for the provided model.
+
+    Parameters
+    ----------
+    name : str
+        Model's name.
+
+    Returns
+    -------
+    str
+        Path to the model's markdown documentation.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the documentation file was not found.
+    """
+    doc = Path(__file__).parent.joinpath("docs").joinpath(f"{name}.md")
+    if doc.exists():
+        return str(doc.absolute())
+    else:
+        raise FileNotFoundError(f"Documentation for {name} was not found.")
+
+
+def get_default_model_specs(
+    name: str, mean: float, std: float, is_3D: bool = False
+) -> dict:
+    """
+    Return the default bioimage.io specs for the provided model's name.
+
+    Currently only supports `Noise2Void` model.
+
+    Parameters
+    ----------
+    name : str
+        Algorithm's name.
+    mean : float
+        Mean of the dataset.
+    std : float
+        Std of the dataset.
+    is_3D : bool, optional
+        Whether the model is 3D or not, by default False.
+
+    Returns
+    -------
+    dict
+        Model specs compatible with bioimage.io export.
+    """
+    rdf = {
+        "name": "Noise2Void",
+        "description": "Self-supervised denoising.",
+        "license": "BSD-3-Clause",
+        "authors": [
+            {"name": "Alexander Krull"},
+            {"name": "Tim-Oliver Buchholz"},
+            {"name": "Florian Jug"},
+        ],
+        "cite": [
+            {
+                "doi": "10.48550/arXiv.1811.10980",
+                "text": 'A. Krull, T.-O. Buchholz and F. Jug, "Noise2Void - Learning '
+                'Denoising From Single Noisy Images," 2019 IEEE/CVF '
+                "Conference on Computer Vision and Pattern Recognition "
+                "(CVPR), 2019, pp. 2124-2132",
+            }
+        ],
+        # "input_axes": ["bcyx"], <- overriden in save_as_bioimage
+        "preprocessing": [  # for multiple inputs
+            [  # multiple processes per input
+                {
+                    "kwargs": {
+                        "axes": "zyx" if is_3D else "yx",
+                        "mean": [mean],
+                        "mode": "fixed",
+                        "std": [std],
+                    },
+                    "name": "zero_mean_unit_variance",
+                }
+            ]
+        ],
+        # "output_axes": ["bcyx"], <- overriden in save_as_bioimage
+        "postprocessing": [  # for multiple outputs
+            [  # multiple processes per input
+                {
+                    "kwargs": {
+                        "axes": "zyx" if is_3D else "yx",
+                        "gain": [std],
+                        "offset": [mean],
+                    },
+                    "name": "scale_linear",
+                }
+            ]
+        ],
+        "tags": ["unet", "denoising", "Noise2Void", "tensorflow", "napari"],
+    }
+
+    rdf["documentation"] = _get_model_doc(name)
+
+    return rdf

{careamics-0.1.0rc1 → careamics-0.1.0rc2}/src/careamics/config/config.py

@@ -13,10 +13,11 @@ from pydantic import (
     model_validator,
 )
 
-from .algorithm import Algorithm
+# ignore typing-only-first-party-import in this file (flake8)
+from .algorithm import Algorithm  # noqa: TCH001
 from .config_filter import paths_to_str
-from .data import Data
-from .training import Training
+from .data import Data  # noqa: TCH001
+from .training import Training  # noqa: TCH001
 
 
 class Configuration(BaseModel):

{careamics-0.1.0rc1 → careamics-0.1.0rc2}/src/careamics/config/data.py

@@ -12,7 +12,7 @@ from pydantic import (
     model_validator,
 )
 
-from ..utils import check_axes_validity
+from careamics.utils import check_axes_validity
 
 
 class SupportedExtension(str, Enum):
@@ -23,7 +23,7 @@ class SupportedExtension(str, Enum):
         - tif/tiff: .tiff files.
     """
 
-    TIFF = "tiff"
+    TIFF = "tiff"  # TODO these should be a single one
     TIF = "tif"
 
     @classmethod

{careamics-0.1.0rc1 → careamics-0.1.0rc2}/src/careamics/dataset/dataset_utils.py

@@ -46,15 +46,11 @@ def _update_axes(array: np.ndarray, axes: str) -> np.ndarray:
         Updated array.
     """
     # concatenate ST axes to N, return NCZYX
-    if ("S" in axes or "T" in axes) and array.dtype != "O":
+    if "S" in axes or "T" in axes:
         new_axes_len = len(axes.replace("Z", "").replace("YX", ""))
         # TODO test reshape as it can scramble data, moveaxis is probably better
         array = array.reshape(-1, *array.shape[new_axes_len:]).astype(np.float32)
 
-    elif array.dtype == "O":
-        for i in range(len(array)):
-            array[i] = np.expand_dims(array[i], axis=0).astype(np.float32)
-
     else:
         array = np.expand_dims(array, axis=0).astype(np.float32)
 

{careamics-0.1.0rc1 → careamics-0.1.0rc2}/src/careamics/dataset/in_memory_dataset.py

@@ -5,8 +5,9 @@ from typing import Callable, Dict, List, Optional, Tuple, Union
 import numpy as np
 import torch
 
-from ..utils import normalize
-from ..utils.logging import get_logger
+from careamics.utils import normalize
+from careamics.utils.logging import get_logger
+
 from .dataset_utils import (
     list_files,
     read_tiff,

{careamics-0.1.0rc1 → careamics-0.1.0rc2}/src/careamics/dataset/patching.py

@@ -9,7 +9,8 @@ from typing import Generator, List, Optional, Tuple, Union
 import numpy as np
 from skimage.util import view_as_windows
 
-from ..utils.logging import get_logger
+from careamics.utils.logging import get_logger
+
 from .extraction_strategy import ExtractionStrategy
 
 logger = get_logger(__name__)
@@ -481,13 +482,11 @@ def generate_patches(
         elif patch_extraction_method == ExtractionStrategy.SEQUENTIAL:
             patches = _extract_patches_sequential(sample, patch_size=patch_size)
 
-        elif patch_extraction_method == ExtractionStrategy.RANDOM:
+        else:
+            # random patching
             patches = _extract_patches_random(sample, patch_size=patch_size)
 
-        if patches is None:
-            raise ValueError("No patch generated")
-
         return patches
     else:
-        # no patching
+        # no patching, return a generator for the sample
         return (sample for _ in range(1))

{careamics-0.1.0rc1 → careamics-0.1.0rc2}/src/careamics/dataset/prepare_dataset.py

@@ -6,9 +6,10 @@ Methods to set up the datasets for training, validation and prediction.
 from pathlib import Path
 from typing import List, Optional, Union
 
-from ..config import Configuration
-from ..manipulation import default_manipulate
-from ..utils import check_tiling_validity
+from careamics.config import Configuration
+from careamics.manipulation import default_manipulate
+from careamics.utils import check_tiling_validity
+
 from .extraction_strategy import ExtractionStrategy
 from .in_memory_dataset import InMemoryDataset
 from .tiff_dataset import TiffDataset

{careamics-0.1.0rc1 → careamics-0.1.0rc2}/src/careamics/dataset/tiff_dataset.py

@@ -10,8 +10,9 @@ from typing import Callable, Dict, Generator, List, Optional, Tuple, Union
 import numpy as np
 import torch
 
-from ..utils import normalize
-from ..utils.logging import get_logger
+from careamics.utils import normalize
+from careamics.utils.logging import get_logger
+
 from .dataset_utils import (
     list_files,
     read_tiff,
@@ -53,7 +54,7 @@ class TiffDataset(torch.utils.data.IterableDataset):
     def __init__(
         self,
         data_path: Union[str, Path],
-        data_format: str,
+        data_format: str,  # TODO: TiffDataset should not know that they are tiff
         axes: str,
         patch_extraction_method: Union[ExtractionStrategy, None],
         patch_size: Optional[Union[List[int], Tuple[int]]] = None,