lls-core 1.0.0__py3-none-any.whl

lls_core/__init__.py

@@ -0,0 +1,55 @@
+
+__version__ = "0.2.6"
+
+from strenum import StrEnum
+from enum import Enum
+from pyclesperanto_prototype._tier8._affine_transform_deskew_3d import DeskewDirection
+from lls_core.models.lattice_data import LatticeData
+from lls_core.models.deconvolution import DeconvolutionParams
+from lls_core.models.lattice_data import CropParams
+
+# Initialize configuration options
+
+#CONFIGURE LOGGING using a dictionary (can also be done with yaml file)
+import logging.config
+LOGGING_CONFIG = {
+    'version': 1,
+    'disable_existing_loggers': False,
+
+    'formatters': {
+        'default_formatter': {
+            'format': '[%(levelname)s:%(asctime)s] %(message)s'
+        },
+    },
+
+    'handlers': {
+        'stream_handler': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'default_formatter',
+        },
+    },
+
+    'loggers': {
+        '': {
+            'handlers': ['stream_handler'],
+            'level': 'INFO',
+            'propagate': True
+        }
+    }
+}
+
+#Configuring logging level with empty string "" under the key loggers means its for root
+#This will override levels for all other python libraries as they haven't been imported yet
+#Specifying levels in each modules will override the root level
+
+# Specify during initialization:
+logging.config.dictConfig(LOGGING_CONFIG)
+logger = logging.getLogger(__name__)
+logger.debug("Logging is configured.")
+
+#specify an enum for log levels
+class Log_Levels(Enum):
+    DEBUG = 10
+    INFO = 20
+    WARNING = 30
+    

lls_core/cmds/__init__.py

@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+

lls_core/cmds/__main__.py

@@ -0,0 +1,234 @@
+# lattice_processing.py
+
+# Run processing on command line instead of napari.
+# Example for deskewing files in a folder
+# python lattice_processing.py --input /home/pradeep/to_deskew --output /home/pradeep/output_save/ --processing deskew
+from __future__ import annotations
+
+from enum import auto
+from pathlib import Path
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
+from strenum import StrEnum
+
+from lls_core.models.lattice_data import LatticeData
+from lls_core.models.deskew import DeskewParams, DefinedPixelSizes
+from lls_core.models.deconvolution import DeconvolutionParams
+from lls_core.models.output import OutputParams
+from lls_core.models.crop import CropParams
+from lls_core.deconvolution import DeconvolutionChoice
+from typer import Typer, Argument, Option, Context, Exit
+from typer.main import get_command
+
+from lls_core.models.output import SaveFileType
+from pydantic.v1 import ValidationError
+from toolz.dicttoolz import merge_with
+
+if TYPE_CHECKING:
+    from lls_core.models.utils import FieldAccessModel
+    from typing import Type, Any, Iterable
+    from rich.table import Table
+
+class CliDeskewDirection(StrEnum):
+    X = auto()
+    Y = auto()
+
+CLI_PARAM_MAP = {
+    "input_image": ["input_image"],
+    "angle": ["angle"],
+    "skew": ["skew"],
+    "physical_pixel_sizes": ["physical_pixel_sizes"],
+    "roi_list": ["crop", "roi_list"],
+    "roi_subset": ["crop", "roi_subset"],
+    "z_range": ["crop", "z_range"],
+    "decon_processing": ["deconvolution", "decon_processing"],
+    "psf": ["deconvolution", "psf"],
+    "decon_num_iter": ["deconvolution", "decon_num_iter"],
+    "background": ["deconvolution", "background"],
+    "workflow": ["workflow"],
+    "time_range": ["time_range"],
+    "channel_range": ["channel_range"],
+    "save_dir": ["save_dir"],
+    "save_name": ["save_name"],
+    "save_type": ["save_type"],
+}
+
+app = Typer(add_completion=False, rich_markup_mode="rich", no_args_is_help=True)
+
+def field_from_model(model: Type[FieldAccessModel], field_name: str, extra_description: str = "", description: Optional[str] = None, default: Optional[Any] = None, **kwargs) -> Any:
+    """
+    Generates a type Field from a Pydantic model field
+    """
+    field = model.__fields__[field_name]
+
+    from enum import Enum
+    if default is None:
+        default = field.get_default()
+    if isinstance(default, Enum):
+        default = default.name
+
+    if description is None:
+        description = f"{field.field_info.description} {extra_description}"
+
+    return Option(
+        default = default,
+        help=description,
+        **kwargs
+    )
+
+def handle_merge(values: list):
+    if len(values) > 1:
+        if all(isinstance(param, dict) for param in values):
+            return merge_with(handle_merge, values)
+        else:
+            raise ValueError(f"A parameter has been passed multiple times! Got: {', '.join(values)}")
+    return values[0]
+
+def rich_validation(e: ValidationError) -> Table:
+    """
+    Converts 
+    """
+    from rich.table import Table
+
+    table = Table(title="Validation Errors")
+    table.add_column("Parameter")
+    # table.add_column("Command Line Argument")
+    table.add_column("Error")
+
+    for error in e.errors():
+        table.add_row(
+            str(error["loc"][0]),
+            str(error["msg"]),
+        )
+
+    return table
+
+def pairwise(iterable: Iterable) -> Iterable:
+    """
+    An implementation of the pairwise() function in Python 3.10+
+    See: https://docs.python.org/3.12/library/itertools.html#itertools.pairwise
+    """
+    from itertools import tee
+    a, b = tee(iterable)
+    next(b, None)
+    return zip(a, b)
+
+def update_nested_data(data: Union[dict, list], keys: list, new_value: Any):
+    current = data
+
+    for key, next_key in pairwise(keys):
+        next = {} if isinstance(next_key, str) else []
+        if isinstance(current, dict):
+            current = current.setdefault(key, next)
+        elif isinstance(current, list):
+            if key >= len(current):
+                current.insert(key, next)
+            current = current[key]
+        else:
+            raise ValueError(f"Unknown data type {type(current)}. Cannot traverse.")
+
+    last_key = keys[-1]
+    if isinstance(current, dict):
+        current[last_key] = new_value
+    elif isinstance(current, list):
+        current.insert(last_key, new_value)
+    else:
+        raise ValueError(f"Unknown data type {type(current)}. Cannot traverse.")
+
+# Example usage:
+@app.command()
+def process(
+    ctx: Context,
+    input_image: Path = Argument(None, help="Path to the image file to read, in a format readable by AICSImageIO, for example .tiff or .czi", show_default=False),
+    skew: CliDeskewDirection = field_from_model(DeskewParams, "skew"),# DeskewParams.make_typer_field("skew"),
+    angle: float = field_from_model(DeskewParams, "angle") ,
+    physical_pixel_sizes: Tuple[float, float, float] = field_from_model(DeskewParams, "physical_pixel_sizes", extra_description="This takes three arguments, corresponding to the Z, Y and X pixel dimensions respectively", default=(
+        DefinedPixelSizes.get_default("Z"),
+        DefinedPixelSizes.get_default("Y"),
+        DefinedPixelSizes.get_default("X")
+    )),
+
+    roi_list: List[Path] = field_from_model(CropParams, "roi_list"), 
+    roi_subset: List[int] = field_from_model(CropParams, "roi_subset"),
+    z_range: Optional[Tuple[int,int]] = field_from_model(CropParams, "z_range", show_default=False),
+    
+    enable_deconvolution: bool = Option(False, "--deconvolution/--disable-deconvolution", rich_help_panel="Deconvolution"),
+    decon_processing: DeconvolutionChoice = field_from_model(DeconvolutionParams, "decon_processing", rich_help_panel="Deconvolution"),
+    psf: List[Path] = field_from_model(DeconvolutionParams, "psf", rich_help_panel="Deconvolution"),
+    decon_num_iter: int = field_from_model(DeconvolutionParams, "decon_num_iter", rich_help_panel="Deconvolution"),
+    background: str = field_from_model(DeconvolutionParams, "background", rich_help_panel="Deconvolution"),
+
+    time_range: Optional[Tuple[int,int]] = field_from_model(OutputParams, "time_range", rich_help_panel="Output"),
+    channel_range: Optional[Tuple[int,int]] = field_from_model(OutputParams,"channel_range", rich_help_panel="Output"),
+        
+    save_dir: Path = field_from_model(OutputParams, "save_dir", rich_help_panel="Output"),
+    save_name: Optional[str] = field_from_model(OutputParams, "save_name", rich_help_panel="Output"),
+    save_type: SaveFileType = field_from_model(OutputParams, "save_type", rich_help_panel="Output"),
+
+    workflow: Optional[Path] = field_from_model(LatticeData, "workflow", show_default=False),
+    json_config: Optional[Path] = Option(None, show_default=False, help="Path to a JSON file from which parameters will be read."),
+    yaml_config: Optional[Path] = Option(None, show_default=False, help="Path to a YAML file from which parameters will be read."),
+
+    show_schema: bool = Option(default=False, help="If provided, image processing will not be performed, and instead a JSON document outlining the JSON/YAML options will be printed to stdout. This can be used to assist with writing a config file for use with the --json-config and --yaml-config options.")
+) -> None:
+    from click.core import ParameterSource
+    from rich.console import Console
+
+    console = Console(stderr=True)
+
+    if show_schema:
+        import json
+        import sys
+        json.dump(
+            LatticeData.to_definition_dict(),
+            sys.stdout,
+            indent=4
+        )
+        return
+
+    # Just print help if the user didn't provide any arguments
+    if all(src != ParameterSource.COMMANDLINE for src in ctx._parameter_source.values()):
+        print(ctx.get_help())
+        raise Exit()
+
+    from toolz.dicttoolz import merge_with
+    cli_args = {}
+    for source, dest in CLI_PARAM_MAP.items():
+        from click.core import ParameterSource
+        if ctx.get_parameter_source(source) != ParameterSource.DEFAULT:
+            update_nested_data(cli_args, dest, ctx.params[source])
+
+    json_args = {}
+    if json_config is not None:
+        import json
+        with json_config.open() as fp:
+            json_args = json.load(fp)
+
+    yaml_args = {}
+    if yaml_config is not None:
+        with yaml_config.open() as fp:
+            from yaml import safe_load
+            yaml_args = safe_load(fp)
+
+    try:
+        lattice = LatticeData.parse_obj(
+            # Merge all three sources of config: YAML, JSON and CLI
+            merge_with(
+                handle_merge,
+                [yaml_args, json_args, cli_args]
+            )
+        )
+    except ValidationError as e:
+        console.print(rich_validation(e))
+        raise Exit(code=1)
+    
+    lattice.save()
+    console.print(f"Processing successful. Results can be found in {lattice.save_dir.resolve()}")
+
+# Used by the docs
+click_app = get_command(app)
+
+def main():
+    app()
+
+if __name__ == '__main__':
+    main()

lls_core/config.py

@@ -0,0 +1,6 @@
+#module for accessing variables across modules in workflow
+#https://docs.python.org/3/faq/programming.html?highlight=global#how-do-i-share-global-variables-across-modules
+channel = 0
+time = 0 
+#configure default logging level for use in packages; inherit from dock_widget or __main__.py
+log_level = 10

lls_core/cropping.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING, NamedTuple, Tuple, List
+
+if TYPE_CHECKING:
+    from lls_core.types import PathLike
+    from typing_extensions import Self
+    from numpy.typing import NDArray
+
+RoiCoord = Tuple[float, float]
+
+class Roi(NamedTuple):
+    top_left: RoiCoord
+    top_right: RoiCoord
+    bottom_left: RoiCoord
+    bottom_right: RoiCoord
+
+    @classmethod
+    def from_array(cls, array: NDArray) -> Self:
+        import numpy as np
+        return Roi(*np.reshape(array, (-1, 2)).tolist())
+
+def read_roi_array(roi: PathLike) -> NDArray:
+    from read_roi import read_roi_file
+    from numpy import array
+    return array(read_roi_file(str(roi)))
+
+def read_imagej_roi(roi_path: PathLike) -> List[Roi]:
+    """Read an ImageJ ROI zip file so it loaded into napari shapes layer
+        If non rectangular ROI, will convert into a rectangle based on extreme points
+    Args:
+        roi_zip_path (zip file): ImageJ ROI zip file
+
+    Returns:
+        list: List of ROIs
+    """
+    from pathlib import Path
+    from os import fspath
+    from read_roi import read_roi_file, read_roi_zip
+
+    roi_path = Path(fspath(roi_path))
+
+    # handle reading single roi or collection of rois in zip file
+    if roi_path.suffix == ".zip":
+        ij_roi = read_roi_zip(roi_path)
+    elif roi_path.suffix == ".roi":
+        ij_roi = read_roi_file(str(roi_path))
+    else:
+        raise Exception("ImageJ ROI file needs to be a zip/roi file")
+
+    if ij_roi is None:
+        raise Exception("Failed reading ROI file")
+
+    # initialise list of rois
+    roi_list = []
+
+    # Read through each roi and create a list so that it matches the organisation of the shapes from napari shapes layer
+    for value in ij_roi.values():
+        if value['type'] in ('oval', 'rectangle'):
+            width = int(value['width'])
+            height = int(value['height'])
+            left = int(value['left'])
+            top = int(value['top'])
+            roi = Roi((top, left), (top, left+width), (top+height, left+width), (top+height, left))
+            roi_list.append(roi)
+        elif value['type'] in ('polygon', 'freehand'):
+            left = min(int(it) for it in value['x'])
+            top = min(int(it) for it in value['y'])
+            right = max(int(it) for it in value['x'])
+            bottom = max(int(it) for it in value['y'])
+            roi = Roi((top, left), (top, right), (bottom, right), (bottom, left))
+            roi_list.append(roi)
+        else:
+            print(f"Cannot read ROI {value}. Recognised as type {value['type']}")
+
+    return roi_list

lls_core/deconvolution.py

@@ -0,0 +1,266 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from strenum import StrEnum
+
+import logging
+import importlib.util
+from typing import Collection, Iterable,Union,Literal, Optional, TYPE_CHECKING
+from aicsimageio.aics_image import AICSImage
+from skimage.io import imread
+from aicspylibczi import CziFile
+from numpy.typing import NDArray
+import os
+import numpy as np
+from dask.array.core import Array as DaskArray
+
+from lls_core.utils import array_to_dask, pad_image_nearest_multiple
+from lls_core.types import ArrayLike, is_arraylike
+
+if TYPE_CHECKING:
+    from lls_core.models.lattice_data import LatticeData
+
+class DeconvolutionChoice(StrEnum):
+    """
+    Deconvolution algorithm
+    """
+    cuda_gpu = "cuda_gpu"
+    opencl_gpu = "opencl_gpu"
+    cpu = "cpu"
+
+logger = logging.getLogger(__name__)
+
+def read_psf(psf_paths: Collection[Path],
+              decon_option: DeconvolutionChoice,
+              lattice_class: LatticeData) -> Iterable[NDArray]:
+    """Read PSF files and return a list of PSF arrays appended to lattice_class.psf
+
+    Args:
+        decon_option (enum): Enum option from DeconvolutionChoice
+        lattice_class: lattice class object, either LLSZWidget.LlszMenu.lattice or lattice class from batch processing
+    """
+
+    # remove empty paths; pathlib returns current directory as "." if None or empty str specified
+    # When running batch processing,empty directory will be an empty string
+
+    logging.debug(f"PSF paths are {psf_paths}")
+    # total no of psf images
+    psf_channels = len(psf_paths)
+    assert psf_channels > 0, f"No images detected for PSF. Check the psf paths -> {psf_paths}"
+
+    # Use CUDA for deconvolution
+    if decon_option == DeconvolutionChoice.cuda_gpu:
+        pycudadecon_import = importlib.util.find_spec("pycudadecon")
+        assert pycudadecon_import, f"Pycudadecon not detected. Please install using: conda install -c conda-forge pycudadecon"
+        otf_names = ["ch1", "ch2", "ch3", "ch4"]
+        channels = [488, 561, 640, 123]
+        # get temp directory to save generated otf
+        import tempfile
+        temp_dir = tempfile.gettempdir()+os.sep
+
+    for psf in psf_paths:
+        if psf.exists() and psf.is_file():
+            if psf.suffix == ".czi":
+                psf_czi = CziFile(psf.__str__())
+                psf_aics = psf_czi.read_image()
+                # make sure shape is 3D
+                psf_aics = psf_aics[0][0]  # np.expand_dims(psf_aics[0],axis=0)
+                # if len(psf_aics[0])>=1:
+                #psf_channels = len(psf_aics[0])
+                assert len(
+                    psf_aics.shape) == 3, f"PSF should be a 3D image (shape of 3), but got {psf_aics.shape}"
+                # pad psf to multiple of 16 for decon
+                yield pad_image_nearest_multiple(img=psf_aics, nearest_multiple=16)
+            else:
+                #Use skimage to read tiff
+                if psf.suffix in [".tif", ".tiff"]:
+                   psf_aics_data = imread(str(psf))
+                   if len(psf_aics_data.shape) != 3:
+                       raise ValueError(f"PSF should be a 3D image (shape of 3), but got {psf_aics.shape}")
+                else:
+                    #Use AICSImageIO
+                    psf_aics = AICSImage(str(psf))
+                    psf_aics_data = psf_aics.data[0][0]
+                    psf_aics_data = pad_image_nearest_multiple(
+                        img=psf_aics_data, nearest_multiple=16)           
+                    if psf_aics.dims.C != lattice_class.channels:
+                        logger.warn(
+                            f"PSF image has {psf_channels} channel/s, whereas image has {lattice_class.channels}")
+
+                yield psf_aics_data
+
+# Ideally we want to use OpenCL, but in the case of deconvolution most CUDA based
+# libraries are better designed.. Atleast until  RL deconvolution is available in pyclesperant
+# Talley Lamberts pycudadecon is a great library and highly optimised.
+def pycuda_decon(
+    image: ArrayLike,
+    otf_path: Optional[str]=None,
+    dzdata: float=0.3,
+    dxdata: float=0.1449922,
+    dzpsf: float=0.3,
+    dxpsf: float=0.1449922,
+    psf: Optional[ArrayLike]=None,
+    num_iter: int = 10,
+    cropping: bool = False,
+    background: Union[float,Literal["auto","second_last"]] = 0 
+):
+    """Perform deconvolution using pycudadecon
+    pycudadecon can return cropped images, so we pad the images with dimensions that are a multiple of 64
+
+    Args:
+        image : _description_
+        otf_path : (path to the generated otf file, if available. Otherwise psf needs to be provided)
+        dzdata : (pixel size in z in microns)
+        dxdata : (pixel size in xy  in microns)
+        dzpsf : (pixel size of original psf file in z  microns)
+        dxpsf : (pixel size of original psf file in xy microns)
+        psf (tiff): option to provide psf instead of the otfpath, this can be used when calling decon function
+        num_iter (int): number of iterations
+        cropping (bool): option to specify if cropping option is enabled
+        background (float,"auto","second_last"): background value to use for deconvolution. Can specify a value. If using auto, defaults to pycudadecon settings which is median of last slice
+        'second_last' means median of second last slice. This option is because last slice can sometimes be incomplete
+    Returns:
+        np.array: _description_
+    """
+    image = np.squeeze(image)
+    assert image.ndim == 3, f"Image needs to be 3D. Got {image.ndim}"
+
+    #Option for median of second last slices to be used
+    #Similar to pycudadecon
+    if isinstance(background, str) and background == "second_last":
+        background = np.median(image[-2])
+    
+    # if dask array, convert to numpy array
+    if isinstance(image, DaskArray):
+        image = np.array(image)
+
+    orig_img_shape = image.shape
+
+    # if cropping, add extra padding to avoid edge artefact
+    # not doing this with with large images as images can be quite large with all the padding leading to memory issues
+    if cropping:
+        # pad image y dimensionswith half of psf shape
+        z_psf_pad, y_psf_pad, x_psf_pad = np.array(psf.shape) // 2
+        image = np.pad(
+            image,
+            (
+                (z_psf_pad, z_psf_pad),
+                (y_psf_pad, y_psf_pad),
+                (x_psf_pad, x_psf_pad),
+            ),
+            mode="reflect",
+        )
+
+    # pad image to a multiple of 64
+    image = pad_image_nearest_multiple(img=image, nearest_multiple=64)
+
+    if is_arraylike(psf):
+        from pycudadecon import RLContext, TemporaryOTF, rl_decon
+
+        psf = np.squeeze(psf)  # remove unit dimensions
+        assert psf.ndim == 3, f"PSF needs to be 3D. Got {psf.ndim}"
+        # Temporary OTF generation; RLContext ensures memory cleanup (runs rl_init and rl_cleanup)
+        with TemporaryOTF(psf) as otf:
+            with RLContext(
+                rawdata_shape=image.shape,
+                otfpath=otf.path,
+                dzdata=dzdata,
+                dxdata=dxdata,
+                dzpsf=dzpsf,
+                dxpsf=dxpsf,
+            ) as ctx:
+                decon_res = rl_decon(
+                    im=image, output_shape=ctx.out_shape, n_iters=num_iter,background=background
+                )
+
+    else:
+        from pycudadecon import rl_decon, rl_init, rl_cleanup
+
+        rl_init(
+            rawdata_shape=image.shape,
+            otfpath=otf_path,
+            dzdata=dzdata,
+            dxdata=dxdata,
+            dzpsf=dzpsf,
+            dxpsf=dxpsf,
+        )
+        decon_res = rl_decon(image)
+        rl_cleanup()
+
+    if cropping:
+        # remove psf padding if cropping was chosen
+        decon_res = decon_res[
+            z_psf_pad:-z_psf_pad, y_psf_pad:-y_psf_pad, x_psf_pad:-x_psf_pad
+        ]
+
+    # remove padding; get shape difference and use this shape difference to remove padding
+    shape_diff = np.array(decon_res.shape) - np.array(orig_img_shape)
+    # if above is negative,
+    if shape_diff[0] == 0:
+        shape_diff[0] = -orig_img_shape[0]
+    if shape_diff[1] == 0:
+        shape_diff[1] = -orig_img_shape[1]
+    if shape_diff[2] == 0:
+        shape_diff[2] = -orig_img_shape[2]
+
+    decon_res = decon_res[: -shape_diff[0], : -shape_diff[1], : -shape_diff[2]]
+    # print(orig_img_shape)
+    # make sure the decon image and the original image shapes are the same
+    assert (
+        decon_res.shape == orig_img_shape
+    ), f"Deconvolved {decon_res.shape} and original image shape {orig_img_shape} do not match."
+    return decon_res
+
+
+def skimage_decon(
+    vol_zyx, psf, num_iter: int, clip: bool, filter_epsilon, boundary: str
+):
+    """Deconvolution using scikit image
+
+    Args:
+        vol_zyx (_type_): _description_
+        psf (_type_): _description_
+        num_iter (_type_): _description_
+        clip (_type_): _description_
+        filter_epsilon (_type_): _description_
+        boundary (_type_): _description_
+
+    Returns:
+        _type_: _description_
+    """
+    from skimage.restoration import richardson_lucy as rl_decon_skimage
+
+    depth = tuple(np.array(psf.shape) // 2)
+    vol_zyx = array_to_dask(vol_zyx)
+    decon_data = vol_zyx.map_overlap(
+        rl_decon_skimage,
+        psf=psf,
+        num_iter=num_iter,
+        clip=clip,
+        filter_epsilon=filter_epsilon,
+        boundary=boundary,
+        depth=depth,
+        trim=True,
+    )
+    return decon_data
+
+
+def pyopencl_decon(vol_zyx, psf, num_iter: int, clip: bool, filter_epsilon):
+    """Deconvolution using RedFishLion library for pyopencl based deconvolution
+    This is slower than pycudadecon
+
+    Args:
+        vol_zyx (_type_): _description_
+        psf (_type_): _description_
+        num_iter (_type_): _description_
+        clip (_type_): _description_
+        filter_epsilon (_type_): _description_
+        boundary (_type_): _description_
+
+    Returns:
+        _type_: _description_
+    """
+    # CROP PSF; Crop psfs when reading and initialising psfs
+
+    return