eubi-bridge 0.0.1__tar.gz

eubi_bridge-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Euro-BioImaging
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

eubi_bridge-0.0.1/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.2
+Name: eubi_bridge
+Version: 0.0.1
+Summary: A package for converting datasets to OME-Zarr format.
+Home-page: https://github.com/Euro-BioImaging/EuBI-Bridge
+Author: Bugra Özdemir
+Author-email: bugraa.ozdemir@gmail.com
+License: MIT
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: author
+Dynamic: author-email
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: summary
+
+A package to convert collections of image data from diverse formats
+to the OME-Zarr format in a distributed manner.

eubi_bridge-0.0.1/README.txt

@@ -0,0 +1,2 @@
+A package to convert collections of image data from diverse formats
+to the OME-Zarr format in a distributed manner.

eubi_bridge-0.0.1/eubi_bridge/__init__.py

@@ -0,0 +1,16 @@
+from eubi_bridge import ebridge
+from eubi_bridge import fileset_io
+from eubi_bridge.base import scale, writers
+from eubi_bridge.ngff import defaults, multiscales
+from eubi_bridge.utils import convenience, dask_client_plugins
+
+__all__ = [
+    'ebridge_base',
+    'fileset_io',
+    'scale',
+    'writers',
+    'defaults',
+    'multiscales',
+    'convenience',
+    'dask_client_plugins',
+]

eubi_bridge-0.0.1/eubi_bridge/base/scale.py

@@ -0,0 +1,137 @@
+import zarr, dataclasses
+from pathlib import Path
+import numpy as np, zarr
+import dask.array as da
+from typing import Callable
+
+
+def simple_downscale(
+                     darr,
+                     scale_factor: (tuple, list, np.ndarray) = None,
+                     backend = 'numpy' # placeholder
+                     ):
+    """
+    Downscale a Dask array along each dimension by given scale factors.
+
+    Parameters:
+    arr (dask.array): The input n-dimensional Dask array.
+    scale_factors (tuple): The downsampling factors for each dimension.
+
+    Returns:
+    dask.array: The downscaled Dask array.
+    """
+    if len(scale_factor) != darr.ndim:
+        raise ValueError("scale_factors must have the same length as the array's number of dimensions")
+    slices = tuple(slice(None, None, int(scale)) for scale in scale_factor)
+    downscaled_arr = darr[slices]
+    return downscaled_arr
+
+def mean_downscale(arr: da.Array,
+                   scale_factor: (tuple, list, np.ndarray) = None
+                   ):
+    if len(scale_factor) != arr.ndim:
+        raise ValueError("scale_factors must have the same length as the array's number of dimensions")
+    axes = dict({idx: factor for idx, factor in enumerate(scale_factor)})
+    downscaled_arr = da.coarsen(da.mean, arr,
+                                axes = axes, trim_excess = True).astype(arr.dtype)
+    return downscaled_arr
+
+def median_downscale(arr: da.Array,
+                   scale_factor: (tuple, list, np.ndarray) = None
+                   ):
+    if len(scale_factor) != arr.ndim:
+        raise ValueError("scale_factors must have the same length as the array's number of dimensions")
+    axes = dict({idx: factor for idx, factor in enumerate(scale_factor)})
+    downscaled_arr = da.coarsen(da.median, arr,
+                                axes = axes, trim_excess = True).astype(arr.dtype)
+    return downscaled_arr
+
+@dataclasses.dataclass
+class DownscaleManager:
+    base_shape: (list, tuple)
+    scale_factor: (list, tuple)
+    n_layers: (list, tuple)
+    scale: (list, tuple) = None
+
+    def __post_init__(self):
+        ndim = len(self.base_shape)
+        assert len(self.scale_factor) == ndim
+
+    @property
+    def _scale_ids(self):
+        return np.arange(self.n_layers).reshape(-1, 1)
+
+    @property
+    def _theoretical_scale_factors(self):
+        return np.power(self.scale_factor, self._scale_ids)
+
+    @property
+    def output_shapes(self):
+        shapes = np.floor_divide(self.base_shape, self._theoretical_scale_factors)
+        shapes[shapes == 0] = 1
+        return shapes
+
+    @property
+    def scale_factors(self):
+        return np.true_divide(self.output_shapes[0], self.output_shapes)
+
+    @property
+    def scales(self):
+        return np.multiply(self.scale, self.scale_factors)
+
+
+@dataclasses.dataclass
+class Downscaler:
+    array: da.Array
+    scale_factor: (list, tuple)
+    n_layers: int
+    scale: (list, tuple) = None
+    output_chunks: (list, tuple) = None
+    backend: str = 'numpy'
+    downscale_method: str = 'simple'
+
+    def __post_init__(self):
+        self.param_names = ['array', 'scale_factor', 'n_layers', 'scale', 'output_chunks', 'backend', 'downscale_method']
+        self.update()
+
+    def get_method(self):
+        if self.downscale_method == 'simple':
+            method = simple_downscale
+        elif self.downscale_method == "mean":
+            method = mean_downscale
+        elif self.downscale_method == "median":
+            method = mean_downscale
+        else:
+            raise NotImplementedError(f"Currently, only 'simple', 'mean' and 'median' methods are implemented.")
+        return method
+
+    def run(self):
+        self.method = self.get_method()
+        assert isinstance(self.array, da.Array)
+        self.dm = DownscaleManager(self.array.shape,
+                                   self.scale_factor,
+                                   self.n_layers,
+                                   self.scale
+                                   )
+        if self.output_chunks is None:
+            self.output_chunks = [self.array.chunksize] * self.n_layers
+
+        downscaled = []
+        for idx, (scale_factor, chunks) in enumerate(zip(self.dm.scale_factors, self.output_chunks)):
+            if idx == 0:
+                downscaled.append(self.array)
+            else:
+                factor = tuple(int(x) for x in scale_factor)
+                res1 = self.method(self.array, scale_factor = factor)
+                downscaled.append(res1)
+        self.downscaled_arrays = {str(i): arr for i, arr in enumerate(downscaled)}
+        return self
+
+    def update(self, **kwargs):
+        for key, value in kwargs.items():
+            if key in self.param_names:
+                self.__setattr__(key, value)
+            else:
+                warnings.warn(f"The given parameter name '{key}' is not valid, ignoring it..")
+        self.run()
+        return self

eubi_bridge-0.0.1/eubi_bridge/base/writers.py

@@ -0,0 +1,369 @@
+import os, itertools, tempfile, shutil, threading
+import zarr, dask, numcodecs
+from dask import delayed
+import rechunker
+from rechunker import rechunk, Rechunked
+import dask.array as da
+# from dask.diagnostics import ProgressBar
+import numpy as np
+# import tensorstore as ts
+from pathlib import Path
+from typing import List, Tuple, Dict, Union, Any, Tuple
+### internal imports
+from eubi_bridge.ngff.multiscales import Multimeta
+from eubi_bridge.utils.convenience import get_chunksize_from_array, is_zarr_group
+
+import logging, warnings
+
+logging.getLogger('distributed.diskutils').setLevel(logging.CRITICAL)
+
+def create_zarr_array(directory: Union[Path, str, zarr.Group],
+                      array_name: str,
+                      shape: Tuple[int, ...],
+                      chunks: Tuple[int, ...],
+                      dtype: Any,
+                      overwrite: bool = False) -> zarr.Array:
+    chunks = tuple(np.minimum(shape, chunks))
+
+    if not isinstance(directory, zarr.Group):
+        path = os.path.join(directory, array_name)
+        dataset = zarr.create(shape=shape,
+                              chunks=chunks,
+                              dtype=dtype,
+                              store=path,
+                              dimension_separator='/',
+                              overwrite=overwrite)
+    else:
+        dataset = directory.create(name=array_name,
+                                   shape=shape,
+                                   chunks=chunks,
+                                   dtype=dtype,
+                                   dimension_separator='/',
+                                   overwrite=overwrite)
+    return dataset
+
+
+def get_regions(array_shape: Tuple[int, ...],
+                region_shape: Tuple[int, ...],
+                as_slices: bool = False) -> list:
+    assert len(array_shape) == len(region_shape)
+    steps = []
+    for size, inc in zip(array_shape, region_shape):
+        seq = np.arange(0, size, inc)
+        if size > seq[-1]:
+            seq = np.append(seq, size)
+        increments = tuple((seq[i], seq[i + 1]) for i in range(len(seq) - 1))
+        if as_slices:
+            steps.append(tuple(slice(*item) for item in increments))
+        else:
+            steps.append(increments)
+    return list(itertools.product(*steps))
+
+def get_compressor(name, **params):
+    name = name.lower()
+    compression_dict = {
+        "blosc": "Blosc",
+        "bz2": "BZ2",
+        "gzip": "GZip",
+        "lzma": "LZMA",
+        "lz4": "LZ4",
+        "pcodec": "PCodec",
+        "zfpy": "ZFPY",
+        "zlib": "Zlib",
+        "zstd": "Zstd"
+    }
+
+    compressor_name = compression_dict[name]
+    compressor_class = getattr(numcodecs, compressor_name)
+    compressor = compressor_class(**params)
+    return compressor
+
+def get_default_fill_value(dtype):
+    if np.issubdtype(dtype, np.integer):
+        return 0
+    elif np.issubdtype(dtype, np.floating):
+        return 0.0
+    elif np.issubdtype(dtype, np.bool_):
+        return False
+    return None
+
+def write_chunk_with_zarrpy(chunk: np.ndarray, zarr_array: zarr.Array, block_info: Dict) -> None:
+    zarr_array[tuple(slice(*b) for b in block_info[0]["array-location"])] = chunk
+
+def write_chunk_with_tensorstore(chunk: np.ndarray, ts_store, block_info: Dict) -> None:
+    ts_store[tuple(slice(*b) for b in block_info[0]["array-location"])] = chunk
+
+def write_with_rechunker(arr: da.Array,
+                         chunks: Tuple[int, ...],
+                         location: Union[str, Path],
+                         overwrite: bool = True,
+                         **kwargs) -> Rechunked:
+    temp_dir = kwargs.get('temp_dir')
+    if not temp_dir:
+        raise ValueError("A temp_dir must be specified.")
+
+    temp_dir_is_auto = temp_dir == 'auto'
+    if temp_dir_is_auto:
+        temp_dir = tempfile.TemporaryDirectory()
+
+    max_mem = kwargs.get('rechunkers_max_mem', "auto")
+    if max_mem == "auto":
+        max_mem = get_chunksize_from_array(arr)
+
+    if overwrite:
+        shutil.rmtree(location, ignore_errors=True)
+
+    target_store = zarr.DirectoryStore(location, dimension_separator='/')
+    temp_store = zarr.DirectoryStore(temp_dir.name if isinstance(temp_dir, tempfile.TemporaryDirectory) else temp_dir,
+                                     dimension_separator='/')
+
+    compressor_name = kwargs.get('compressor', 'blosc')
+    compressor_params = kwargs.get('compressor_params', {})
+    compressor = get_compressor(compressor_name, **compressor_params)
+
+    dtype = kwargs.get('dtype', arr.dtype)
+    if dtype == 'auto':
+        dtype = arr.dtype
+
+    fill_value = kwargs.get('fill_value', get_default_fill_value(dtype))
+
+    # Use rechunker (without fill_value)
+    rechunked = rechunk(source=arr,
+                        target_chunks=chunks,
+                        target_store=target_store,
+                        temp_store=temp_store,
+                        max_mem=max_mem,
+                        executor='dask',
+                        target_options={'overwrite': True,
+                                        'compressor': compressor,
+                                        'write_empty_chunks': True})  # No fill_value here
+
+    # **Reopen Zarr array and update fill_value properly**
+    zarr_array = zarr.open_array(target_store, mode="a")  # Open in append mode
+    zarr_array.fill_value = fill_value  # Set fill_value correctly
+
+    # Cleanup temporary directory if auto-generated
+    if temp_dir_is_auto:
+        temp_dir.cleanup()
+
+    return rechunked
+
+def write_with_zarrpy(arr: da.Array,
+                      chunks: Tuple[int, ...],
+                      location: Union[str, Path],
+                      overwrite: bool = True,
+                      **kwargs) -> da.Array:
+    rechunk_method = kwargs.get('rechunk_method', 'tasks')
+
+    if not np.equal(arr.chunksize, chunks).all():
+        arr = arr.rechunk(chunks, method=rechunk_method #, threshold = 1_000_000
+        )
+
+    store = zarr.DirectoryStore(location, dimension_separator='/')
+    try:
+        zarr_array = zarr.open_array(location, mode='w')
+    except:
+        compressor_name = kwargs.get('compressor', 'blosc')
+        compressor_params = kwargs.get('compressor_params', {})
+        compressor = get_compressor(compressor_name, **compressor_params)
+        dtype = kwargs.get('dtype', arr.dtype)
+        if dtype == 'auto':
+            dtype = arr.dtype
+
+        fill_value = kwargs.get('fill_value', get_default_fill_value(dtype))
+
+        zarr_array = zarr.create(shape=arr.shape, chunks=chunks, dtype=dtype, compressor = compressor, store=store, overwrite=overwrite, fill_value = fill_value)
+
+    return arr.map_blocks(write_chunk_with_zarrpy, zarr_array=zarr_array, dtype=dtype)
+
+
+def write_with_tensorstore(arr: da.Array,
+                           chunks: Tuple[int, ...],
+                           location: Union[str, Path],
+                           overwrite: bool = True,
+                           **kwargs) -> da.Array:
+
+    try:
+        import tensorstore as ts
+    except:
+        raise ModuleNotFoundError(f"The module tensorstore has not been found. Try 'conda install -c conda-forge tensorstore'")
+    rechunk_method = kwargs.get('rechunk_method', 'tasks')
+
+    compressor_name = kwargs.get('compressor', 'blosc')
+    compressor_params = kwargs.get('compressor_params', {})
+    compressor = dict(id = compressor_name, **compressor_params)
+    dtype = kwargs.get('dtype', arr.dtype)
+    fill_value = kwargs.get('fill_value', get_default_fill_value(dtype))
+
+    if dtype == 'auto':
+        dtype = arr.dtype
+
+    zarr_spec = {
+        "driver": "zarr",
+        "kvstore": {
+            "driver": "file",
+            "path": location,
+        },
+        "metadata": {
+            "dtype": dtype.str,
+            "shape": arr.shape,
+            "chunks": chunks,
+            "compressor": compressor,
+            "dimension_separator": "/",
+            "fill_value": fill_value
+        },
+    }
+
+    if not np.equal(arr.chunksize, chunks).all():
+        arr = arr.rechunk(chunks, method=rechunk_method)
+
+    ts_store = ts.open(zarr_spec, create=True, delete_existing=overwrite).result()
+    return arr.map_blocks(write_chunk_with_tensorstore, ts_store=ts_store, dtype=arr.dtype)
+
+def write_with_dask(arr: da.Array,
+                    chunks: Tuple[int, ...],
+                    location: Union[str, Path],
+                    overwrite: bool = True,
+                    **kwargs: Any
+                    ) -> List[da.Array]:
+    rechunk_method: str = kwargs.get('rechunk_method', 'tasks')
+
+    if not np.equal(arr.chunksize, chunks).all():
+        res: da.Array = arr.rechunk(chunks, method=rechunk_method)
+    else:
+        res: da.Array = arr
+
+    store: zarr.DirectoryStore = zarr.DirectoryStore(location, dimension_separator='/')
+    try:
+        zarr_array: zarr.Array = zarr.open_array(location, mode='r')
+    except:
+        compressor_name = kwargs.get('compressor', 'blosc')
+        compressor_params = kwargs.get('compressor_params', {})
+        compressor = get_compressor(compressor_name, **compressor_params)
+        dtype = kwargs.get('dtype', arr.dtype)
+        if dtype == 'auto':
+            dtype = arr.dtype
+        fill_value = kwargs.get('fill_value', get_default_fill_value(dtype))
+        zarr_array = zarr.create(shape=res.shape, chunks=chunks, dtype=dtype, compressor=compressor, store=store,
+                                 overwrite=overwrite, fill_value = fill_value)
+
+    region_shape: Tuple[int, ...] = kwargs.get('region_shape', chunks)
+    regions: List[Tuple[slice, ...]] = get_regions(arr.shape, region_shape, as_slices=True)
+    result: List[da.Array] = []
+    for slc in regions:
+        res: da.Array = da.to_zarr(
+            arr=arr[slc],
+            region=slc,
+            url=zarr_array,
+            compute=False,
+        )
+        result.append(res)
+
+    return result
+
+@delayed
+def count_threads():
+    return threading.active_count()
+
+def store_arrays(arrays: Dict[str, Dict[str, da.Array]],
+                 output_path: Union[Path, str],
+                 scales: Dict[str, Dict[str, Tuple[float, ...]]],
+                 units: list,
+                 output_chunks: Tuple[int, ...] = None,
+                 compute: bool = False,
+                 overwrite: bool = False,
+                 **kwargs) -> Dict[str, da.Array]:
+
+    rechunk_method = kwargs.get('rechunk_method', 'tasks')
+    use_tensorstore = kwargs.get('use_tensorstore', False)
+    verbose = kwargs.get('verbose', False)
+
+    arrays = {k: {'0': v} if not isinstance(v, dict) else v for k, v in arrays.items()}
+    flatarrays = {os.path.join(output_path, f"{key}.zarr" if not key.endswith('zarr') else key, str(level)): arr
+                  for key, subarrays in arrays.items()
+                  for level, arr in subarrays.items()}
+    flatscales = {os.path.join(output_path, f"{key}.zarr" if not key.endswith('zarr') else key, str(level)): scale
+                  for key, subscales in scales.items()
+                  for level, scale in subscales.items()}
+
+    if rechunk_method == 'rechunker':
+        writer_func = write_with_rechunker
+        if use_tensorstore:
+            raise NotImplementedError("The rechunker method cannot be used with tensorstore.")
+        if 'region_shape' in kwargs:
+            raise NotImplementedError("The rechunker method is not compatible with region-based writing.")
+    elif 'region_shape' in kwargs:
+        writer_func = write_with_dask
+        if use_tensorstore:
+            raise NotImplementedError("Region-based writing is not possible with tensorstore.")
+    else:
+        writer_func = write_with_tensorstore if use_tensorstore else write_with_zarrpy
+
+    try:
+        zarr.group(output_path, overwrite=overwrite)
+        results = {}
+        for key, arr in flatarrays.items():
+            flatscale = flatscales[key]
+            # Make sure chunk size is not larger than array shape in any dimension.
+            chunks = np.minimum(output_chunks or arr.chunksize, arr.shape)
+
+            if rechunk_method in (None, 'auto'):
+                if np.all(np.less_equal(chunks, arr.chunksize)):
+                    rechunk_method = 'rechunker'
+                    kwargs['rechunk_method'] = rechunk_method
+                    writer_func = write_with_rechunker
+                    if use_tensorstore:
+                        raise NotImplementedError("The rechunker method cannot be used with tensorstore.")
+                    if 'region_shape' in kwargs:
+                        raise NotImplementedError("The rechunker method is not compatible with region-based writing.")
+                else:
+                    kwargs['rechunk_method'] = 'tasks'
+
+            if rechunk_method != 'rechunker':
+                if 'temp_dir' in kwargs:
+                    kwargs.pop('temp_dir')
+
+            dirpath = os.path.dirname(key)
+            arrpath = os.path.basename(key)
+
+            gr = zarr.open_group(dirpath, mode='a') if is_zarr_group(dirpath) else zarr.group(dirpath, overwrite=overwrite)
+
+            meta = Multimeta()
+            try:
+                meta.from_ngff(gr)
+            except:
+                pass
+            if not meta.has_axes:
+                meta.parse_axes(axis_order='tczyx', unit_list=units)
+
+            meta.add_dataset(path=arrpath, scale=flatscale, overwrite=True)
+            meta.retag(os.path.basename(dirpath))
+            meta.to_ngff(gr)
+
+            if verbose:
+                print(f"Writer function: {writer_func}")
+                print(f"Rechunk method: {rechunk_method}")
+            results[key] = writer_func(arr=arr,
+                                       chunks=chunks,
+                                       location=key, # compressor = compressor, dtype = dtype,
+                                       overwrite=overwrite,
+                                       **kwargs
+                                       )
+
+        if compute:
+            if rechunk_method == 'rechunker':
+                for result in results.values():
+                    result.execute()
+            else:
+                dask.compute(list(results.values()))
+        else:
+            return results
+    except Exception as e:
+    #     print(e)
+        pass
+    return results
+
+
+
+
+

eubi_bridge-0.0.1/eubi_bridge/cmd.py

@@ -0,0 +1,6 @@
+import fire
+from eubi_bridge.ebridge import EuBIBridge
+
+def eubibridge_cmd():
+    _ = fire.Fire(EuBIBridge)
+    return