rechunkit 0.1.0__tar.gz

rechunkit-0.1.0/.gitignore

@@ -0,0 +1,119 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+junit/
+junit.xml
+test.db
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+.ruff*
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# .vscode
+.vscode/
+
+# OS files
+.DS_Store
+
+# Temp data
+data/*
+
+# Test config files
+/rechunkit/tests/*.toml
+/rechunkit/tests/*.yml

rechunkit-0.1.0/LICENSE

@@ -0,0 +1,16 @@
+Apache Software License 2.0
+
+Copyright (c) 2025, Mike Kittridge
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+

rechunkit-0.1.0/PKG-INFO

@@ -0,0 +1,138 @@
+Metadata-Version: 2.4
+Name: rechunkit
+Version: 0.1.0
+Summary: Functions to efficiently rechunk multidimensional arrays
+Project-URL: Documentation, https://mullenkamp.github.io/rechunkit/
+Project-URL: Source, https://github.com/mullenkamp/rechunkit
+Author-email: mullenkamp <mullenkamp1@gmail.com>
+License-File: LICENSE
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.9
+Requires-Dist: numpy>=1.26
+Description-Content-Type: text/markdown
+
+# rechunkit
+
+<p align="center">
+    <em>Functions to efficiently rechunk multidimensional arrays</em>
+</p>
+
+[![build](https://github.com/mullenkamp/rechunkit/workflows/Build/badge.svg)](https://github.com/mullenkamp/rechunkit/actions)
+[![codecov](https://codecov.io/gh/mullenkamp/rechunkit/branch/master/graph/badge.svg)](https://codecov.io/gh/mullenkamp/rechunkit)
+[![PyPI version](https://badge.fury.io/py/rechunkit.svg)](https://badge.fury.io/py/rechunkit)
+
+---
+
+**Source Code**: <a href="https://github.com/mullenkamp/rechunkit" target="_blank">https://github.com/mullenkamp/rechunkit</a>
+
+---
+## Introduction
+Rechunkit is a set of functions to allow efficient rechunking of multidimensional arrays that have been stored as chunks of numpy ndarrays. It allows for rechunking on-the-fly via python generators instead of requiring the user to save the full target array. It also contains several other handy tools for assisting the user as part of the rechunking process (e.g. estimating an optimal or ideal chunking size, iterating over chunks with a range-type function, etc).  
+
+
+## Installation
+```
+pip install rechunkit
+```
+I can add it to conda-forge if there is demand.
+
+## Usage
+Import the necessary modules and assign some parameters for the examples:
+
+```python
+from rechunkit import guess_chunk_shape, chunk_range, calc_ideal_read_chunk_shape, calc_ideal_read_chunk_mem, calc_source_read_chunk_shape, calc_n_chunks, calc_n_reads_simple, calc_n_reads_rechunker, rechunker
+
+source_shape = (31, 31, 31)
+shape = source_shape
+
+sel = (slice(3, 21), slice(11, 27), slice(7, 17))
+
+source_chunk_shape = (5, 2, 4)
+target_chunk_shape = (4, 5, 3)
+max_mem = 2000 # smaller than the ideal chunk size
+
+dtype = np.dtype('int32')
+```
+
+### Preprocessing tools
+We have defined our target_chunk_shape above, but rechunkit has a function to guess a good chunk shape given a user-defined amount of memory per chunk:
+
+```python
+new_chunk_shape = guess_chunk_shape(source_shape, dtype, 400)
+```
+
+Chunks will be assigned to the highest composite number within the target_chunk_size. Using composite numbers will benefit the rehunking process as there is a very high likelihood that the least common multiple (LCM) of two composite numbers will be significantly lower than the product of those two numbers. The LCM is used to determine the ideal chunk size for the rechunking process.
+
+Speaking of the ideal chunk size, we can determine the ideal chunk shape and size via a couple functions:
+
+```python
+ideal_read_chunk_shape = calc_ideal_read_chunk_shape(source_chunk_shape, target_chunk_shape) # (20, 10, 12)
+
+ideal_read_chunk_size = calc_ideal_read_chunk_mem(ideal_read_chunk_shape, dtype.itemsize) # 9600 bytes
+```
+
+If the ideal_read_chunk_size can comfortably fit in your memory, then you should use this value. Using the ideal chunk size will mean that you will only need to read all chunks in the source once. If the chunk size (called max_mem in the functions) is less than the ideal, then some chunks will need to be read multiple times. 
+
+To see how many reads are required if no optimization is performed during rechunking (i.e. every target chunk must iterate over every associated source chunk), you can use the calc_n_reads_simple function and compare it to the total number of chunks in the source:
+
+```python
+n_chunks_source = calc_n_chunks(source_shape, source_chunk_shape) # 896
+n_chunks_target = calc_n_chunks(source_shape, target_chunk_shape) # 616
+
+n_reads_simple = calc_n_reads_simple(source_shape, source_chunk_shape, target_chunk_shape) # 3952
+```
+
+Using the simple brute force method requires one chunk to be read 4.4 times on average.
+
+There's also a function to check the number of reads (and writes) using the optimized algorithm:
+
+```python
+n_reads, n_writes = calc_n_reads_rechunker(source_shape, dtype, source_chunk_shape, target_chunk_shape, max_mem) # 2044, 616
+```
+
+In this case, we only require one chunk to be read 2.28 times on average. The more max_mem you give to the rechunker, the less reads per chunk is required (to a minium of 1 in the ideal case).
+
+
+### Rechunking
+We need a source dataset to get data from. Rechunkit requires that the source input is a function/method that has a single parameter input of a tuple of slices. The slices contain the start and stop of the chunk to be read in the source. 
+
+For example, we can simply use a numpy array and it's `__getitem__` method as the source:
+
+```python
+source_data = np.arange(1, prod(source_shape) + 1, dtype=dtype).reshape(source_shape)
+source = source_data.__getitem__
+```
+
+And again as a simple example, we can use a numpy array as the target:
+
+```python
+target = np.zeros(source_shape, dtype=dtype)
+```
+
+We don't necessarily need the target as an array to be filled, because the rechunker function returns a generator that can be iterated over. The generator returns a tuple of slices (representing the target chunk) and the associated numpy array data:
+
+```python
+for write_chunk, data in rechunker(source, source_shape, dtype, source_chunk_shape, target_chunk_shape, max_mem):
+        target[write_chunk] = data
+    
+assert np.all(source(()) == target) # Should pass!
+```
+
+#### Subsets of the source
+There are many use-cases where you don't want the entire dataset. Rather you want a subset of the dataset, but you also want the subset rechunked. The rechunker function has a `sel` parameter which needs to be a tuple of slices of the number of dimensions.
+
+```python
+n_reads, n_writes = calc_n_reads_rechunker(source_shape, dtype, source_chunk_shape, target_chunk_shape, max_mem, sel) # 288, 80
+
+target = np.zeros(source_shape, dtype=dtype)[sel]
+
+for write_chunk, data in rechunker(source, source_shape, dtype, source_chunk_shape, target_chunk_shape, max_mem, sel):
+    target[write_chunk] = data
+
+assert np.all(source(sel) == target) # Should pass!
+```
+
+
+## License
+
+This project is licensed under the terms of the Apache Software License 2.0.

rechunkit-0.1.0/README.md

@@ -0,0 +1,125 @@
+# rechunkit
+
+<p align="center">
+    <em>Functions to efficiently rechunk multidimensional arrays</em>
+</p>
+
+[![build](https://github.com/mullenkamp/rechunkit/workflows/Build/badge.svg)](https://github.com/mullenkamp/rechunkit/actions)
+[![codecov](https://codecov.io/gh/mullenkamp/rechunkit/branch/master/graph/badge.svg)](https://codecov.io/gh/mullenkamp/rechunkit)
+[![PyPI version](https://badge.fury.io/py/rechunkit.svg)](https://badge.fury.io/py/rechunkit)
+
+---
+
+**Source Code**: <a href="https://github.com/mullenkamp/rechunkit" target="_blank">https://github.com/mullenkamp/rechunkit</a>
+
+---
+## Introduction
+Rechunkit is a set of functions to allow efficient rechunking of multidimensional arrays that have been stored as chunks of numpy ndarrays. It allows for rechunking on-the-fly via python generators instead of requiring the user to save the full target array. It also contains several other handy tools for assisting the user as part of the rechunking process (e.g. estimating an optimal or ideal chunking size, iterating over chunks with a range-type function, etc).  
+
+
+## Installation
+```
+pip install rechunkit
+```
+I can add it to conda-forge if there is demand.
+
+## Usage
+Import the necessary modules and assign some parameters for the examples:
+
+```python
+from rechunkit import guess_chunk_shape, chunk_range, calc_ideal_read_chunk_shape, calc_ideal_read_chunk_mem, calc_source_read_chunk_shape, calc_n_chunks, calc_n_reads_simple, calc_n_reads_rechunker, rechunker
+
+source_shape = (31, 31, 31)
+shape = source_shape
+
+sel = (slice(3, 21), slice(11, 27), slice(7, 17))
+
+source_chunk_shape = (5, 2, 4)
+target_chunk_shape = (4, 5, 3)
+max_mem = 2000 # smaller than the ideal chunk size
+
+dtype = np.dtype('int32')
+```
+
+### Preprocessing tools
+We have defined our target_chunk_shape above, but rechunkit has a function to guess a good chunk shape given a user-defined amount of memory per chunk:
+
+```python
+new_chunk_shape = guess_chunk_shape(source_shape, dtype, 400)
+```
+
+Chunks will be assigned to the highest composite number within the target_chunk_size. Using composite numbers will benefit the rehunking process as there is a very high likelihood that the least common multiple (LCM) of two composite numbers will be significantly lower than the product of those two numbers. The LCM is used to determine the ideal chunk size for the rechunking process.
+
+Speaking of the ideal chunk size, we can determine the ideal chunk shape and size via a couple functions:
+
+```python
+ideal_read_chunk_shape = calc_ideal_read_chunk_shape(source_chunk_shape, target_chunk_shape) # (20, 10, 12)
+
+ideal_read_chunk_size = calc_ideal_read_chunk_mem(ideal_read_chunk_shape, dtype.itemsize) # 9600 bytes
+```
+
+If the ideal_read_chunk_size can comfortably fit in your memory, then you should use this value. Using the ideal chunk size will mean that you will only need to read all chunks in the source once. If the chunk size (called max_mem in the functions) is less than the ideal, then some chunks will need to be read multiple times. 
+
+To see how many reads are required if no optimization is performed during rechunking (i.e. every target chunk must iterate over every associated source chunk), you can use the calc_n_reads_simple function and compare it to the total number of chunks in the source:
+
+```python
+n_chunks_source = calc_n_chunks(source_shape, source_chunk_shape) # 896
+n_chunks_target = calc_n_chunks(source_shape, target_chunk_shape) # 616
+
+n_reads_simple = calc_n_reads_simple(source_shape, source_chunk_shape, target_chunk_shape) # 3952
+```
+
+Using the simple brute force method requires one chunk to be read 4.4 times on average.
+
+There's also a function to check the number of reads (and writes) using the optimized algorithm:
+
+```python
+n_reads, n_writes = calc_n_reads_rechunker(source_shape, dtype, source_chunk_shape, target_chunk_shape, max_mem) # 2044, 616
+```
+
+In this case, we only require one chunk to be read 2.28 times on average. The more max_mem you give to the rechunker, the less reads per chunk is required (to a minium of 1 in the ideal case).
+
+
+### Rechunking
+We need a source dataset to get data from. Rechunkit requires that the source input is a function/method that has a single parameter input of a tuple of slices. The slices contain the start and stop of the chunk to be read in the source. 
+
+For example, we can simply use a numpy array and it's `__getitem__` method as the source:
+
+```python
+source_data = np.arange(1, prod(source_shape) + 1, dtype=dtype).reshape(source_shape)
+source = source_data.__getitem__
+```
+
+And again as a simple example, we can use a numpy array as the target:
+
+```python
+target = np.zeros(source_shape, dtype=dtype)
+```
+
+We don't necessarily need the target as an array to be filled, because the rechunker function returns a generator that can be iterated over. The generator returns a tuple of slices (representing the target chunk) and the associated numpy array data:
+
+```python
+for write_chunk, data in rechunker(source, source_shape, dtype, source_chunk_shape, target_chunk_shape, max_mem):
+        target[write_chunk] = data
+    
+assert np.all(source(()) == target) # Should pass!
+```
+
+#### Subsets of the source
+There are many use-cases where you don't want the entire dataset. Rather you want a subset of the dataset, but you also want the subset rechunked. The rechunker function has a `sel` parameter which needs to be a tuple of slices of the number of dimensions.
+
+```python
+n_reads, n_writes = calc_n_reads_rechunker(source_shape, dtype, source_chunk_shape, target_chunk_shape, max_mem, sel) # 288, 80
+
+target = np.zeros(source_shape, dtype=dtype)[sel]
+
+for write_chunk, data in rechunker(source, source_shape, dtype, source_chunk_shape, target_chunk_shape, max_mem, sel):
+    target[write_chunk] = data
+
+assert np.all(source(sel) == target) # Should pass!
+```
+
+
+## License
+
+This project is licensed under the terms of the Apache Software License 2.0.

rechunkit-0.1.0/pyproject.toml

@@ -0,0 +1,187 @@
+[project]
+name = "rechunkit"
+authors = [
+  { name = "mullenkamp", email = "mullenkamp1@gmail.com" }
+]
+description = "Functions to efficiently rechunk multidimensional arrays"
+readme = "README.md"
+dynamic = ["version"]
+classifiers = [
+    "Programming Language :: Python :: 3 :: Only",
+]
+requires-python = ">=3.9"
+dependencies = [
+  'numpy>=1.26'
+]
+
+[dependency-groups]
+dev = [
+  "spyder-kernels==2.5.2",
+  "black",
+  "mypy",
+  "ruff",
+  "pytest",
+  "pytest-cov",
+]
+
+[tool.hatch]
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+# [tool.hatch.version]
+# source = "regex_commit"
+# commit_extra_args = ["-e"]
+# path = "rechunkit/__init__.py"
+
+[tool.hatch.envs.default]
+python = "3.11"
+dependencies = [
+    "spyder-kernels==2.5.2",
+    "black",
+    "mypy",
+    "ruff",
+    "pytest",
+    "pytest-cov",
+    "mkdocs-material",
+    "mkdocstrings[python]",
+    
+]
+
+[[tool.hatch.envs.all.matrix]]
+python = ['3.10', '3.11', '3.12']
+
+[tool.hatch.envs.lint]
+detached = true
+dependencies = [
+  "black>=23.1.0",
+  "mypy>=1.0.0",
+  "ruff>=0.0.243",
+]
+[tool.hatch.envs.lint.scripts]
+typing = "mypy --install-types --non-interactive {args:src/rechunkit tests}"
+style = [
+  "ruff {args:.}",
+  "black --check --diff {args:.}",
+]
+fmt = [
+  "black {args:.}",
+  "ruff --fix {args:.}",
+  "style",
+]
+all = [
+  "style",
+  "typing",
+]
+
+[tool.hatch.envs.default.scripts]
+test = "pytest {args:tests}"
+test-cov = "coverage run -m pytest {args:tests}"
+cov-report = [
+  "- coverage combine",
+  "coverage report",
+]
+cov = [
+  "test-cov",
+  "cov-report",
+]
+docs-serve = "mkdocs serve"
+docs-build = "mkdocs build"
+
+[tool.black]
+target-version = ["py37"]
+line-length = 120
+skip-string-normalization = true
+
+[tool.ruff]
+target-version = "py311"
+line-length = 120
+select = [
+  "A",
+  "ARG",
+  "B",
+  "C",
+  "DTZ",
+  "E",
+  "EM",
+  "F",
+  "FBT",
+  "I",
+  "ICN",
+  "ISC",
+  "N",
+  "PLC",
+  "PLE",
+  "PLR",
+  "PLW",
+  "Q",
+  "RUF",
+  "S",
+  "T",
+  "TID",
+  "UP",
+  "W",
+  "YTT",
+]
+ignore = [
+  # Allow non-abstract empty methods in abstract base classes
+  "B027",
+  # Allow boolean positional values in function calls, like `dict.get(... True)`
+  "FBT003",
+  # Ignore checks for possible passwords
+  "S105", "S106", "S107",
+  # Ignore complexity
+  "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
+]
+unfixable = [
+  # Don't touch unused imports
+  "F401",
+]
+
+[tool.ruff.isort]
+known-first-party = ["rechunkit"]
+
+[tool.ruff.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.per-file-ignores]
+# Tests can use magic values, assertions, and relative imports
+"tests/**/*" = ["PLR2004", "S101", "TID252"]
+
+[tool.coverage.run]
+source_pkgs = ["rechunkit", "tests"]
+branch = true
+parallel = true
+omit = [
+  "src/rechunkit/__about__.py",
+]
+
+[tool.coverage.paths]
+rechunkit = ["src/rechunkit", "*/rechunkit/src/rechunkit"]
+tests = ["tests", "*/rechunkit/tests"]
+
+[tool.coverage.report]
+exclude_lines = [
+  "no cov",
+  "if __name__ == .__main__.:",
+  "if TYPE_CHECKING:",
+]
+
+[build-system]
+requires = ["hatchling>=1.26.1"]
+build-backend = "hatchling.build"
+
+[project.urls]
+Documentation = "https://mullenkamp.github.io/rechunkit/"
+Source = "https://github.com/mullenkamp/rechunkit"
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "/rechunkit",
+]
+exclude = [
+    "/rechunkit/tests/*",
+]
+
+[tool.hatch.version]
+path = "rechunkit/__init__.py"

rechunkit-0.1.0/rechunkit/__init__.py

@@ -0,0 +1,5 @@
+"""Functions to efficiently rechunk multidimensional arrays"""
+from rechunkit.main import guess_chunk_shape, chunk_range, calc_ideal_read_chunk_shape, calc_ideal_read_chunk_mem, calc_source_read_chunk_shape, calc_n_chunks, calc_n_reads_simple, calc_n_reads_rechunker, rechunker
+
+
+__version__ = '0.1.0'

rechunkit-0.1.0/rechunkit/main.py

@@ -0,0 +1,557 @@
+"""Core rechunking algorithm stuff."""
+# import copy
+from typing import List, Optional, Sequence, Tuple, Iterator, Generator
+import numpy as np
+import itertools
+# from time import time
+from math import prod, lcm, ceil
+from collections import Counter, deque
+from itertools import count
+from bisect import bisect
+
+########################################################
+### Parameters
+
+composite_numbers = (1, 2, 4, 6, 12, 24, 36, 48, 60, 120, 180, 240, 360, 720, 840, 1260, 1680, 2520, 5040, 7560, 10080, 15120, 20160, 25200, 27720, 45360, 50400, 55440, 83160, 110880, 166320, 221760, 277200, 332640, 498960, 554400, 665280, 720720, 1081080, 1441440, 2162160)
+
+########################################################
+### Functions
+
+
+def guess_chunk_shape(shape: Tuple[int, ...], dtype: np.dtype, target_chunk_size: int = 2**21) -> Tuple[int, ...]:
+    """
+    Guess an appropriate chunk layout for a dataset, given its shape and
+    the size of each element in bytes.  Will allocate chunks only as large
+    as target_chunk_size. Chunks will be assigned to the highest composite number within the target_chunk_size. Using composite numbers will benefit the rehunking process as there is a very high likelihood that the least common multiple of two composite numbers will be significantly lower than the product of those two numbers.
+
+    Parameters
+    ----------
+    shape: tuple of ints
+        Shape of the array.
+    dtype: np.dtype or str
+        The dtype of the array.
+    target_chunk_size: int
+        The maximum size per chunk in bytes.
+
+    Returns
+    -------
+    tuple of ints
+        shape of the chunk
+    """
+    ndims = len(shape)
+
+    if ndims > 0:
+
+        if not all(isinstance(v, int) for v in shape):
+            raise TypeError('All values in the shape must be ints.')
+
+        chunks = np.array(shape, dtype='=f8')
+        if not np.all(np.isfinite(chunks)):
+            raise ValueError("Illegal value in chunk tuple")
+
+        dtype = np.dtype(dtype)
+        typesize = dtype.itemsize
+
+        idx = 0
+        while True:
+            chunk_bytes = prod(chunks)*typesize
+
+            if (chunk_bytes < target_chunk_size or \
+             abs(chunk_bytes - target_chunk_size)/target_chunk_size < 0.5):
+                break
+
+            if prod(chunks) == 1:
+                break
+
+            chunks[idx%ndims] = ceil(chunks[idx%ndims] / 2.0)
+            idx += 1
+
+        return tuple(composite_numbers[bisect(composite_numbers, int(x)) - 1] for x in chunks)
+    else:
+        return None
+
+
+def get_slice_min_max(read_slices, write_slices):
+    """
+    Function to get the max start position and the min stop position.
+    """
+    slices = tuple(slice(max(rs.start, ws.start), min(rs.stop, ws.stop)) for rs, ws in zip(read_slices, write_slices))
+
+    return slices
+
+
+def chunk_range(
+    chunk_start: Tuple[int, ...], chunk_stop: Tuple[int, ...], chunk_step: Tuple[int, ...], include_partial_chunks=True, clip_ends=True,
+) -> Iterator[Tuple[slice, ...]]:
+    """
+    Generator like the Python range function, but for multiple dimensions and it returns tuples of slices.
+
+    Parameters
+    ----------
+    chunk_start: tuple of int
+        The start positions of the chunks.
+    chunk_stop: tuple of int
+        The stop positions of the chunks.
+    chunk_step: tuple of int
+        The chunking step.
+    include_partial_chunks: bool
+        Should partial chunks be included? True by default.
+    clip_ends: bool
+        Only applies when include_partial_chunks == True. Should the chunks be clipped to the overall extents? True by default.
+
+    Returns
+    -------
+    Generator with tuples of slices
+    """
+    if not isinstance(chunk_start, tuple):
+        chunk_start = tuple(0 for i in range(len(chunk_stop)))
+
+    if include_partial_chunks:
+        start_ranges = [cs * (sc//cs) for cs, sc in zip(chunk_step, chunk_start)]
+    else:
+        start_ranges = [cs * (((sc - 1)//cs) + 1) for cs, sc in zip(chunk_step, chunk_start)]
+
+    ranges = [range(sr, ec, cs) for ec, cs, sr in zip(chunk_stop, chunk_step, start_ranges)]
+
+    for indices in itertools.product(*ranges):
+        # print(indices)
+        inside = True
+        res = []
+        for i, ec, cs, sc in zip(indices, chunk_stop, chunk_step, chunk_start):
+            stop = i + cs
+            if stop > ec:
+                if clip_ends:
+                    stop = ec
+                inside = False
+
+            start = i
+            if start < sc:
+                if clip_ends:
+                    start = sc
+                inside = False
+
+            res.append(slice(start, stop))
+
+        if inside or include_partial_chunks:
+            yield tuple(res)
+
+
+def calc_ideal_read_chunk_shape(source_chunk_shape, target_chunk_shape):
+    """
+    Calculates the minimum ideal read chunk shape between a source and target.
+    """
+    return tuple(lcm(i, s) for i, s in zip(source_chunk_shape, target_chunk_shape))
+
+
+def calc_ideal_read_chunk_mem(ideal_read_chunk_shape, itemsize):
+    """
+    Calculates the minimum ideal read chunk memory between a source and target.
+    """
+    return int(prod(ideal_read_chunk_shape) * itemsize)
+
+
+def calc_source_read_chunk_shape(source_chunk_shape, target_chunk_shape, itemsize, max_mem):
+    """
+    Calculates the optimum read chunk shape given a maximum amount of available memory.
+
+    Parameters
+    ----------
+    source_chunk_shape: tuple of int
+        The source chunk shape
+    target_chunk_shape: tuple of int
+        The target chunk shape
+    itemsize: int
+        The byte length of the data type.
+    max_mem: int
+        The max allocated memory to perform the chunking operation in bytes.
+
+    Returns
+    -------
+    optimal chunk shape: tuple of ints
+    """
+    max_cells = max_mem // itemsize
+    source_len = len(source_chunk_shape)
+    target_len = len(target_chunk_shape)
+
+    if source_len != target_len:
+        raise ValueError('The source_chunk_shape and target_chunk_shape do not have the same number of dims.')
+
+    tot_source = prod(source_chunk_shape)
+    if tot_source >= max_cells:
+        return source_chunk_shape
+
+    new_chunks = list(calc_ideal_read_chunk_shape(source_chunk_shape, target_chunk_shape))
+
+    ## Max mem
+    tot_target = prod(new_chunks)
+    pos = 0
+    while tot_target > max_cells:
+        prod_chunk = new_chunks[pos]
+        source_chunk = source_chunk_shape[pos]
+        if prod_chunk > source_chunk:
+            new_chunks[pos] = prod_chunk - source_chunk
+
+        tot_target = prod(new_chunks)
+
+        if tot_target == tot_source:
+            return source_chunk_shape
+        else:
+            if pos + 1 == source_len:
+                pos = 0
+            else:
+                pos += 1
+
+    ## Min mem
+    n_chunks_write = tuple(s//target_chunk_shape[i] for i, s in enumerate(new_chunks))
+    for i in range(len(new_chunks)):
+        while True:
+            n_chunk_write = n_chunks_write[i]
+            prod_chunk = new_chunks[i]
+            source_chunk = source_chunk_shape[i]
+            target_chunk = target_chunk_shape[i]
+            new_chunk = prod_chunk - source_chunk
+            if new_chunk//target_chunk == n_chunk_write:
+                new_chunks[i] = new_chunk
+            else:
+                break
+
+    return tuple(new_chunks)
+
+
+def calc_n_chunks_per_read(source_chunk_shape, source_read_chunk_shape):
+    """
+
+    """
+    return prod(tuple(nc//sc for nc, sc in zip(source_read_chunk_shape, source_chunk_shape)))
+
+
+def calc_n_chunks(shape, chunk_shape):
+    """
+
+    """
+    chunk_start = tuple(0 for i in range(len(shape)))
+    chunk_iter = chunk_range(chunk_start, shape, chunk_shape)
+
+    counter = count()
+    deque(zip(chunk_iter, counter), maxlen=0)
+
+    return next(counter)
+
+
+def calc_n_reads_simple(shape, source_chunk_shape, target_chunk_shape):
+    """
+    Brute force chunking read count. Every target chunk must iterate over every associated source chunk. This should be considered the maximum number of reads between a source and target (most inefficient). The number of writes is the total number of chunks in the target.
+
+    Parameters
+    ----------
+    shape: tuple of ints
+        The shape of the source dataset, which will also be the shape of the target dataset.
+    dtype: np.dtype
+        The numpy data type of the source/target.
+    source_chunk_shape: tuple of ints
+        The chunk_shape of the source.
+    target_chunk_shape: tuple of ints
+        The chunk_shape of the target.
+
+    Returns
+    -------
+    int
+        Count of the number of reads
+    """
+    chunk_start = tuple(0 for i in range(len(shape)))
+    read_counter = count()
+
+    for write_chunk in chunk_range(chunk_start, shape, target_chunk_shape):
+        write_chunk_start = tuple(rc.start for rc in write_chunk)
+        write_chunk_stop = tuple(rc.stop for rc in write_chunk)
+        for chunk_slice in chunk_range(write_chunk_start, write_chunk_stop, source_chunk_shape):
+            next(read_counter)
+
+    return next(read_counter)
+
+
+def calc_n_reads_rechunker(shape: Tuple[int, ...], dtype: np.dtype, source_chunk_shape: Tuple[int, ...], target_chunk_shape: Tuple[int, ...], max_mem: int, sel=None) -> Tuple[int, int]:
+    """
+    This function calculates the total number of reads (aand writes) using the more optimized rechunking algorithm. It optimises the rechunking by using an in-memory numpy ndarray with a size defined by the max_mem provided by the user.
+
+    Parameters
+    ----------
+    source: array-like
+        The source function to read the dataset/array. The function must have a single parameter input as a tuple of slices to retrieve an array chunk of data.
+    shape: tuple of ints
+        The shape of the source dataset, which will also be the shape of the target dataset.
+    dtype: np.dtype
+        The numpy data type of the source/target.
+    source_chunk_shape: tuple of ints
+        The chunk_shape of the source.
+    target_chunk_shape: tuple of ints
+        The chunk_shape of the target.
+    max_mem: int
+        The max allocated memory to perform the chunking operation in bytes.
+    sel: tuple of slices
+        A subset selection of the source in the form of a tuple of slices. The starts and stops must be within the shape of the source.
+
+    Returns
+    -------
+    tuple
+        number of reads, number of writes
+    """
+    itemsize = dtype.itemsize
+
+    ## Calc the optimum read_chunk_shape
+    source_read_chunk_shape = calc_source_read_chunk_shape(source_chunk_shape, target_chunk_shape, itemsize, max_mem)
+
+    ## Calc ideal read chunking shape
+    ideal_read_chunk_shape = calc_ideal_read_chunk_shape(source_chunk_shape, target_chunk_shape)
+
+    chunk_start = tuple(0 for i in range(len(shape)))
+
+    if sel is None:
+        target_shape = shape
+    else:
+        # Checks
+        for s, sh in zip(sel, shape):
+            if s.start < 0 or s.stop > sh:
+                raise ValueError('The selection must be a subset of the source.')
+
+        target_shape = tuple(s.stop - s.start for s in sel)
+
+    ## Counters
+    read_counter = count()
+    write_counter = count()
+    # write_counter2 = count()
+
+    ## If the read chunking is set to the ideal chunking case, then use the simple implementation. Otherwise, use the more complicated one.
+    if source_read_chunk_shape == ideal_read_chunk_shape:
+        read_chunk_iter = chunk_range(chunk_start, target_shape, source_read_chunk_shape)
+        for read_chunk_grp in read_chunk_iter:
+            read_chunk_grp_start = tuple(s.start for s in read_chunk_grp)
+            read_chunk_grp_stop = tuple(s.stop for s in read_chunk_grp)
+            for read_chunk in chunk_range(read_chunk_grp_start, read_chunk_grp_stop, source_chunk_shape):
+                next(read_counter)
+
+            for write_chunk1 in chunk_range(read_chunk_grp_start, read_chunk_grp_stop, target_chunk_shape):
+                next(write_counter)
+
+    else:
+        writen_chunks = set() # Need to keep track of the bulk writes
+
+        write_chunk_iter = chunk_range(chunk_start, target_shape, target_chunk_shape)
+
+        for write_chunk in write_chunk_iter:
+            write_chunk_start = tuple(s.start for s in write_chunk)
+            if write_chunk_start not in writen_chunks:
+                write_chunk_stop = tuple(s.stop for s in write_chunk)
+
+                read_chunk_start = tuple(rc * (wc//rc) for wc, rc in zip(write_chunk_start, source_chunk_shape))
+                read_chunk_stop = tuple(min(max(rcs + rc, wc), sh) for rcs, rc, wc, sh in zip(read_chunk_start, source_read_chunk_shape, write_chunk_stop, shape))
+                read_chunks_iter = chunk_range(read_chunk_start, read_chunk_stop, source_chunk_shape, True, False)
+
+                if all(stop - start <= rcs for start, stop, rcs in zip(read_chunk_start, read_chunk_stop, source_read_chunk_shape)):
+                    for read_chunk in read_chunks_iter:
+                        next(read_counter)
+
+                    is_end_chunk = any(wc.stop == ts for wc, ts in zip(write_chunk, target_shape))
+                    for write_chunk1 in chunk_range(write_chunk_start, read_chunk_stop, target_chunk_shape, include_partial_chunks=is_end_chunk, clip_ends=False):
+                        write_chunk2 = tuple(slice(wc.start, min(wc.stop, s)) for wc, s in zip(write_chunk1, target_shape))
+                        if all(all((wc.stop - wcs <= src, wc.start < wc.stop)) for wcs, wc, src in zip(read_chunk_start, write_chunk2, source_read_chunk_shape)):
+                            write_chunk1_start = tuple(s.start for s in write_chunk2)
+                            if write_chunk1_start not in writen_chunks:
+                                next(write_counter)
+
+                                writen_chunks.add(write_chunk1_start)
+                else:
+                    for read_chunk in read_chunks_iter:
+                        next(read_counter)
+
+                    next(write_counter)
+
+                    writen_chunks.add(write_chunk_start)
+
+    return next(read_counter), next(write_counter)
+
+
+def rechunker(source: object, shape: Tuple[int, ...], dtype: np.dtype, source_chunk_shape: Tuple[int, ...], target_chunk_shape: Tuple[int, ...], max_mem: int, sel=None) -> Iterator[Tuple[Tuple[slice, ...], np.ndarray]]:
+    """
+    This function takes a source dataset function with a specific chunk_shape and returns a generator that converts to a new chunk_shape. It optimises the rechunking by using an in-memory numpy ndarray with a size defined by the max_mem provided by the user. 
+
+    Parameters
+    ----------
+    source: array-like
+        The source function to read the dataset/array. The function must have a single parameter input as a tuple of slices to retrieve an array chunk of data.
+    shape: tuple of ints
+        The shape of the source dataset, which will also be the shape of the target dataset unless sel is passed.
+    dtype: np.dtype
+        The numpy data type of the source/target.
+    source_chunk_shape: tuple of ints
+        The chunk_shape of the source.
+    target_chunk_shape: tuple of ints
+        The chunk_shape of the target.
+    max_mem: int
+        The max allocated memory to perform the chunking operation in bytes. This will only be as large as necessary for an optimum size chunk for the rechunking.
+    sel: tuple of slices
+        A subset selection of the source in the form of a tuple of slices. The starts and stops must be within the shape of the source.
+
+    Returns
+    -------
+    Generator
+        tuple of the target slices to the np.ndarray of data
+    """
+    itemsize = dtype.itemsize
+
+    ## Calc the optimum read_chunk_shape
+    source_read_chunk_shape = calc_source_read_chunk_shape(source_chunk_shape, target_chunk_shape, itemsize, max_mem)
+
+    mem_arr1 = np.zeros(source_read_chunk_shape, dtype=dtype)
+
+    ## Calc ideal read chunking shape
+    ideal_read_chunk_shape = calc_ideal_read_chunk_shape(source_chunk_shape, target_chunk_shape)
+
+    chunk_start = tuple(0 for i in range(len(shape)))
+
+    if sel is None:
+        chunk_read_offset = chunk_start
+        target_shape = shape
+    else:
+        # Checks
+        for s, sh in zip(sel, shape):
+            if s.start < 0 or s.stop > sh:
+                raise ValueError('The selection must be a subset of the source.')
+
+        chunk_read_offset = tuple(s.start for s in sel)
+        target_shape = tuple(s.stop - s.start for s in sel)
+
+    # target = np.zeros(target_shape, dtype=dtype)
+
+    ## If the read chunking is set to the ideal chunking case, then use the simple implementation. Otherwise, use the more complicated one.
+    if source_read_chunk_shape == ideal_read_chunk_shape:
+        read_chunk_iter = chunk_range(chunk_start, target_shape, source_read_chunk_shape)
+        for read_chunk_grp in read_chunk_iter:
+            read_chunk_grp_start = tuple(s.start for s in read_chunk_grp)
+            read_chunk_grp_stop = tuple(s.stop for s in read_chunk_grp)
+            for read_chunk in chunk_range(read_chunk_grp_start, read_chunk_grp_stop, source_chunk_shape):
+                offset_slices = tuple(slice(rc.start - wcs, rc.stop - wcs) for wcs, rc in zip(read_chunk_grp_start, read_chunk))
+                read_chunk1 = tuple(slice(rc.start + cro, rc.stop + cro) for rc, cro in zip(read_chunk, chunk_read_offset))
+                mem_arr1[offset_slices] = source(read_chunk1)
+
+            for write_chunk1 in chunk_range(read_chunk_grp_start, read_chunk_grp_stop, target_chunk_shape):
+                offset_slices = tuple(slice(wc.start - wcs, wc.stop - wcs) for wcs, wc in zip(read_chunk_grp_start, write_chunk1))
+
+                # target[write_chunk1] = mem_arr1[offset_slices]
+
+                yield write_chunk1, mem_arr1[offset_slices]
+
+    else:
+        writen_chunks = set() # Need to keep track of the bulk writes
+
+        write_chunk_iter = chunk_range(chunk_start, target_shape, target_chunk_shape)
+
+        for write_chunk in write_chunk_iter:
+            write_chunk_start = tuple(s.start for s in write_chunk)
+            if write_chunk_start not in writen_chunks:
+                write_chunk_stop = tuple(s.stop for s in write_chunk)
+
+                read_chunk_start = tuple(rc * (wc//rc) for wc, rc in zip(write_chunk_start, source_chunk_shape))
+                read_chunk_stop = tuple(min(max(rcs + rc, wc), sh) for rcs, rc, wc, sh in zip(read_chunk_start, source_read_chunk_shape, write_chunk_stop, shape))
+                read_chunks_iter = chunk_range(read_chunk_start, read_chunk_stop, source_chunk_shape, True, False)
+
+                if all(stop - start <= rcs for start, stop, rcs in zip(read_chunk_start, read_chunk_stop, source_read_chunk_shape)):
+                    for read_chunk in read_chunks_iter:
+                        read_chunk1 = tuple(slice(rc.start + cro, min(rc.stop + cro, s)) for rc, cro, s in zip(read_chunk, chunk_read_offset, shape))
+                        offset_slices = tuple(slice(rc.start - rcs - rco, rc.stop - rcs - rco) for rcs, rco, rc in zip(read_chunk_start, chunk_read_offset, read_chunk1))
+
+                        mem_arr1[offset_slices] = source(read_chunk1)
+
+                    is_end_chunk = any(wc.stop == ts for wc, ts in zip(write_chunk, target_shape))
+                    for write_chunk1 in chunk_range(write_chunk_start, read_chunk_stop, target_chunk_shape, include_partial_chunks=is_end_chunk, clip_ends=False):
+                        write_chunk2 = tuple(slice(wc.start, min(wc.stop, s)) for wc, s in zip(write_chunk1, target_shape))
+                        if all(all((wc.stop - wcs <= src, wc.start < wc.stop)) for wcs, wc, src in zip(read_chunk_start, write_chunk2, source_read_chunk_shape)):
+                            write_chunk1_start = tuple(s.start for s in write_chunk2)
+                            if write_chunk1_start not in writen_chunks:
+                                offset_slices = tuple(slice(wc.start - rcs, wc.stop - rcs) for rcs, wc in zip(read_chunk_start, write_chunk2))
+                                # print(write_chunk1, offset_slices)
+
+                                # target[write_chunk2] = mem_arr1[offset_slices]
+
+                                yield write_chunk2, mem_arr1[offset_slices]
+
+                                writen_chunks.add(write_chunk1_start)
+                                # if write_chunk1_start == (14, 5):
+                                #     raise ValueError()
+                else:
+                    mem_read_chunk_slice = tuple(slice(0, wc.stop - wc.start) for wc in write_chunk)
+                    # for read_chunk in chunk_range(write_chunk_start, write_chunk_stop, source_chunk_shape, True, False):
+                    for read_chunk in read_chunks_iter:
+                        read_chunk1 = tuple(slice(rc.start + cro, rc.stop + cro) for rc, cro in zip(read_chunk, chunk_read_offset))
+                        clip_read_chunk = get_slice_min_max(read_chunk, write_chunk)
+                        read_slice = tuple(slice(cc.start - rc.start, cc.stop - rc.start) for cc, rc in zip(clip_read_chunk, read_chunk))
+                        write_slice = tuple(slice(cc.start - rc.start, cc.stop - rc.start) for cc, rc in zip(clip_read_chunk, write_chunk))
+
+                        # print(read_chunk, read_slice, write_slice)
+                        mem_arr1[write_slice] = source(read_chunk1)[read_slice]
+
+                    # target[write_chunk] = mem_arr1[mem_read_chunk_slice]
+
+                    yield write_chunk, mem_arr1[mem_read_chunk_slice]
+
+                    writen_chunks.add(write_chunk_start)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+