ziptif 0.1.0__tar.gz

ziptif-0.1.0/.gitignore

@@ -0,0 +1,14 @@
+# pixi environments
+.pixi
+*.egg-info
+*.log
+outputs/
+*.mypy_cache/
+*.ruff_cache/
+__pycache__/
+.vscode/
+*.*~
+__marimo__/
+*.out
+*.png
+.aider*

ziptif-0.1.0/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.5
+Name: ziptif
+Version: 0.1.0
+Summary: ZipTif file format for Satellite Image Time Series.
+Author: Centre Borelli
+License-Expression: MIT
+Requires-Python: >=3.11
+Requires-Dist: fire>=0.7.1
+Requires-Dist: polars>=1.33.1
+Requires-Dist: tifffile>=2025.9.20
+Requires-Dist: xarray>=2024.0.0
+Requires-Dist: zstd>=1.5.7.2
+Description-Content-Type: text/markdown
+
+File format to store multimodal Satellite Image Time Series.
+
+Reading a partial subset (site, time, space, spectral, modality) is optimized for training neural networks.
+The format is composed of a zip containing tiffs (sorted for optimal reading speed), and a binary index file (`.ziptif`).
+
+The reader supports lazy concatenation datasets along the 'site' and 'modality' axis, thus allowing arbitrarly large training dataset.
+
+Tested with Sentinel-1 and Sentinel-2 for now. Functions to create a dataset make a few assumptions that are not yet documented here.
+

ziptif-0.1.0/README.md

@@ -0,0 +1,9 @@
+File format to store multimodal Satellite Image Time Series.
+
+Reading a partial subset (site, time, space, spectral, modality) is optimized for training neural networks.
+The format is composed of a zip containing tiffs (sorted for optimal reading speed), and a binary index file (`.ziptif`).
+
+The reader supports lazy concatenation datasets along the 'site' and 'modality' axis, thus allowing arbitrarly large training dataset.
+
+Tested with Sentinel-1 and Sentinel-2 for now. Functions to create a dataset make a few assumptions that are not yet documented here.
+

ziptif-0.1.0/pyproject.toml

@@ -0,0 +1,39 @@
+[project]
+name = "ziptif"
+license = "MIT"
+readme = "README.md"
+description = "ZipTif file format for Satellite Image Time Series."
+requires-python = ">= 3.11"
+version = "0.1.0"
+dependencies = [
+  "tifffile>=2025.9.20"
+, "zstd>=1.5.7.2"
+, "polars>=1.33.1"
+, "fire>=0.7.1"
+, "xarray>=2024.0.0"]
+authors = [
+  { name = "Centre Borelli" },
+]
+
+[build-system]
+build-backend = "hatchling.build"
+requires = ["hatchling"]
+
+[tool.hatch.build.targets.sdist]
+only-include = ["src/ziptif"]
+
+[tool.pixi.workspace]
+channels = ["conda-forge"]
+platforms = ["linux-64", "osx-arm64"]
+
+[tool.pixi.pypi-dependencies]
+ziptif = { path = ".", editable = true }
+
+[tool.pixi.tasks]
+
+[dependency-groups]
+dev = ["pytest"]
+
+[tool.pixi.environments]
+default = { features = [] }
+dev = { features = ["dev"] }

ziptif-0.1.0/src/ziptif/__init__.py

@@ -0,0 +1,5 @@
+__all__ = [
+    "ziptif",
+    "loader",
+    "xarray_backend",
+]

ziptif-0.1.0/src/ziptif/buffer.py

@@ -0,0 +1,50 @@
+import abc
+from dataclasses import dataclass
+from typing import Protocol, override
+
+
+class IBuffer(Protocol):
+    @abc.abstractmethod
+    def read_at(self, offset: int, length: int) -> bytes: ...
+
+
+@dataclass(frozen=True)
+class OffsetedBuffer(IBuffer):
+    buffer: bytes
+    offset: int
+
+    @override
+    def read_at(self, offset: int, length: int) -> bytes:
+        offset_in_buffer = offset - self.offset
+        return self.buffer[offset_in_buffer : offset_in_buffer + length]
+
+
+@dataclass(frozen=True)
+class MultiOffsetedBuffer(IBuffer):
+    buffers: list[OffsetedBuffer]
+
+    def __post_init__(self):
+        # assert that buffers are sorted by offset
+        sorted_buffers = sorted(self.buffers, key=lambda b: b.offset)
+        if sorted_buffers != self.buffers:
+            raise ValueError("Buffers must be sorted by offset")
+
+    @override
+    def read_at(self, offset: int, length: int) -> bytes:
+        result = b""
+        remaining = length
+        current_offset = offset
+
+        for buf in self.buffers:
+            if buf.offset <= current_offset < buf.offset + len(buf.buffer):
+                to_read = min(
+                    remaining, len(buf.buffer) - (current_offset - buf.offset)
+                )
+                result += buf.read_at(current_offset, to_read)
+                remaining -= to_read
+                current_offset += to_read
+
+                if remaining == 0:
+                    return result
+
+        raise ValueError(f"Offset {offset} out of range for length {length}")

ziptif-0.1.0/src/ziptif/cache.py

@@ -0,0 +1,295 @@
+"""
+Buffer caching for ziptif format.
+
+Implements LRU caching at the byte-range level to avoid redundant disk reads
+when accessing overlapping or repeated data regions.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from ziptif.buffer import OffsetedBuffer
+
+if TYPE_CHECKING:
+    from ziptif.fs import ZipsReader
+
+
+@dataclass
+class CachedRange:
+    """A cached byte range from a zip file."""
+
+    offset: int
+    buffer: bytes
+    last_access: float
+    size: int
+
+    def __post_init__(self):
+        assert self.size == len(self.buffer)
+
+    def contains(self, offset: int, length: int) -> bool:
+        """Check if this cached range fully contains the requested range."""
+        return self.offset <= offset and offset + length <= self.offset + self.size
+
+    def overlaps(self, offset: int, length: int) -> bool:
+        """Check if this cached range overlaps with the requested range."""
+        return not (offset + length <= self.offset or offset >= self.offset + self.size)
+
+    def get_overlap(self, offset: int, length: int) -> tuple[int, int] | None:
+        """
+        Get the overlapping portion of this cache with the requested range.
+        Returns (overlap_offset, overlap_length) or None if no overlap.
+        """
+        if not self.overlaps(offset, length):
+            return None
+
+        overlap_start = max(self.offset, offset)
+        overlap_end = min(self.offset + self.size, offset + length)
+        return overlap_start, overlap_end - overlap_start
+
+
+class BufferCache:
+    """
+    LRU cache for byte ranges read from zip files.
+
+    The cache works at the (zip_id, offset, length) level. When a read is requested,
+    the cache determines which parts are already available and only reads missing
+    byte ranges from disk.
+    """
+
+    def __init__(self, max_size_bytes: int = 1_000_000_000):
+        """
+        Initialize buffer cache.
+
+        Parameters
+        ----------
+        max_size_bytes : int
+            Maximum cache size in bytes (default: 1GB)
+        """
+        self._cached_ranges: dict[str, list[CachedRange]] = {}
+        self._max_size = max_size_bytes
+        self._current_size = 0
+        self._lock = threading.Lock()
+
+    def _evict_lru(self) -> None:
+        """Evict least recently used cache entries until under size limit."""
+        while self._current_size > self._max_size:
+            # Find the LRU entry across all zip files
+            oldest_zip_id = None
+            oldest_idx = None
+            oldest_time = float("inf")
+
+            for zip_id, ranges in self._cached_ranges.items():
+                for idx, cached_range in enumerate(ranges):
+                    if cached_range.last_access < oldest_time:
+                        oldest_time = cached_range.last_access
+                        oldest_zip_id = zip_id
+                        oldest_idx = idx
+
+            if oldest_zip_id is None:
+                break
+
+            # Remove the oldest entry
+            removed = self._cached_ranges[oldest_zip_id].pop(oldest_idx)
+            self._current_size -= removed.size
+
+            # Clean up empty lists
+            if not self._cached_ranges[oldest_zip_id]:
+                del self._cached_ranges[oldest_zip_id]
+
+    def _add_to_cache(self, zip_id: str, offset: int, buffer: bytes) -> None:
+        """Add a new range to the cache."""
+        cached_range = CachedRange(
+            offset=offset, buffer=buffer, last_access=time.time(), size=len(buffer)
+        )
+
+        if zip_id not in self._cached_ranges:
+            self._cached_ranges[zip_id] = []
+
+        self._cached_ranges[zip_id].append(cached_range)
+        self._current_size += len(buffer)
+
+        # Evict if over limit
+        if self._current_size > self._max_size:
+            self._evict_lru()
+
+    def _get_cached_buffer(self, zip_id: str, offset: int, length: int) -> bytes | None:
+        """
+        Try to get a buffer from cache that fully contains the requested range.
+        Updates last_access time if found.
+        """
+        if zip_id not in self._cached_ranges:
+            return None
+
+        current_time = time.time()
+        for cached_range in self._cached_ranges[zip_id]:
+            if cached_range.contains(offset, length):
+                # Update access time
+                cached_range.last_access = current_time
+                # Extract the requested portion
+                start_in_cache = offset - cached_range.offset
+                return cached_range.buffer[start_in_cache : start_in_cache + length]
+
+        return None
+
+    def _compute_missing_ranges(
+        self, zip_id: str, requested_ranges: list[tuple[int, int]]
+    ) -> list[tuple[int, int]]:
+        """
+        Compute which byte ranges need to be read from disk.
+
+        For each requested range, subtract any overlapping cached ranges
+        to find the missing parts.
+        """
+        if zip_id not in self._cached_ranges:
+            return requested_ranges
+
+        missing_ranges: list[tuple[int, int]] = []
+
+        for req_offset, req_length in requested_ranges:
+            # Start with the full requested range
+            uncovered: list[tuple[int, int]] = [(req_offset, req_length)]
+
+            # Subtract each overlapping cached range
+            for cached_range in self._cached_ranges[zip_id]:
+                overlap = cached_range.get_overlap(req_offset, req_length)
+                if overlap is None:
+                    continue
+
+                overlap_offset, overlap_length = overlap
+                overlap_end = overlap_offset + overlap_length
+
+                # Update last access time for this cached range
+                cached_range.last_access = time.time()
+
+                # Split uncovered ranges around the overlap
+                new_uncovered: list[tuple[int, int]] = []
+                for unc_offset, unc_length in uncovered:
+                    unc_end = unc_offset + unc_length
+
+                    # Part before overlap
+                    if unc_offset < overlap_offset:
+                        new_uncovered.append(
+                            (unc_offset, min(overlap_offset, unc_end) - unc_offset)
+                        )
+
+                    # Part after overlap
+                    if unc_end > overlap_end:
+                        new_uncovered.append(
+                            (
+                                max(overlap_end, unc_offset),
+                                unc_end - max(overlap_end, unc_offset),
+                            )
+                        )
+
+                uncovered = new_uncovered
+
+            missing_ranges.extend(uncovered)
+
+        return missing_ranges
+
+    def _assemble_from_cache(self, zip_id: str, offset: int, length: int) -> bytes:
+        """
+        Try to assemble the requested buffer from potentially multiple cached ranges.
+        """
+        result = bytearray(length)
+        current_time = time.time()
+
+        for cached_range in self._cached_ranges[zip_id]:
+            overlap = cached_range.get_overlap(offset, length)
+            if overlap is None:
+                continue
+
+            overlap_offset, overlap_length = overlap
+            # Update access time
+            cached_range.last_access = current_time
+
+            # Copy the overlapping portion
+            start_in_result = overlap_offset - offset
+            start_in_cache = overlap_offset - cached_range.offset
+            result[start_in_result : start_in_result + overlap_length] = (
+                cached_range.buffer[start_in_cache : start_in_cache + overlap_length]
+            )
+
+        return bytes(result)
+
+    def get_buffers(
+        self,
+        zip_id: str,
+        requested_ranges: list[tuple[int, int]],
+        zips_reader: ZipsReader,
+    ) -> list[OffsetedBuffer]:
+        """
+        Get buffers covering all requested ranges.
+
+        Uses cached data where available and reads missing ranges from disk.
+        All newly read data is added to the cache.
+
+        Parameters
+        ----------
+        zip_id : str
+            Identifier for the zip file
+        requested_ranges : list[tuple[int, int]]
+            List of (offset, length) tuples to read. Must be sorted by offset.
+        zips_reader : ZipsReader
+            Reader to use for fetching missing data from disk
+
+        Returns
+        -------
+        list[OffsetedBuffer]
+            List of buffers covering all requested ranges, sorted by offset
+        """
+        with self._lock:
+            # Compute what needs to be read from disk
+            missing_ranges = self._compute_missing_ranges(zip_id, requested_ranges)
+
+            # Read missing ranges from disk using batched read
+            if missing_ranges:
+                # TODO: unlock here?
+                read_result = zips_reader.batch_read_at({zip_id: missing_ranges})
+                multi_buffer = read_result[zip_id]
+
+                # Extract and cache individual ranges from the result
+                for offset, length in missing_ranges:
+                    buffer = multi_buffer.read_at(offset, length)
+                    self._add_to_cache(zip_id, offset, buffer)
+
+            # Now assemble the result from cache
+            result_buffers: list[OffsetedBuffer] = []
+            for req_offset, req_length in requested_ranges:
+                # Try to assemble from cache (possibly multiple ranges)
+                cached_buffer = self._assemble_from_cache(
+                    zip_id, req_offset, req_length
+                )
+
+                # Cache the assembled buffer for faster future access
+                if req_offset not in [
+                    cr.offset for cr in self._cached_ranges.get(zip_id, [])
+                ]:
+                    self._add_to_cache(zip_id, req_offset, cached_buffer)
+
+                result_buffers.append(
+                    OffsetedBuffer(buffer=cached_buffer, offset=req_offset)
+                )
+
+        return result_buffers
+
+    def clear(self) -> None:
+        """Clear all cached data."""
+        with self._lock:
+            self._cached_ranges.clear()
+            self._current_size = 0
+
+    def get_stats(self) -> dict[str, int]:
+        """Get cache statistics."""
+        with self._lock:
+            total_ranges = sum(len(ranges) for ranges in self._cached_ranges.values())
+            return {
+                "current_size_bytes": self._current_size,
+                "max_size_bytes": self._max_size,
+                "num_zip_files": len(self._cached_ranges),
+                "total_cached_ranges": total_ranges,
+            }