kaparoo-python 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

kaparoo/data/__init__.py

@@ -0,0 +1,19 @@
+__all__ = (
+    "ConcatSequence",
+    "DataSequence",
+    "FileFolderSequence",
+    "SingleFileSequence",
+    "SlicedSequence",
+    "WindowedSequence",
+    "generate_batches",
+)
+
+from kaparoo.data.sequences import (
+    ConcatSequence,
+    DataSequence,
+    FileFolderSequence,
+    SingleFileSequence,
+    SlicedSequence,
+    WindowedSequence,
+    generate_batches,
+)

kaparoo/data/sequences/__init__.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+__all__ = (
+    "ConcatSequence",
+    "DataSequence",
+    "FileFolderSequence",
+    "SingleFileSequence",
+    "SlicedSequence",
+    "WindowedSequence",
+    "generate_batches",
+)
+
+from kaparoo.data.sequences.base import DataSequence
+from kaparoo.data.sequences.composers import (
+    ConcatSequence,
+    SlicedSequence,
+    WindowedSequence,
+)
+from kaparoo.data.sequences.templates import (
+    FileFolderSequence,
+    SingleFileSequence,
+)
+from kaparoo.data.sequences.utils import generate_batches

kaparoo/data/sequences/base.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+__all__ = ("DataSequence",)
+
+from abc import abstractmethod
+from collections.abc import Sequence
+from typing import overload
+
+
+class DataSequence[T, M = None](Sequence[T]):
+    """An ordered, lazily-loaded, read-only sequence with per-item metadata.
+
+    Subclasses implement `get_item` (and `__len__`) to fetch a single
+    item by index. Sequence operations (`ds[i]`, `ds[i:j]`, `for x in
+    ds`, `x in ds`, `reversed(ds)`, ...) come from the inherited
+    `collections.abc.Sequence` protocol; only `get_item` need be
+    overridden, and `get_items` may be overridden for batch-fetch
+    optimization.
+
+    The second type parameter `M` carries per-item metadata (labels,
+    source paths, timestamps, ...). Subclasses implement `get_meta`.
+    When the data has no metadata, parameterize as `DataSequence[T]`
+    (so `M` defaults to `None`) and let `get_meta` simply return
+    `None`.
+
+    Type Parameters:
+        T: Element type. `ds[i]` and `get_item(i)` return `T`.
+        M: Per-item metadata type. `get_meta(i)` returns `M`. Defaults
+            to `None` -- meaning "no metadata", in which case
+            subclasses still implement `get_meta` but as a no-op.
+    """
+
+    @abstractmethod
+    def __len__(self) -> int:
+        raise NotImplementedError
+
+    # --- item access -------------------------------------------------------
+
+    @overload
+    def __getitem__(self, index: int, /) -> T: ...
+
+    @overload
+    def __getitem__(self, index: slice, /) -> Sequence[T]: ...
+
+    def __getitem__(self, index: int | slice, /) -> T | Sequence[T]:
+        if isinstance(index, slice):
+            start, stop, step = index.indices(len(self))
+            return self.get_items(range(start, stop, step))
+        return self.get_item(index)
+
+    @abstractmethod
+    def get_item(self, index: int) -> T:
+        raise NotImplementedError
+
+    def get_items(self, indices: Sequence[int]) -> Sequence[T]:
+        return [self.get_item(index) for index in indices]
+
+    # --- metadata access ---------------------------------------------------
+
+    @abstractmethod
+    def get_meta(self, index: int) -> M:
+        raise NotImplementedError
+
+    def get_metas(self, indices: Sequence[int]) -> Sequence[M]:
+        return [self.get_meta(index) for index in indices]
+
+    # --- combined item + metadata ------------------------------------------
+
+    def get_pair(self, index: int) -> tuple[T, M]:
+        return self.get_item(index), self.get_meta(index)
+
+    def get_pairs(self, indices: Sequence[int]) -> Sequence[tuple[T, M]]:
+        return [self.get_pair(index) for index in indices]

kaparoo/data/sequences/composers.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+__all__ = ("ConcatSequence", "SlicedSequence", "WindowedSequence")
+
+from abc import abstractmethod
+from bisect import bisect_right
+from typing import TYPE_CHECKING
+
+from kaparoo.data.sequences.base import DataSequence
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+
+class SlicedSequence[T, M](DataSequence[T, M]):
+    """A view of `source` exposing only items at the given `indices`.
+
+    `indices` is materialized as a tuple at construction time so that the
+    view has a stable length and supports O(1) random access. Negative
+    and out-of-range indices delegate to Python's tuple semantics
+    (negative wraps, out-of-range raises `IndexError`).
+
+    `indices` is taken as-is: duplicates are allowed (the same source
+    item is yielded multiple times) and order is preserved (no sorting).
+    Bounds against `source` are not validated at construction; an
+    out-of-range entry surfaces only when that position is accessed.
+
+    Example:
+        >>> sliced = SlicedSequence(full_dataset, [3, 7, 11])
+        >>> sliced[0]  # == full_dataset[3]
+        >>> sliced[1]  # == full_dataset[7]
+    """
+
+    def __init__(
+        self,
+        source: DataSequence[T, M],
+        indices: Sequence[int],
+    ) -> None:
+        self._source = source
+        self._indices = tuple(indices)
+
+    @property
+    def source(self) -> DataSequence[T, M]:
+        """The wrapped sequence."""
+        return self._source
+
+    @property
+    def indices(self) -> tuple[int, ...]:
+        """The index map into `source`, frozen at construction."""
+        return self._indices
+
+    def __len__(self) -> int:
+        return len(self._indices)
+
+    def get_item(self, index: int) -> T:
+        return self._source.get_item(self._indices[index])
+
+    def get_meta(self, index: int) -> M:
+        return self._source.get_meta(self._indices[index])
+
+
+class ConcatSequence[T, M](DataSequence[T, M]):
+    """The end-to-end concatenation of zero or more `sources`.
+
+    Indexing maps to `(source, local_index)` via a precomputed cumulative
+    length array and `bisect`, so a lookup is O(log N) in the number of
+    sources. Negative indices are normalized; out-of-range indices raise
+    `IndexError`.
+
+    Example:
+        >>> combined = ConcatSequence(train_a, train_b, train_c)
+        >>> len(combined)  # == len(train_a) + len(train_b) + len(train_c)
+    """
+
+    def __init__(self, *sources: DataSequence[T, M]) -> None:
+        self._sources = sources
+        cumulative = [0]
+        for s in sources:
+            cumulative.append(cumulative[-1] + len(s))
+        self._cumulative = tuple(cumulative)
+
+    @property
+    def sources(self) -> tuple[DataSequence[T, M], ...]:
+        """The wrapped sequences, in the order they were passed in."""
+        return self._sources
+
+    def __len__(self) -> int:
+        return self._cumulative[-1]
+
+    def _locate(self, index: int) -> tuple[DataSequence[T, M], int]:
+        """Resolve a logical index to `(source, local_index)`.
+
+        Raises:
+            IndexError: If `index` is outside `[-len(self), len(self))`.
+        """
+        n = self._cumulative[-1]
+        original = index
+        if index < 0:
+            index += n
+        if not 0 <= index < n:
+            msg = f"index {original} out of range for length {n}"
+            raise IndexError(msg)
+        i = bisect_right(self._cumulative, index) - 1
+        return self._sources[i], index - self._cumulative[i]
+
+    def get_item(self, index: int) -> T:
+        source, local = self._locate(index)
+        return source.get_item(local)
+
+    def get_meta(self, index: int) -> M:
+        source, local = self._locate(index)
+        return source.get_meta(local)
+
+
+class WindowedSequence[T, M_in, M_out](DataSequence[tuple[T, ...], M_out]):
+    """An abstract sliding-window view over `source`.
+
+    Each item is a tuple of `size` items from `source`, starting at
+    position `i * step`, with intra-window stride `skip`. Indexed item
+    access (`get_item`) is implemented; **the window's metadata
+    strategy is intentionally left abstract** so the relationship
+    between per-frame `M_in` and window-level `M_out` is decided at
+    subclass-definition time.
+
+    Subclasses use the `source`, `size`, `step`, `skip` properties and
+    should call `_normalize_index` from `get_meta` so negative and
+    out-of-range window indices behave the same way as in `get_item`.
+
+    Type Parameters:
+        T: Item type of `source` (also the per-frame type within each
+            window).
+        M_in: Metadata type of `source` (per-frame metadata).
+        M_out: Metadata type of the window. Determined by the
+            subclass's `get_meta` return.
+
+    Args:
+        source: The sequence to window over.
+        size: Number of items per window. Must be positive.
+        step: Position advance between consecutive windows. Defaults
+            to 1 (overlapping windows by `size - 1`).
+        skip: Intra-window stride. Defaults to 1 (consecutive frames).
+
+    Raises:
+        ValueError: If `size`, `step`, or `skip` is non-positive.
+    """
+
+    def __init__(
+        self,
+        source: DataSequence[T, M_in],
+        size: int,
+        *,
+        step: int = 1,
+        skip: int = 1,
+    ) -> None:
+        if size <= 0 or step <= 0 or skip <= 0:
+            msg = (
+                f"size, step, skip must be positive "
+                f"(got size={size}, step={step}, skip={skip})"
+            )
+            raise ValueError(msg)
+        self._source = source
+        self._size = size
+        self._step = step
+        self._skip = skip
+        # The window spans `(size - 1) * skip + 1` source positions; the
+        # number of complete windows is then `(len(source) - span) // step + 1`.
+        span = (size - 1) * skip + 1
+        self._length = max(0, (len(source) - span) // step + 1)
+
+    @property
+    def source(self) -> DataSequence[T, M_in]:
+        """The wrapped sequence."""
+        return self._source
+
+    @property
+    def size(self) -> int:
+        """Number of items per window."""
+        return self._size
+
+    @property
+    def step(self) -> int:
+        """Position advance between consecutive windows."""
+        return self._step
+
+    @property
+    def skip(self) -> int:
+        """Intra-window stride."""
+        return self._skip
+
+    def __len__(self) -> int:
+        return self._length
+
+    def _normalize_index(self, index: int) -> int:
+        """Normalize a possibly-negative window index and validate range.
+
+        Subclasses should call this from `get_meta` to apply the same
+        negative-index handling and bounds checking that `get_item`
+        performs.
+
+        Raises:
+            IndexError: If `index` is outside `[-len(self), len(self))`.
+        """
+        n = self._length
+        original = index
+        if index < 0:
+            index += n
+        if not 0 <= index < n:
+            msg = f"index {original} out of range for length {n}"
+            raise IndexError(msg)
+        return index
+
+    def get_item(self, index: int) -> tuple[T, ...]:
+        index = self._normalize_index(index)
+        start = index * self._step
+        return tuple(
+            self._source.get_item(start + j * self._skip) for j in range(self._size)
+        )
+
+    @abstractmethod
+    def get_meta(self, index: int) -> M_out:
+        raise NotImplementedError

kaparoo/data/sequences/templates.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+__all__ = ("FileFolderSequence", "SingleFileSequence")
+
+from abc import abstractmethod
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from kaparoo.data.sequences.base import DataSequence
+from kaparoo.filesystem.existence import ensure_dir_exists, ensure_file_exists
+from kaparoo.filesystem.utils import stringify_paths, wrap_path
+
+if TYPE_CHECKING:
+    from kaparoo.filesystem.types import StrPath
+
+
+class FileFolderSequence[T, M = Path](DataSequence[T, M]):
+    """A folder-rooted `DataSequence` whose items live in individual files.
+
+    The base class handles file discovery, indexing, and root-relative
+    path bookkeeping. Subclasses are responsible for three things:
+
+    - **`list_files(self, root)`** (abstract): return the full `Path`
+      of every file to expose, in the desired order. Called once from
+      `__init__` after `root` has been validated. Every returned path
+      must be under `root`; otherwise construction raises `ValueError`.
+      Subclasses can read instance state to parameterize the listing
+      (see "Parameterized subclasses" below).
+    - **`load_file(self, path)`** (abstract): decode a single file.
+      Called lazily on each `get_item`, never at construction time.
+    - **`get_meta(self, index)`** (abstract): produce per-item
+      metadata. When the metadata IS the source path, `M` defaults
+      to `Path` and `get_meta(i)` can be the one-liner
+      `return self.get_file(i)`.
+
+    The base exposes:
+
+    - `root: Path` — the base directory.
+    - `files: tuple[Path, ...]` — full paths as an immutable snapshot.
+    - `get_file(index) -> Path` — full path of the i-th file.
+
+    Paths are kept internally in their root-relative form so that
+    memory stays low for large datasets and the sequence survives
+    `root` relocations; the conversion is transparent to subclasses
+    and external callers.
+
+    Parameterized subclasses:
+        When a subclass needs instance-level options (e.g. `pattern`,
+        `recursive`, label maps), set them on `self` **before** calling
+        `super().__init__(root)` -- the base class invokes
+        `self.list_files(root)` from its own `__init__`, so any state
+        `list_files` will read must already be in place. State that
+        `list_files` does *not* read (caches, label tables, ...) can
+        be set after `super().__init__(root)` as usual.
+
+    Type Parameters:
+        T: Item type returned by `get_item`.
+        M: Per-item metadata type. Defaults to `Path`; override when
+            the metadata is something else (label, line number, ...).
+
+    Args:
+        root: The base directory. Must exist and be a directory.
+
+    Raises:
+        DirectoryNotFoundError: If `root` does not exist.
+        NotADirectoryError: If `root` exists but is not a directory.
+        ValueError: If any path returned by `list_files` is not under
+            `root`.
+
+    Example:
+        >>> from pathlib import Path
+        >>> class GlobFolder(FileFolderSequence[bytes]):
+        ...     def __init__(
+        ...         self, root, *, pattern: str = "*", recursive: bool = False
+        ...     ) -> None:
+        ...         # Set state BEFORE super().__init__() so list_files
+        ...         # can read it.
+        ...         self._pattern = pattern
+        ...         self._recursive = recursive
+        ...         super().__init__(root)
+        ...
+        ...     def list_files(self, root: Path) -> list[Path]:
+        ...         glob_fn = root.rglob if self._recursive else root.glob
+        ...         return sorted(p for p in glob_fn(self._pattern) if p.is_file())
+        ...
+        ...     def get_meta(self, index: int) -> Path:
+        ...         return self.get_file(index)
+        ...
+        ...     def load_file(self, path: Path) -> bytes:
+        ...         return path.read_bytes()
+        >>>
+        >>> folder = GlobFolder("data", pattern="*.png", recursive=True)
+    """
+
+    def __init__(self, root: StrPath) -> None:
+        self._root = ensure_dir_exists(root)
+        self._files = list(
+            stringify_paths(self.list_files(self._root), after=self._root)
+        )
+
+    def __len__(self) -> int:
+        return len(self._files)
+
+    @property
+    def root(self) -> Path:
+        """The base directory the sequence was constructed from."""
+        return self._root
+
+    @property
+    def files(self) -> tuple[Path, ...]:
+        """Immutable snapshot of the full file paths this sequence exposes.
+
+        Returns a fresh `tuple[Path, ...]` on each access, in the order
+        established by `list_files`.
+        """
+        return tuple(self.get_file(i) for i in range(len(self)))
+
+    def get_file(self, index: int) -> Path:
+        """Full Path of the file at `index`."""
+        return wrap_path(self._files[index], prepend=self._root)
+
+    def get_item(self, index: int) -> T:
+        return self.load_file(self.get_file(index))
+
+    @abstractmethod
+    def get_meta(self, index: int) -> M:
+        raise NotImplementedError
+
+    @abstractmethod
+    def load_file(self, path: Path) -> T:
+        """Decode a single file into an item of type `T`.
+
+        Called lazily on each `get_item` -- not at construction time.
+        Subclasses may freely use external libraries (PIL, librosa,
+        cv2, ...) to decode.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def list_files(self, root: Path) -> list[Path]:
+        """Return the full Path of every file to expose, in order.
+
+        Called once from `__init__` after `root` has been validated.
+        Every returned path must be under `root`; construction raises
+        `ValueError` otherwise. May read instance state set before
+        `super().__init__(root)` -- see the class docstring's
+        "Parameterized subclasses" note.
+        """
+        raise NotImplementedError
+
+
+class SingleFileSequence[T, M = None](DataSequence[T, M]):
+    """A `DataSequence` backed by a single file that holds multiple records.
+
+    Thin abstract base for the "one file, many records" pattern
+    (a video file with many frames; a CSV with many rows; a binary
+    blob with fixed-size records; ...). Indexing strategies vary too
+    widely across formats to abstract here -- subclasses are
+    responsible for opening, indexing, and decoding the file.
+
+    `__init__` validates that `path` exists and is a regular file and
+    makes it available via the `path` property. Subclasses typically
+    override `__init__` to additionally open or pre-scan the file,
+    calling `super().__init__(path)` first.
+
+    Args:
+        path: The file to read. Must exist and be a regular file.
+
+    Raises:
+        FileNotFoundError: If `path` does not exist.
+        NotAFileError: If `path` exists but is not a regular file.
+
+    Example:
+        >>> from pathlib import Path
+        >>> class LinesFile(SingleFileSequence[str, int]):
+        ...     def __init__(self, path) -> None:
+        ...         super().__init__(path)
+        ...         self._lines = tuple(self.path.read_text().splitlines())
+        ...
+        ...     def __len__(self) -> int:
+        ...         return len(self._lines)
+        ...
+        ...     def get_item(self, index: int) -> str:
+        ...         return self._lines[index]
+        ...
+        ...     def get_meta(self, index: int) -> int:
+        ...         return index + 1  # 1-based line number
+    """
+
+    def __init__(self, path: StrPath) -> None:
+        self._path = ensure_file_exists(path)
+
+    @property
+    def path(self) -> Path:
+        """The wrapped file's path."""
+        return self._path

kaparoo/data/sequences/utils.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+__all__ = ("generate_batches",)
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator, Sequence
+
+
+def generate_batches[T](
+    sequence: Sequence[T],
+    size: int,
+    *,
+    step: int = 1,
+    skip: int = 1,
+    start: int = 0,
+    stop: int | None = None,
+    drop_last: bool = True,
+) -> Iterator[Sequence[T]]:
+    """Yield sliding windows from `sequence`.
+
+    Each yielded batch is `sequence[head : tail : skip]` where `head`
+    advances by `step` per iteration. With the defaults (`size=3,
+    step=1, skip=1, drop_last=True`), this produces overlapping
+    consecutive-frame windows; pair this with a non-overlapping `step
+    >= size` for a classic non-overlapping batch loader.
+
+    Traversal is constrained to the index range `[start, stop)`.
+    `stop=None` defaults to `len(sequence)`. An empty range (`start ==
+    stop`) yields nothing -- the function returns without error.
+
+    Args:
+        sequence: The sequence to slide windows over.
+        size: Number of items per window. Must be positive.
+        step: Position advance between consecutive windows. Defaults
+            to 1 (overlapping windows by `size - 1`).
+        skip: Intra-window stride. Defaults to 1 (consecutive items).
+        start: Inclusive lower bound on source indices. Defaults to 0.
+            Must satisfy `0 <= start <= stop`.
+        stop: Exclusive upper bound on source indices. Defaults to
+            `len(sequence)`. The partial window (when `drop_last=False`)
+            respects `stop` and never extends past it.
+        drop_last: If False, yield a final partial (possibly shorter
+            than `size`) window when items remain after the last full
+            window. Defaults to True.
+
+    Yields:
+        Sub-sequences of `sequence` obtained by slicing.
+
+    Raises:
+        ValueError: If `size`, `step`, or `skip` is non-positive, or
+            if the range is not `0 <= start <= stop <= len(sequence)`.
+    """
+    if size <= 0 or step <= 0 or skip <= 0:
+        msg = (
+            f"size, step, skip must be positive "
+            f"(got size={size}, step={step}, skip={skip})"
+        )
+        raise ValueError(msg)
+
+    length = len(sequence)
+    stop = stop if stop is not None else length
+    if not 0 <= start <= stop <= length:
+        msg = f"invalid range [{start}, {stop}) for sequence of length {length}"
+        raise ValueError(msg)
+
+    head = start
+    tail = head + (size - 1) * skip + 1
+
+    while tail <= stop:
+        yield sequence[head:tail:skip]
+        head += step
+        tail += step
+
+    # Final partial window must respect `stop` (not `tail`, which has
+    # advanced past `stop` by the time we get here).
+    if not drop_last and head < stop:
+        yield sequence[head:stop:skip]