kaparoo-python 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

kaparoo/data/__init__.py

@@ -5,7 +5,9 @@ __all__ = (
     "FileListSequence",
     "SingleFileSequence",
     "SlicedSequence",
+    "TransformedSequence",
     "WindowedSequence",
+    "ZippedSequence",
     "generate_batches",
 )
 
@@ -16,6 +18,8 @@ from kaparoo.data.sequences import (
     FileListSequence,
     SingleFileSequence,
     SlicedSequence,
+    TransformedSequence,
     WindowedSequence,
+    ZippedSequence,
     generate_batches,
 )

kaparoo/data/sequences/__init__.py

@@ -7,7 +7,9 @@ __all__ = (
     "FileListSequence",
     "SingleFileSequence",
     "SlicedSequence",
+    "TransformedSequence",
     "WindowedSequence",
+    "ZippedSequence",
     "generate_batches",
 )
 
@@ -15,7 +17,9 @@ from kaparoo.data.sequences.base import DataSequence
 from kaparoo.data.sequences.composers import (
     ConcatSequence,
     SlicedSequence,
+    TransformedSequence,
     WindowedSequence,
+    ZippedSequence,
 )
 from kaparoo.data.sequences.templates import (
     FileFolderSequence,

kaparoo/data/sequences/composers.py

@@ -1,15 +1,21 @@
 from __future__ import annotations
 
-__all__ = ("ConcatSequence", "SlicedSequence", "WindowedSequence")
+__all__ = (
+    "ConcatSequence",
+    "SlicedSequence",
+    "TransformedSequence",
+    "WindowedSequence",
+    "ZippedSequence",
+)
 
 from abc import abstractmethod
 from bisect import bisect_right
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, cast
 
 from kaparoo.data.sequences.base import DataSequence
 
 if TYPE_CHECKING:
-    from collections.abc import Sequence
+    from collections.abc import Callable, Sequence
 
 
 class SlicedSequence[T, M](DataSequence[T, M]):
@@ -59,6 +65,61 @@ class SlicedSequence[T, M](DataSequence[T, M]):
         return self._source.get_meta(self._indices[index])
 
 
+class TransformedSequence[T_in, M_in, T_out = T_in, M_out = M_in](
+    DataSequence[T_out, M_out]
+):
+    """A view of `source` with `transform` applied lazily to each item.
+
+    `transform` is called on demand in `get_item`; nothing is loaded or
+    converted at construction time. `get_meta` passes through
+    `source.get_meta` unchanged by default -- override it in a subclass
+    when `M_out` differs from `M_in`.
+
+    Type Parameters:
+        T_in: Item type of `source`.
+        M_in: Metadata type of `source`.
+        T_out: Item type after the transform. Defaults to `T_in`.
+        M_out: Metadata type exposed by this view. Defaults to `M_in`.
+            When `M_out != M_in`, override `get_meta` in a subclass;
+            the default passthrough is only safe when `M_out == M_in`.
+
+    Example:
+        >>> # Item-only transform; metadata passes through unchanged.
+        >>> normalized = TransformedSequence(image_folder, normalize)
+
+        >>> # Meta transform via subclassing:
+        >>> class Augmented(TransformedSequence[ndarray, Path, ndarray, AugMeta]):
+        ...     def get_meta(self, index: int) -> AugMeta:
+        ...         return AugMeta(
+        ...             path=self.source.get_meta(index),
+        ...             applied="normalize",
+        ...         )
+    """
+
+    def __init__(
+        self,
+        source: DataSequence[T_in, M_in],
+        transform: Callable[[T_in], T_out],
+    ) -> None:
+        self._source = source
+        self._transform = transform
+
+    @property
+    def source(self) -> DataSequence[T_in, M_in]:
+        """The wrapped sequence."""
+        return self._source
+
+    def __len__(self) -> int:
+        return len(self._source)
+
+    def get_item(self, index: int) -> T_out:
+        return self._transform(self._source.get_item(index))
+
+    def get_meta(self, index: int) -> M_out:
+        # Passthrough by default. Override when M_out != M_in.
+        return cast("M_out", self._source.get_meta(index))
+
+
 class ConcatSequence[T, M](DataSequence[T, M]):
     """The end-to-end concatenation of zero or more `sources`.
 
@@ -112,7 +173,7 @@ class ConcatSequence[T, M](DataSequence[T, M]):
         return source.get_meta(local)
 
 
-class WindowedSequence[T, M_in, M_out](DataSequence[tuple[T, ...], M_out]):
+class WindowedSequence[T, M_in, M_out = M_in](DataSequence[tuple[T, ...], M_out]):
     """An abstract sliding-window view over `source`.
 
     Each item is a tuple of `size` items from `source`, starting at
@@ -130,8 +191,8 @@ class WindowedSequence[T, M_in, M_out](DataSequence[tuple[T, ...], M_out]):
         T: Item type of `source` (also the per-frame type within each
             window).
         M_in: Metadata type of `source` (per-frame metadata).
-        M_out: Metadata type of the window. Determined by the
-            subclass's `get_meta` return.
+        M_out: Metadata type of the window. Defaults to `M_in`.
+            Determined by the subclass's `get_meta` return.
 
     Args:
         source: The sequence to window over.
@@ -219,3 +280,115 @@ class WindowedSequence[T, M_in, M_out](DataSequence[tuple[T, ...], M_out]):
     @abstractmethod
     def get_meta(self, index: int) -> M_out:
         raise NotImplementedError
+
+
+class ZippedSequence[T1, T2, M1 = None, M2 = None](
+    DataSequence[tuple[T1, T2], tuple[M1, M2]]
+):
+    """Element-wise zip of two sequences.
+
+    Item `i` is `(first[i], second[i])` and metadata `i` is
+    `(first.get_meta(i), second.get_meta(i))` -- the "paired image + label"
+    pattern that `ConcatSequence` (end-to-end) cannot express.
+
+    With `strict=True` (the default) the two sequences must have the same
+    length; a mismatch raises `ValueError` at construction. With
+    `strict=False` the view is truncated to the shorter length, like the
+    builtin `zip`. For a different combined-metadata shape, subclass and
+    override `get_meta`.
+
+    Type Parameters:
+        T1: Item type of the first source.
+        T2: Item type of the second source.
+        M1: Metadata type of the first source. Defaults to `None`.
+        M2: Metadata type of the second source. Defaults to `None`.
+
+    Args:
+        first: The first sequence.
+        second: The second sequence.
+        strict: When True (default), require equal lengths and raise on a
+            mismatch. When False, truncate to the shorter length.
+
+    Raises:
+        ValueError: If `strict` is True and the sequences differ in length.
+
+    Example:
+        >>> pairs = ZippedSequence(images, labels)
+        >>> pairs[0]  # (images[0], labels[0])
+        >>> pairs.get_meta(0)  # (images.get_meta(0), labels.get_meta(0))
+    """
+
+    def __init__(
+        self,
+        first: DataSequence[T1, M1],
+        second: DataSequence[T2, M2],
+        *,
+        strict: bool = True,
+    ) -> None:
+        if strict and len(first) != len(second):
+            msg = f"sequences differ in length: {len(first)} != {len(second)}"
+            raise ValueError(msg)
+        self._first = first
+        self._second = second
+        self._length = len(first) if strict else min(len(first), len(second))
+
+    @property
+    def first(self) -> DataSequence[T1, M1]:
+        """The first wrapped sequence."""
+        return self._first
+
+    @property
+    def second(self) -> DataSequence[T2, M2]:
+        """The second wrapped sequence."""
+        return self._second
+
+    def __len__(self) -> int:
+        return self._length
+
+    def _normalize_index(self, index: int) -> int:
+        """Normalize a possibly-negative index and validate range.
+
+        Indices resolve against the zipped length (the shorter source when
+        `strict=False`), so they address the same position in both sources.
+
+        Raises:
+            IndexError: If `index` is outside `[-len(self), len(self))`.
+        """
+        n = self._length
+        original = index
+        if index < 0:
+            index += n
+        if not 0 <= index < n:
+            msg = f"index {original} out of range for length {n}"
+            raise IndexError(msg)
+        return index
+
+    def get_item(self, index: int) -> tuple[T1, T2]:
+        index = self._normalize_index(index)
+        return self._first.get_item(index), self._second.get_item(index)
+
+    def get_items(self, indices: Sequence[int]) -> Sequence[tuple[T1, T2]]:
+        # Normalize, then bulk-delegate so each source's `get_items`
+        # optimization is used.
+        normalized = [self._normalize_index(i) for i in indices]
+        return list(
+            zip(
+                self._first.get_items(normalized),
+                self._second.get_items(normalized),
+                strict=True,
+            )
+        )
+
+    def get_meta(self, index: int) -> tuple[M1, M2]:
+        index = self._normalize_index(index)
+        return self._first.get_meta(index), self._second.get_meta(index)
+
+    def get_metas(self, indices: Sequence[int]) -> Sequence[tuple[M1, M2]]:
+        normalized = [self._normalize_index(i) for i in indices]
+        return list(
+            zip(
+                self._first.get_metas(normalized),
+                self._second.get_metas(normalized),
+                strict=True,
+            )
+        )

kaparoo/data/sequences/templates.py

@@ -14,11 +14,92 @@ if TYPE_CHECKING:
     from kaparoo.filesystem.types import StrPath, StrPaths
 
 
-class FileFolderSequence[T, M = Path](DataSequence[T, M]):
-    """A folder-rooted `DataSequence` whose items live in individual files.
+class FileListSequence[T, M = Path](DataSequence[T, M]):
+    """A `DataSequence` over an explicit, ordered list of files.
+
+    Items live one-per-file; subclasses implement `load_file` and `get_meta`.
+    The files are given directly rather than discovered under a `root`, so
+    they may live in unrelated directories -- or, on Windows, on different
+    drives. (`FileFolderSequence` is the special case where the list is
+    discovered under a single root and stored relative to it.)
+
+    The given order is preserved verbatim and duplicates are kept; sort the
+    input yourself (`sorted(files, key=...)`) if a particular order is
+    needed. Paths are not checked for existence at construction; `load_file`
+    is called lazily on each `get_item`.
+
+    The base exposes:
+
+    - `files: tuple[Path, ...]` — full paths as an immutable snapshot.
+    - `get_file(index) -> Path` — full path of the i-th file.
 
-    The base class handles file discovery, indexing, and root-relative
-    path bookkeeping. Subclasses are responsible for three things:
+    Type Parameters:
+        T: Item type returned by `get_item`.
+        M: Per-item metadata type. Defaults to `Path`; override when the
+            metadata is something else (label, line number, ...).
+
+    Args:
+        files: The file paths to expose, in order.
+
+    Example:
+        >>> from pathlib import Path
+        >>> class BytesList(FileListSequence[bytes]):
+        ...     def get_meta(self, index: int) -> Path:
+        ...         return self.get_file(index)
+        ...
+        ...     def load_file(self, path: Path) -> bytes:
+        ...         return path.read_bytes()
+        >>>
+        >>> data = BytesList(["images/a.png", "/other/b.png"])
+    """
+
+    def __init__(self, files: StrPaths) -> None:
+        self._files = list(stringify_paths(files))
+
+    def __len__(self) -> int:
+        return len(self._files)
+
+    @property
+    def files(self) -> tuple[Path, ...]:
+        """Immutable snapshot of the full file paths, in order.
+
+        Returns a fresh `tuple[Path, ...]` on each access.
+        """
+        return tuple(self.get_file(i) for i in range(len(self)))
+
+    def get_file(self, index: int) -> Path:
+        """Full Path of the file at `index`."""
+        return Path(self._files[index])
+
+    def get_item(self, index: int) -> T:
+        return self.load_file(self.get_file(index))
+
+    @abstractmethod
+    def get_meta(self, index: int) -> M:
+        raise NotImplementedError
+
+    @abstractmethod
+    def load_file(self, path: Path) -> T:
+        """Decode a single file into an item of type `T`.
+
+        Called lazily on each `get_item` -- not at construction time.
+        Subclasses may freely use external libraries (PIL, librosa,
+        cv2, ...) to decode.
+        """
+        raise NotImplementedError
+
+
+class FileFolderSequence[T, M = Path](FileListSequence[T, M]):
+    """A `FileListSequence` whose file list is discovered under a root.
+
+    The special case of `FileListSequence` where every file lives under one
+    base directory. The list is produced by `list_files(root)`, validated to
+    be under `root`, and stored in root-relative form so memory stays low for
+    large datasets and the paths survive a `root` relocation; `get_file`
+    transparently re-prepends `root`. `load_file`, `get_item`, `files`, and
+    `__len__` are inherited unchanged.
+
+    Subclasses are responsible for three things:
 
     - **`list_files(self, root)`** (abstract): return the full `Path`
       of every file to expose, in the desired order. Called once from
@@ -33,16 +114,9 @@ class FileFolderSequence[T, M = Path](DataSequence[T, M]):
       to `Path` and `get_meta(i)` can be the one-liner
       `return self.get_file(i)`.
 
-    The base exposes:
+    The base adds, on top of `FileListSequence`:
 
     - `root: Path` — the base directory.
-    - `files: tuple[Path, ...]` — full paths as an immutable snapshot.
-    - `get_file(index) -> Path` — full path of the i-th file.
-
-    Paths are kept internally in their root-relative form so that
-    memory stays low for large datasets and the sequence survives
-    `root` relocations; the conversion is transparent to subclasses
-    and external callers.
 
     Parameterized subclasses:
         When a subclass needs instance-level options (e.g. `pattern`,
@@ -94,48 +168,20 @@ class FileFolderSequence[T, M = Path](DataSequence[T, M]):
 
     def __init__(self, root: StrPath) -> None:
         self._root = ensure_dir_exists(root)
-        self._files = list(
-            stringify_paths(self.list_files(self._root), after=self._root)
-        )
-
-    def __len__(self) -> int:
-        return len(self._files)
+        # `after=root` makes each path root-relative and raises ValueError if
+        # any file is not under `root`. The base then stores the relative
+        # form; `get_file` re-prepends `root`.
+        super().__init__(stringify_paths(self.list_files(self._root), after=self._root))
 
     @property
     def root(self) -> Path:
         """The base directory the sequence was constructed from."""
         return self._root
 
-    @property
-    def files(self) -> tuple[Path, ...]:
-        """Immutable snapshot of the full file paths this sequence exposes.
-
-        Returns a fresh `tuple[Path, ...]` on each access, in the order
-        established by `list_files`.
-        """
-        return tuple(self.get_file(i) for i in range(len(self)))
-
     def get_file(self, index: int) -> Path:
         """Full Path of the file at `index`."""
         return wrap_path(self._files[index], prepend=self._root)
 
-    def get_item(self, index: int) -> T:
-        return self.load_file(self.get_file(index))
-
-    @abstractmethod
-    def get_meta(self, index: int) -> M:
-        raise NotImplementedError
-
-    @abstractmethod
-    def load_file(self, path: Path) -> T:
-        """Decode a single file into an item of type `T`.
-
-        Called lazily on each `get_item` -- not at construction time.
-        Subclasses may freely use external libraries (PIL, librosa,
-        cv2, ...) to decode.
-        """
-        raise NotImplementedError
-
     @abstractmethod
     def list_files(self, root: Path) -> list[Path]:
         """Return the full Path of every file to expose, in order.
@@ -149,82 +195,6 @@ class FileFolderSequence[T, M = Path](DataSequence[T, M]):
         raise NotImplementedError
 
 
-class FileListSequence[T, M = Path](DataSequence[T, M]):
-    """A `DataSequence` over an explicit, ordered list of files.
-
-    Like `FileFolderSequence`, items live one-per-file and subclasses
-    implement `load_file` and `get_meta`. Unlike it, the files are given
-    directly rather than discovered under a `root`, so they may live in
-    unrelated directories -- or, on Windows, on different drives -- which
-    `FileFolderSequence` cannot represent (it stores paths relative to one
-    root). There is no `list_files`: the input list *is* the listing.
-
-    The given order is preserved verbatim and duplicates are kept; sort the
-    input yourself (`sorted(files, key=...)`) if a particular order is
-    needed. Paths are not checked for existence at construction; `load_file`
-    is called lazily on each `get_item`.
-
-    The base exposes:
-
-    - `files: tuple[Path, ...]` — full paths as an immutable snapshot.
-    - `get_file(index) -> Path` — full path of the i-th file.
-
-    Type Parameters:
-        T: Item type returned by `get_item`.
-        M: Per-item metadata type. Defaults to `Path`; override when the
-            metadata is something else (label, line number, ...).
-
-    Args:
-        files: The file paths to expose, in order.
-
-    Example:
-        >>> from pathlib import Path
-        >>> class BytesList(FileListSequence[bytes]):
-        ...     def get_meta(self, index: int) -> Path:
-        ...         return self.get_file(index)
-        ...
-        ...     def load_file(self, path: Path) -> bytes:
-        ...         return path.read_bytes()
-        >>>
-        >>> data = BytesList(["images/a.png", "/other/b.png"])
-    """
-
-    def __init__(self, files: StrPaths) -> None:
-        self._files = list(stringify_paths(files))
-
-    def __len__(self) -> int:
-        return len(self._files)
-
-    @property
-    def files(self) -> tuple[Path, ...]:
-        """Immutable snapshot of the full file paths, in the given order.
-
-        Returns a fresh `tuple[Path, ...]` on each access.
-        """
-        return tuple(self.get_file(i) for i in range(len(self)))
-
-    def get_file(self, index: int) -> Path:
-        """Full Path of the file at `index`."""
-        return Path(self._files[index])
-
-    def get_item(self, index: int) -> T:
-        return self.load_file(self.get_file(index))
-
-    @abstractmethod
-    def get_meta(self, index: int) -> M:
-        raise NotImplementedError
-
-    @abstractmethod
-    def load_file(self, path: Path) -> T:
-        """Decode a single file into an item of type `T`.
-
-        Called lazily on each `get_item` -- not at construction time.
-        Subclasses may freely use external libraries (PIL, librosa,
-        cv2, ...) to decode.
-        """
-        raise NotImplementedError
-
-
 class SingleFileSequence[T, M = None](DataSequence[T, M]):
     """A `DataSequence` backed by a single file that holds multiple records.
 

{kaparoo_python-0.5.0.dist-info → kaparoo_python-0.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kaparoo-python
-Version: 0.5.0
+Version: 0.6.0
 Summary: Personally common and useful Python features
 Keywords: filesystem,pathlib,paths,utilities
 Author: Jaewoo Park
@@ -67,16 +67,17 @@ hook for custom filter kinds.
 
 `Timer` / `SegmentTimer` context-manager-and-decorator timers (with
 `lap`-split and `measure`-block timings); `Aggregator` for nested,
-pluggable metric aggregation (the batch → epoch → run pattern); plus a
-small family of helpers for working with `Optional[T]` values
-(`replace_if_none`, `unwrap_or_default`, ...).
+pluggable metric aggregation (the batch → epoch → run pattern;
+experimental); plus a small family of helpers for working with
+`Optional[T]` values (`replace_if_none`, `unwrap_or_default`, ...).
 
 ### [`kaparoo.data`](https://github.com/kaparoo/kaparoo-python/tree/main/kaparoo/data)
 
 Building blocks for dataset code: `DataSequence[T, M]` ABC (item +
 metadata), composers (`SlicedSequence`, `ConcatSequence`,
-`WindowedSequence`), file-backed templates (`FileFolderSequence`,
-`SingleFileSequence`), and `generate_batches`.
+`TransformedSequence`, `WindowedSequence`, `ZippedSequence`), file-backed
+templates (`FileFolderSequence`, `FileListSequence`, `SingleFileSequence`),
+and `generate_batches`.
 
 ## 🎯 Quick example
 

{kaparoo_python-0.5.0.dist-info → kaparoo_python-0.6.0.dist-info}/RECORD

@@ -1,9 +1,9 @@
 kaparoo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kaparoo/data/__init__.py,sha256=yNGPKpHhqM-x5qUEGELmnDLkvJEhRGjMk9uS0pAan9A,414
-kaparoo/data/sequences/__init__.py,sha256=SrZlzMTOZsVHD4tVPRgNRv93TRJkxq_LvvizzgvQ9_A,580
+kaparoo/data/__init__.py,sha256=dgbmVOVq_As2ovxRw_JQRdVjQ8d1UTkMFt_A50YfCfM,508
+kaparoo/data/sequences/__init__.py,sha256=3WPq8yDzGvGFXRxu0Dtbp0WPLga32qMDDkL_CZlmFpE,674
 kaparoo/data/sequences/base.py,sha256=m2JcIcT-SLrTzsjFCtgrQ9I5XVpB6PYBigyooEpg4VE,2628
-kaparoo/data/sequences/composers.py,sha256=NjQ-2fxSffE5FjTA1xfZMS65w8N13ZOd_BHj1-NiC1w,7364
-kaparoo/data/sequences/templates.py,sha256=26AR2I2GaxdU9GQxdezAElzr3gKvs1FAuLUcri2QVio,10391
+kaparoo/data/sequences/composers.py,sha256=eaWW8VKyVq4zzJAvIh_LKYt6g-R-HqzzpoV4NKtFeqc,13373
+kaparoo/data/sequences/templates.py,sha256=9a_vM-c9OaF-9qY0ybqRaGkVkywJzrQbNlEHfe39MHU,9572
 kaparoo/data/sequences/utils.py,sha256=oe0qWwnAjsf-9CBUPSlkxkeuQS8kjn1sYhx2eDIwPKI,2808
 kaparoo/filesystem/__init__.py,sha256=uES_e8DYBE0db5z-_E7N2-vSGvi9-uJiSOWnKHdtuPs,1797
 kaparoo/filesystem/directory.py,sha256=Pr15aMl0tz2-VU14rkaFzsV0Zo2oqaevDM0JLW-ZOwk,10421
@@ -28,7 +28,7 @@ kaparoo/utils/__init__.py,sha256=QR-aXuDvxCtOXZLiqHNrKozLlzg_v60UaKI6x2S3YtU,785
 kaparoo/utils/aggregate.py,sha256=8apzZiqLAxoSO51DDDMsOnkrsEaejudXDTc6h3uKRZc,13953
 kaparoo/utils/optional.py,sha256=UgNhGDzl317PE_ESt9hW7yl9MYcdL0nV6Ly0mpqIz0U,4224
 kaparoo/utils/timer.py,sha256=n2RenrYik51v1Dmo9JmZpG3_cPafRDgGMbxdvNoRhgs,17001
-kaparoo_python-0.5.0.dist-info/licenses/LICENSE,sha256=hb6LWYP2rtcoz4V2HpawmblDfHwjwsg9N3cz0c5JQJE,1067
-kaparoo_python-0.5.0.dist-info/WHEEL,sha256=V5-3dKee3Zs8C4JP6swr6zdqriLsOpItBEQxe6_oWpY,81
-kaparoo_python-0.5.0.dist-info/METADATA,sha256=C8tUdR7ywruDpQWtKGObiPRR0PGsKn7-mCeXZPQ6yYI,4252
-kaparoo_python-0.5.0.dist-info/RECORD,,
+kaparoo_python-0.6.0.dist-info/licenses/LICENSE,sha256=hb6LWYP2rtcoz4V2HpawmblDfHwjwsg9N3cz0c5JQJE,1067
+kaparoo_python-0.6.0.dist-info/WHEEL,sha256=V5-3dKee3Zs8C4JP6swr6zdqriLsOpItBEQxe6_oWpY,81
+kaparoo_python-0.6.0.dist-info/METADATA,sha256=I9qqiuRdVQeIh0cHdadR_3UQlyLgtKpTLJ-ycLtYWjQ,4327
+kaparoo_python-0.6.0.dist-info/RECORD,,