kaparoo-python 0.3.0__tar.gz → 0.5.0__tar.gz

{kaparoo_python-0.3.0 → kaparoo_python-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kaparoo-python
-Version: 0.3.0
+Version: 0.5.0
 Summary: Personally common and useful Python features
 Keywords: filesystem,pathlib,paths,utilities
 Author: Jaewoo Park
@@ -51,8 +51,10 @@ Each submodule ships its own README with focused examples.
 ### [`kaparoo.filesystem`](https://github.com/kaparoo/kaparoo-python/tree/main/kaparoo/filesystem)
 
 `pathlib`-based filesystem helpers: existence checks (`*_exists`),
-`ensure_*` validators, `make_dir(s)`, `dir_empty(s)`, path
-stringification, and a small exception hierarchy.
+`ensure_*` validators, `make_dir(s)` (with a destructive `clean` reset
+option), `dir_empty(s)`, `reserve_path(s)` guards for not-yet-existing
+destinations, `StagedFile` / `StagedDirectory` for safe (atomic) writes,
+path stringification, and a small exception hierarchy.
 
 ### [`kaparoo.filesystem.search`](https://github.com/kaparoo/kaparoo-python/tree/main/kaparoo/filesystem/search)
 
@@ -63,8 +65,10 @@ hook for custom filter kinds.
 
 ### [`kaparoo.utils`](https://github.com/kaparoo/kaparoo-python/tree/main/kaparoo/utils)
 
-`Timer` / `LapTimer` context-manager-and-decorator timers, plus a small
-family of helpers for working with `Optional[T]` values
+`Timer` / `SegmentTimer` context-manager-and-decorator timers (with
+`lap`-split and `measure`-block timings); `Aggregator` for nested,
+pluggable metric aggregation (the batch → epoch → run pattern); plus a
+small family of helpers for working with `Optional[T]` values
 (`replace_if_none`, `unwrap_or_default`, ...).
 
 ### [`kaparoo.data`](https://github.com/kaparoo/kaparoo-python/tree/main/kaparoo/data)

{kaparoo_python-0.3.0 → kaparoo_python-0.5.0}/README.md

@@ -30,8 +30,10 @@ Each submodule ships its own README with focused examples.
 ### [`kaparoo.filesystem`](https://github.com/kaparoo/kaparoo-python/tree/main/kaparoo/filesystem)
 
 `pathlib`-based filesystem helpers: existence checks (`*_exists`),
-`ensure_*` validators, `make_dir(s)`, `dir_empty(s)`, path
-stringification, and a small exception hierarchy.
+`ensure_*` validators, `make_dir(s)` (with a destructive `clean` reset
+option), `dir_empty(s)`, `reserve_path(s)` guards for not-yet-existing
+destinations, `StagedFile` / `StagedDirectory` for safe (atomic) writes,
+path stringification, and a small exception hierarchy.
 
 ### [`kaparoo.filesystem.search`](https://github.com/kaparoo/kaparoo-python/tree/main/kaparoo/filesystem/search)
 
@@ -42,8 +44,10 @@ hook for custom filter kinds.
 
 ### [`kaparoo.utils`](https://github.com/kaparoo/kaparoo-python/tree/main/kaparoo/utils)
 
-`Timer` / `LapTimer` context-manager-and-decorator timers, plus a small
-family of helpers for working with `Optional[T]` values
+`Timer` / `SegmentTimer` context-manager-and-decorator timers (with
+`lap`-split and `measure`-block timings); `Aggregator` for nested,
+pluggable metric aggregation (the batch → epoch → run pattern); plus a
+small family of helpers for working with `Optional[T]` values
 (`replace_if_none`, `unwrap_or_default`, ...).
 
 ### [`kaparoo.data`](https://github.com/kaparoo/kaparoo-python/tree/main/kaparoo/data)

{kaparoo_python-0.3.0 → kaparoo_python-0.5.0}/kaparoo/data/README.md

@@ -9,7 +9,7 @@ small set of composers, and ready-to-subclass file-backed templates.
 - [`sequences/composers`](./sequences/composers.py) — `SlicedSequence`,
   `ConcatSequence`, `WindowedSequence`
 - [`sequences/templates`](./sequences/templates.py) — `FileFolderSequence`,
-  `SingleFileSequence`
+  `FileListSequence`, `SingleFileSequence`
 - [`sequences/utils`](./sequences/utils.py) — `generate_batches`
 
 All public symbols are re-exported from both `kaparoo.data` and
@@ -158,6 +158,30 @@ class GlobFolder(FileFolderSequence[bytes]):
 folder = GlobFolder("data", pattern="*.png", recursive=True)
 ```
 
+### `FileListSequence`
+
+Same "one file per item" contract as `FileFolderSequence`, but the files
+are given as an explicit list instead of discovered under a `root` — so
+they may live in unrelated directories (or, on Windows, different drives),
+which `FileFolderSequence` cannot represent. There is no `list_files`;
+subclasses implement only `load_file` and `get_meta`. The input order is
+preserved verbatim (duplicates kept) — sort it yourself if needed.
+
+```python
+from pathlib import Path
+from kaparoo.data.sequences import FileListSequence
+
+class BytesList(FileListSequence[bytes]):
+    def get_meta(self, index):
+        return self.get_file(index)
+
+    def load_file(self, path):
+        return path.read_bytes()
+
+# Files from anywhere, in the order given:
+data = BytesList(["images/a.png", "/other/disk/b.png"])
+```
+
 ### `SingleFileSequence`
 
 Thin ABC for the "one file, many records" pattern (a video with many

{kaparoo_python-0.3.0 → kaparoo_python-0.5.0}/kaparoo/data/__init__.py

@@ -2,6 +2,7 @@ __all__ = (
     "ConcatSequence",
     "DataSequence",
     "FileFolderSequence",
+    "FileListSequence",
     "SingleFileSequence",
     "SlicedSequence",
     "WindowedSequence",
@@ -12,6 +13,7 @@ from kaparoo.data.sequences import (
     ConcatSequence,
     DataSequence,
     FileFolderSequence,
+    FileListSequence,
     SingleFileSequence,
     SlicedSequence,
     WindowedSequence,

{kaparoo_python-0.3.0 → kaparoo_python-0.5.0}/kaparoo/data/sequences/__init__.py

@@ -4,6 +4,7 @@ __all__ = (
     "ConcatSequence",
     "DataSequence",
     "FileFolderSequence",
+    "FileListSequence",
     "SingleFileSequence",
     "SlicedSequence",
     "WindowedSequence",
@@ -18,6 +19,7 @@ from kaparoo.data.sequences.composers import (
 )
 from kaparoo.data.sequences.templates import (
     FileFolderSequence,
+    FileListSequence,
     SingleFileSequence,
 )
 from kaparoo.data.sequences.utils import generate_batches

{kaparoo_python-0.3.0 → kaparoo_python-0.5.0}/kaparoo/data/sequences/templates.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__all__ = ("FileFolderSequence", "SingleFileSequence")
+__all__ = ("FileFolderSequence", "FileListSequence", "SingleFileSequence")
 
 from abc import abstractmethod
 from pathlib import Path
@@ -11,7 +11,7 @@ from kaparoo.filesystem.existence import ensure_dir_exists, ensure_file_exists
 from kaparoo.filesystem.utils import stringify_paths, wrap_path
 
 if TYPE_CHECKING:
-    from kaparoo.filesystem.types import StrPath
+    from kaparoo.filesystem.types import StrPath, StrPaths
 
 
 class FileFolderSequence[T, M = Path](DataSequence[T, M]):
@@ -149,6 +149,82 @@ class FileFolderSequence[T, M = Path](DataSequence[T, M]):
         raise NotImplementedError
 
 
+class FileListSequence[T, M = Path](DataSequence[T, M]):
+    """A `DataSequence` over an explicit, ordered list of files.
+
+    Like `FileFolderSequence`, items live one-per-file and subclasses
+    implement `load_file` and `get_meta`. Unlike it, the files are given
+    directly rather than discovered under a `root`, so they may live in
+    unrelated directories -- or, on Windows, on different drives -- which
+    `FileFolderSequence` cannot represent (it stores paths relative to one
+    root). There is no `list_files`: the input list *is* the listing.
+
+    The given order is preserved verbatim and duplicates are kept; sort the
+    input yourself (`sorted(files, key=...)`) if a particular order is
+    needed. Paths are not checked for existence at construction; `load_file`
+    is called lazily on each `get_item`.
+
+    The base exposes:
+
+    - `files: tuple[Path, ...]` — full paths as an immutable snapshot.
+    - `get_file(index) -> Path` — full path of the i-th file.
+
+    Type Parameters:
+        T: Item type returned by `get_item`.
+        M: Per-item metadata type. Defaults to `Path`; override when the
+            metadata is something else (label, line number, ...).
+
+    Args:
+        files: The file paths to expose, in order.
+
+    Example:
+        >>> from pathlib import Path
+        >>> class BytesList(FileListSequence[bytes]):
+        ...     def get_meta(self, index: int) -> Path:
+        ...         return self.get_file(index)
+        ...
+        ...     def load_file(self, path: Path) -> bytes:
+        ...         return path.read_bytes()
+        >>>
+        >>> data = BytesList(["images/a.png", "/other/b.png"])
+    """
+
+    def __init__(self, files: StrPaths) -> None:
+        self._files = list(stringify_paths(files))
+
+    def __len__(self) -> int:
+        return len(self._files)
+
+    @property
+    def files(self) -> tuple[Path, ...]:
+        """Immutable snapshot of the full file paths, in the given order.
+
+        Returns a fresh `tuple[Path, ...]` on each access.
+        """
+        return tuple(self.get_file(i) for i in range(len(self)))
+
+    def get_file(self, index: int) -> Path:
+        """Full Path of the file at `index`."""
+        return Path(self._files[index])
+
+    def get_item(self, index: int) -> T:
+        return self.load_file(self.get_file(index))
+
+    @abstractmethod
+    def get_meta(self, index: int) -> M:
+        raise NotImplementedError
+
+    @abstractmethod
+    def load_file(self, path: Path) -> T:
+        """Decode a single file into an item of type `T`.
+
+        Called lazily on each `get_item` -- not at construction time.
+        Subclasses may freely use external libraries (PIL, librosa,
+        cv2, ...) to decode.
+        """
+        raise NotImplementedError
+
+
 class SingleFileSequence[T, M = None](DataSequence[T, M]):
     """A `DataSequence` backed by a single file that holds multiple records.
 

kaparoo_python-0.5.0/kaparoo/filesystem/README.md

@@ -0,0 +1,230 @@
+# `kaparoo.filesystem`
+
+`pathlib`-based filesystem helpers.
+
+## Modules
+
+- [`existence`](./existence.py) — boolean predicates (`*_exists`) and
+  validating `ensure_*` variants
+- [`directory`](./directory.py) — `make_dir(s)`, `dir_empty(s)` /
+  `dir_not_empty(s)` with validation, plus `_unsafe` variants that skip
+  pre-checks
+- [`utils`](./utils.py) — `stringify_path(s)`, `wrap_path(s)`,
+  `reserve_path(s)`
+- [`staged`](./staged.py) — `StagedFile` / `StagedDirectory`, safe
+  (atomic) writers usable as a context manager or explicitly
+- [`exceptions`](./exceptions.py) — `DirectoryNotFoundError`, `NotAFileError`
+- [`types`](./types.py) — `StrPath`, `StrPaths` type aliases
+- [`search/`](./search/) — composable filesystem search (own README)
+
+All public symbols are re-exported from the top-level `kaparoo.filesystem`
+namespace.
+
+## Existence checks
+
+`*_exists` return a bool; `ensure_*` raise on failure and return the
+(optionally stringified) path.
+
+```python
+from kaparoo.filesystem import (
+    dir_exists, ensure_dir_exists, ensure_files_exist, file_exists,
+)
+
+if file_exists("config.toml"):
+    ...
+
+# Single path: raises FileNotFoundError / NotAFileError / NotADirectoryError
+config = ensure_dir_exists("var/cache", make=True)  # create if missing
+report = ensure_dir_exists("var/cache", make=0o755)  # mode bits: POSIX only
+
+# Bulk with a shared root; each entry is resolved relative to it.
+files = ensure_files_exist(
+    ["a.txt", "b.txt"],
+    root="data",
+)
+```
+
+## Exception hierarchy
+
+`DirectoryNotFoundError` subclasses `FileNotFoundError`, so callers may
+catch the broader type:
+
+```python
+from kaparoo.filesystem import DirectoryNotFoundError, ensure_dir_exists
+
+try:
+    ensure_dir_exists("var/missing")
+except FileNotFoundError:  # catches DirectoryNotFoundError too
+    ...
+```
+
+## Creating and emptying directories
+
+```python
+from kaparoo.filesystem import (
+    dir_empty, dir_not_empty, dirs_empty, make_dir, make_dirs,
+)
+
+cache_dir = make_dir("var/cache", exist_ok=True)
+
+# Start from a clean slate: wipe an existing directory's contents and
+# recreate it empty. Destructive, and only ever wipes a *directory* (a
+# non-directory -- or a symlink -- at the path still raises). `clean=True`
+# makes `exist_ok` moot, since the directory is removed and remade.
+run_dir = make_dir("out/run_42", clean=True)
+
+# Bulk creation with a shared root
+make_dirs(["logs", "tmp"], root="var", exist_ok=True)
+
+# Empty checks (raise if missing or not a directory)
+assert dir_empty(cache_dir)
+assert dirs_empty(["logs", "tmp"], root="var")
+
+# ...and their negations
+(cache_dir / "data.bin").touch()
+assert dir_not_empty(cache_dir)
+```
+
+Each check has a negated counterpart (`dir_not_empty`, `dirs_not_empty`);
+`dirs_not_empty` is True only when *every* directory is non-empty. The
+`_unsafe` variants (`dir_empty_unsafe`, `dir_not_empty_unsafe`,
+`dirs_empty_unsafe`, `dirs_not_empty_unsafe`) skip existence/type
+validation and are intended for hot paths where the caller has already
+validated.
+
+## Path manipulation
+
+`stringify_path(s)` converts to forward-slash strings, optionally
+trimming a leading or trailing portion. `wrap_path(s)` prepends and/or
+appends path components, rejecting absolute inputs where ambiguous.
+
+```python
+from pathlib import Path
+from kaparoo.filesystem import stringify_path, stringify_paths, wrap_path
+
+# "path/to/file.txt" on every platform (including Windows)
+stringify_path(Path("path") / "to" / "file.txt")
+
+# Trim leading or trailing components
+stringify_path("a/b/c", after="a")     # "b/c"
+stringify_path("a/b/c", before="c")    # "a/b"
+
+# Bulk stringify with a shared base.
+stringify_paths(["data/a.txt", "data/b.txt"], after="data")  # ["a.txt", "b.txt"]
+
+# Compose paths without joining manually
+wrap_path("logs", prepend="var", append="server.log")  # var/logs/server.log
+```
+
+## Reserving a destination
+
+`reserve_path` guards a path that should *not* yet exist, so you don't
+clobber something when creating a new file or directory there. It only
+checks (and optionally creates the parent) — it never creates or deletes
+the target itself.
+
+```python
+from kaparoo.filesystem import reserve_path
+
+# Raises FileExistsError if out/run.json exists; otherwise creates the
+# missing parent directory and returns the path ready to write to.
+out = reserve_path("out/run.json", make_parents=True)
+out.write_text("{}")
+
+# `exist_ok` (named as in make_dir / Path.mkdir) is a non-destructive
+# bypass: it suppresses the conflict but deletes nothing, so a later write
+# overwrites in place.
+out = reserve_path("out/run.json", exist_ok=True)
+
+# `reserve_paths` is the bulk form (fail-fast on the first conflict). It
+# takes no `root`; compose with `wrap_paths` to share a base directory.
+from kaparoo.filesystem import reserve_paths, wrap_paths
+a, b = reserve_paths(wrap_paths(["a.bin", "b.bin"], prepend="out"))
+```
+
+For a *directory* destination, `make_dir(..., exist_ok=...)` both guards
+and creates it; for an exclusive *file* create, the stdlib `open(path,
+"x")` raises the same `FileExistsError` directly. Reach for `reserve_path`
+when you want the check (and parent setup) decoupled from the creation.
+
+`reserve_path` is intentionally **non-destructive** — it never removes an
+existing target. To start a directory from a clean slate, use the
+`clean` option on `make_dir` / `make_dirs` (see below), which is the only
+destructive operation here and is named to say so.
+
+## Safe (atomic) writes
+
+`StagedFile` saves a file safely: it stages the content in a temporary file
+in the destination's own directory and moves it into place only on commit.
+A reader never sees a half-written file, and a failed write leaves any
+existing file untouched. It works as a context manager — commit on a clean
+exit, discard on an exception — or explicitly, like a file object.
+
+```python
+from kaparoo.filesystem import StagedFile
+
+# Text (the default), as a context manager: commit on success, discard
+# on error.
+with StagedFile("out/report.json", encoding="utf-8") as f:
+    f.write(json.dumps(data))  # an exception here leaves out/ untouched
+
+# Binary mode, explicitly: write, then commit (or abort to discard).
+f = StagedFile("out/data.bin", binary=True)
+f.write(payload)
+f.commit()  # returns the destination Path; idempotent
+```
+
+The default is text (`StagedFile[str]`) with optional `encoding` / `newline`,
+as with `open`; pass `binary=True` for a binary writer (`StagedFile[bytes]`).
+The type parameter follows the mode, so `write` and `file` are typed `str`
+or `bytes` accordingly.
+
+With `overwrite=False` (the default) an existing destination raises
+`FileExistsError` up front, and the commit creates the file atomically —
+never clobbering a file that appeared meanwhile. With `overwrite=True` the
+destination is replaced in one atomic step, keeping its previous
+permissions. Pass `make_parents=True` to create the destination's parent
+directory if it is missing. An uncommitted writer (an explicit instance
+dropped without `commit()`) discards its staged file on garbage collection,
+so a partial write is never promoted by accident.
+
+The committed file gets the usual umask-based permissions.
+
+`StagedDirectory` is the directory counterpart: you populate its `workdir`
+(a temporary directory in the destination's parent) and it is moved into
+place on commit.
+
+```python
+from kaparoo.filesystem import StagedDirectory
+
+with StagedDirectory("out/dataset", make_parents=True) as d:
+    (d.workdir / "train.json").write_text(payload)
+    (d.workdir / "shards").mkdir()
+# out/dataset appears in one step; an exception would leave it absent
+```
+
+Creating a new directory (`overwrite=False`) is atomic — a single rename.
+Replacing an existing one (`overwrite=True`) is *not* fully atomic: the old
+directory is swapped aside and removed, so there is a brief window where the
+destination is absent and, on a rare failure mid-swap, the previous contents
+remain in a sibling `<name>.old` directory for recovery.
+
+## Platform notes
+
+- **Directory mode bits**: `mode` (on `make_dir` / `make_dirs`) and
+  `make=<int>` (on `ensure_dir_exists` / `ensure_dirs_exist`) are
+  validated against the `0o1`–`0o7777` range and applied to the created
+  directory on **POSIX systems only**. On Windows, mode values are still
+  accepted (so cross-platform code stays clean) but the range check is
+  skipped and the OS ignores the bits — see
+  [`os.mkdir`](https://docs.python.org/3/library/os.html#os.mkdir).
+- **Path separators**: `stringify_path` and `stringify_paths` normalize
+  backslashes to forward slashes on Windows. Functions that return
+  strings via `stringify=True` (`make_dir`, `ensure_dir_exists`,
+  `wrap_path`, ...) inherit this normalization. If you need a native
+  Windows path string, call `str(path)` directly on a `Path`.
+
+## See also
+
+- [`search/`](./search/) for filesystem traversal with filters
+- [`kaparoo.utils`](../utils/) for `Timer` and Optional helpers

{kaparoo_python-0.3.0 → kaparoo_python-0.5.0}/kaparoo/filesystem/__init__.py

@@ -1,12 +1,18 @@
 __all__ = (
     "DirectoryNotFoundError",
     "NotAFileError",
+    "StagedDirectory",
+    "StagedFile",
     "dir_empty",
     "dir_empty_unsafe",
     "dir_exists",
+    "dir_not_empty",
+    "dir_not_empty_unsafe",
     "dirs_empty",
     "dirs_empty_unsafe",
     "dirs_exist",
+    "dirs_not_empty",
+    "dirs_not_empty_unsafe",
     "ensure_dir_exists",
     "ensure_dirs_exist",
     "ensure_file_exists",
@@ -22,6 +28,8 @@ __all__ = (
     "make_dirs",
     "path_exists",
     "paths_exist",
+    "reserve_path",
+    "reserve_paths",
     "search_dirs",
     "search_files",
     "search_paths",
@@ -34,8 +42,12 @@ __all__ = (
 from kaparoo.filesystem.directory import (
     dir_empty,
     dir_empty_unsafe,
+    dir_not_empty,
+    dir_not_empty_unsafe,
     dirs_empty,
     dirs_empty_unsafe,
+    dirs_not_empty,
+    dirs_not_empty_unsafe,
     make_dir,
     make_dirs,
 )
@@ -65,7 +77,10 @@ from kaparoo.filesystem.search import (
     search_files,
     search_paths,
 )
+from kaparoo.filesystem.staged import StagedDirectory, StagedFile
 from kaparoo.filesystem.utils import (
+    reserve_path,
+    reserve_paths,
     stringify_path,
     stringify_paths,
     wrap_path,