kaparoo-python 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

kaparoo/data/__init__.py

@@ -2,6 +2,7 @@ __all__ = (
     "ConcatSequence",
     "DataSequence",
     "FileFolderSequence",
+    "FileListSequence",
     "SingleFileSequence",
     "SlicedSequence",
     "WindowedSequence",
@@ -12,6 +13,7 @@ from kaparoo.data.sequences import (
     ConcatSequence,
     DataSequence,
     FileFolderSequence,
+    FileListSequence,
     SingleFileSequence,
     SlicedSequence,
     WindowedSequence,

kaparoo/data/sequences/__init__.py

@@ -4,6 +4,7 @@ __all__ = (
     "ConcatSequence",
     "DataSequence",
     "FileFolderSequence",
+    "FileListSequence",
     "SingleFileSequence",
     "SlicedSequence",
     "WindowedSequence",
@@ -18,6 +19,7 @@ from kaparoo.data.sequences.composers import (
 )
 from kaparoo.data.sequences.templates import (
     FileFolderSequence,
+    FileListSequence,
     SingleFileSequence,
 )
 from kaparoo.data.sequences.utils import generate_batches

kaparoo/data/sequences/templates.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__all__ = ("FileFolderSequence", "SingleFileSequence")
+__all__ = ("FileFolderSequence", "FileListSequence", "SingleFileSequence")
 
 from abc import abstractmethod
 from pathlib import Path
@@ -11,7 +11,7 @@ from kaparoo.filesystem.existence import ensure_dir_exists, ensure_file_exists
 from kaparoo.filesystem.utils import stringify_paths, wrap_path
 
 if TYPE_CHECKING:
-    from kaparoo.filesystem.types import StrPath
+    from kaparoo.filesystem.types import StrPath, StrPaths
 
 
 class FileFolderSequence[T, M = Path](DataSequence[T, M]):
@@ -149,6 +149,82 @@ class FileFolderSequence[T, M = Path](DataSequence[T, M]):
         raise NotImplementedError
 
 
+class FileListSequence[T, M = Path](DataSequence[T, M]):
+    """A `DataSequence` over an explicit, ordered list of files.
+
+    Like `FileFolderSequence`, items live one-per-file and subclasses
+    implement `load_file` and `get_meta`. Unlike it, the files are given
+    directly rather than discovered under a `root`, so they may live in
+    unrelated directories -- or, on Windows, on different drives -- which
+    `FileFolderSequence` cannot represent (it stores paths relative to one
+    root). There is no `list_files`: the input list *is* the listing.
+
+    The given order is preserved verbatim and duplicates are kept; sort the
+    input yourself (`sorted(files, key=...)`) if a particular order is
+    needed. Paths are not checked for existence at construction; `load_file`
+    is called lazily on each `get_item`.
+
+    The base exposes:
+
+    - `files: tuple[Path, ...]` — full paths as an immutable snapshot.
+    - `get_file(index) -> Path` — full path of the i-th file.
+
+    Type Parameters:
+        T: Item type returned by `get_item`.
+        M: Per-item metadata type. Defaults to `Path`; override when the
+            metadata is something else (label, line number, ...).
+
+    Args:
+        files: The file paths to expose, in order.
+
+    Example:
+        >>> from pathlib import Path
+        >>> class BytesList(FileListSequence[bytes]):
+        ...     def get_meta(self, index: int) -> Path:
+        ...         return self.get_file(index)
+        ...
+        ...     def load_file(self, path: Path) -> bytes:
+        ...         return path.read_bytes()
+        >>>
+        >>> data = BytesList(["images/a.png", "/other/b.png"])
+    """
+
+    def __init__(self, files: StrPaths) -> None:
+        self._files = list(stringify_paths(files))
+
+    def __len__(self) -> int:
+        return len(self._files)
+
+    @property
+    def files(self) -> tuple[Path, ...]:
+        """Immutable snapshot of the full file paths, in the given order.
+
+        Returns a fresh `tuple[Path, ...]` on each access.
+        """
+        return tuple(self.get_file(i) for i in range(len(self)))
+
+    def get_file(self, index: int) -> Path:
+        """Full Path of the file at `index`."""
+        return Path(self._files[index])
+
+    def get_item(self, index: int) -> T:
+        return self.load_file(self.get_file(index))
+
+    @abstractmethod
+    def get_meta(self, index: int) -> M:
+        raise NotImplementedError
+
+    @abstractmethod
+    def load_file(self, path: Path) -> T:
+        """Decode a single file into an item of type `T`.
+
+        Called lazily on each `get_item` -- not at construction time.
+        Subclasses may freely use external libraries (PIL, librosa,
+        cv2, ...) to decode.
+        """
+        raise NotImplementedError
+
+
 class SingleFileSequence[T, M = None](DataSequence[T, M]):
     """A `DataSequence` backed by a single file that holds multiple records.
 

kaparoo/filesystem/__init__.py

@@ -1,12 +1,18 @@
 __all__ = (
     "DirectoryNotFoundError",
     "NotAFileError",
+    "StagedDirectory",
+    "StagedFile",
     "dir_empty",
     "dir_empty_unsafe",
     "dir_exists",
+    "dir_not_empty",
+    "dir_not_empty_unsafe",
     "dirs_empty",
     "dirs_empty_unsafe",
     "dirs_exist",
+    "dirs_not_empty",
+    "dirs_not_empty_unsafe",
     "ensure_dir_exists",
     "ensure_dirs_exist",
     "ensure_file_exists",
@@ -22,6 +28,8 @@ __all__ = (
     "make_dirs",
     "path_exists",
     "paths_exist",
+    "reserve_path",
+    "reserve_paths",
     "search_dirs",
     "search_files",
     "search_paths",
@@ -34,8 +42,12 @@ __all__ = (
 from kaparoo.filesystem.directory import (
     dir_empty,
     dir_empty_unsafe,
+    dir_not_empty,
+    dir_not_empty_unsafe,
     dirs_empty,
     dirs_empty_unsafe,
+    dirs_not_empty,
+    dirs_not_empty_unsafe,
     make_dir,
     make_dirs,
 )
@@ -65,7 +77,10 @@ from kaparoo.filesystem.search import (
     search_files,
     search_paths,
 )
+from kaparoo.filesystem.staged import StagedDirectory, StagedFile
 from kaparoo.filesystem.utils import (
+    reserve_path,
+    reserve_paths,
     stringify_path,
     stringify_paths,
     wrap_path,

kaparoo/filesystem/directory.py

@@ -3,13 +3,18 @@ from __future__ import annotations
 __all__ = (
     "dir_empty",
     "dir_empty_unsafe",
+    "dir_not_empty",
+    "dir_not_empty_unsafe",
     "dirs_empty",
     "dirs_empty_unsafe",
+    "dirs_not_empty",
+    "dirs_not_empty_unsafe",
     "make_dir",
     "make_dirs",
 )
 
 import os
+import shutil
 from pathlib import Path
 from typing import TYPE_CHECKING, overload
 
@@ -33,12 +38,27 @@ if TYPE_CHECKING:
 # ========================== #
 
 
+def _ensure_directory_target(path: Path, *, clean: bool) -> None:
+    """Reject a path that cannot serve as a directory target.
+
+    Raises `NotADirectoryError` when `path` exists but is not a directory,
+    or when `clean` is requested on a symlink -- cleaning must operate on a
+    real directory, never through a link (which would otherwise reach the
+    link's target). A symlink to a directory is accepted only when `clean`
+    is False.
+    """
+    if (path.exists() and not path.is_dir()) or (clean and path.is_symlink()):
+        msg = f"not a usable directory target: {path}"
+        raise NotADirectoryError(msg)
+
+
 @overload
 def make_dir(
     path: StrPath,
     *,
     mode: int = 0o777,
     exist_ok: bool = False,
+    clean: bool = False,
     stringify: Literal[False] = False,
 ) -> Path: ...
 
@@ -49,6 +69,7 @@ def make_dir(
     *,
     mode: int = 0o777,
     exist_ok: bool = False,
+    clean: bool = False,
     stringify: Literal[True],
 ) -> str: ...
 
@@ -59,6 +80,7 @@ def make_dir(
     *,
     mode: int = 0o777,
     exist_ok: bool = False,
+    clean: bool = False,
     stringify: bool,
 ) -> Path | str: ...
 
@@ -68,6 +90,7 @@ def make_dir(
     *,
     mode: int = 0o777,
     exist_ok: bool = False,
+    clean: bool = False,
     stringify: bool = False,
 ) -> Path | str:
     """Recursively create a directory.
@@ -77,6 +100,11 @@ def make_dir(
         mode: The mode to use when creating the directory. Defaults to 0o777.
         exist_ok: Whether to suppress OSError if the path already exists.
             Defaults to False.
+        clean: Whether to recreate the directory empty when it already exists,
+            removing its contents first. Only an existing *directory* is wiped;
+            a non-directory -- or a symlink -- still raises. Because the
+            directory is removed and remade, `clean=True` makes `exist_ok`
+            moot. **Destructive.** Defaults to False.
         stringify: Whether to return the path as a string. Defaults to False.
 
     Returns:
@@ -86,14 +114,16 @@ def make_dir(
     Raises:
         ValueError: If `mode` is outside the range 0o1-0o7777
             (not checked on Windows, where the mode is ignored).
-        NotADirectoryError: If the path exists but is not a directory.
-        OSError: If `exist_ok` is False and the path already exists.
+        NotADirectoryError: If the path exists but is not a directory, or
+            `clean` is True and the path is a symlink.
+        OSError: If `exist_ok` is False, `clean` is False, and the path
+            already exists.
     """
     _validate_mode(mode)
     path = Path(path)
-    if path.exists() and not path.is_dir():
-        msg = f"not a directory: {path}"
-        raise NotADirectoryError(msg)
+    _ensure_directory_target(path, clean=clean)
+    if clean and path.is_dir():
+        shutil.rmtree(path)
     path.mkdir(mode=mode, parents=True, exist_ok=exist_ok)
     return stringify_path(path) if stringify else path
 
@@ -105,6 +135,7 @@ def make_dirs(
     root: StrPath | None = None,
     mode: int = 0o777,
     exist_ok: bool = False,
+    clean: bool = False,
     stringify: Literal[False] = False,
 ) -> Sequence[Path]: ...
 
@@ -116,6 +147,7 @@ def make_dirs(
     root: StrPath | None = None,
     mode: int = 0o777,
     exist_ok: bool = False,
+    clean: bool = False,
     stringify: Literal[True],
 ) -> Sequence[str]: ...
 
@@ -127,6 +159,7 @@ def make_dirs(
     root: StrPath | None = None,
     mode: int = 0o777,
     exist_ok: bool = False,
+    clean: bool = False,
     stringify: bool,
 ) -> Sequence[Path] | Sequence[str]: ...
 
@@ -137,6 +170,7 @@ def make_dirs(
     root: StrPath | None = None,
     mode: int = 0o777,
     exist_ok: bool = False,
+    clean: bool = False,
     stringify: bool = False,
 ) -> Sequence[Path] | Sequence[str]:
     """Recursively create directories.
@@ -147,6 +181,11 @@ def make_dirs(
         mode: The mode to use when creating the directories. Defaults to 0o777.
         exist_ok: Whether to suppress OSError if any of the paths already exist.
             Defaults to False.
+        clean: Whether to recreate each directory empty when it already exists,
+            removing its contents first. Only an existing *directory* is wiped;
+            a non-directory -- or a symlink -- still raises. Because the
+            directory is removed and remade, `clean=True` makes `exist_ok`
+            moot. **Destructive.** Defaults to False.
         stringify: Whether to return the paths as strings. Defaults to False.
 
     Returns:
@@ -157,15 +196,29 @@ def make_dirs(
         ValueError: If `mode` is outside the range 0o1-0o7777
             (not checked on Windows, where the mode is ignored).
         DirectoryNotFoundError: If `root` is provided and does not exist.
-        NotADirectoryError: If `root` is provided and is not a directory.
+        NotADirectoryError: If `root` is provided and is not a directory, if
+            any path exists but is not a directory, or `clean` is True and
+            any path is a symlink.
         ValueError: If `root` is provided and any of the paths are absolute.
-        OSError: If `exist_ok` is False and any of the paths already exist.
-        OSError: If any of the paths are not directories.
+        OSError: If `exist_ok` is False, `clean` is False, and any of the
+            paths already exist.
+
+    Note:
+        Every path is validated (the non-directory / symlink checks above)
+        *before* any directory is wiped or created, so a deterministically
+        bad entry -- e.g. a file in the list -- fails without partially
+        cleaning earlier entries. Creation/cleanup is otherwise per-path and
+        not transactional, so a runtime failure (a race, a permission error)
+        partway through can still leave earlier entries created or cleaned.
     """
     _validate_mode(mode)
     paths = _join_root_if_provided(paths, root)
     directories = [Path(p) for p in paths]
     for directory in directories:
+        _ensure_directory_target(directory, clean=clean)
+    for directory in directories:
+        if clean and directory.is_dir():
+            shutil.rmtree(directory)
         directory.mkdir(mode=mode, parents=True, exist_ok=exist_ok)
     return stringify_paths(directories) if stringify else directories
 
@@ -224,3 +277,53 @@ def dirs_empty(paths: StrPaths, *, root: StrPath | None = None) -> bool:
     """
     paths = ensure_dirs_exist(paths, root=root)
     return all(dir_empty_unsafe(p) for p in paths)
+
+
+def dir_not_empty_unsafe(path: StrPath) -> bool:
+    """Check if a directory is not empty without existence checks."""
+    return not dir_empty_unsafe(path)
+
+
+def dirs_not_empty_unsafe(paths: StrPaths, *, root: StrPath | None = None) -> bool:
+    """Check if directories are not empty without existence checks."""
+    if root is not None:
+        paths = [Path(root) / p for p in paths]
+    return all(dir_not_empty_unsafe(p) for p in paths)
+
+
+def dir_not_empty(path: StrPath) -> bool:
+    """Check if a directory is not empty.
+
+    Args:
+        path: The directory path to check.
+
+    Returns:
+        True if the directory is not empty, False otherwise.
+
+    Raises:
+        DirectoryNotFoundError: If the path does not exist.
+        NotADirectoryError: If the path is not a directory.
+    """
+    path = ensure_dir_exists(path)
+    return dir_not_empty_unsafe(path)
+
+
+def dirs_not_empty(paths: StrPaths, *, root: StrPath | None = None) -> bool:
+    """Check if directories are not empty.
+
+    Args:
+        paths: A sequence of directory paths to check.
+        root: The root directory to prepend to each path. Defaults to None.
+
+    Returns:
+        True if all directories are not empty, False otherwise.
+
+    Raises:
+        DirectoryNotFoundError: If `root` is provided and does not exist.
+        DirectoryNotFoundError: If any of the paths do not exist.
+        NotADirectoryError: If `root` is provided and is not a directory.
+        NotADirectoryError: If any of the paths are not directories.
+        ValueError: If `root` is provided and any of the paths are absolute.
+    """
+    paths = ensure_dirs_exist(paths, root=root)
+    return all(dir_not_empty_unsafe(p) for p in paths)

kaparoo/filesystem/search/classes.py

@@ -169,7 +169,7 @@ class Search(ABC):
 
         for dirpath, dirnames, filenames in root.walk():
             child_depth = len(dirpath.parts) - root_depth + 1
-            part = stringify_path(dirpath.relative_to(root))
+            part = stringify_path(dirpath, after=root)
 
             if child_depth >= min_depth and cls._filter_part(part, part_filter):
                 names = cls._select_names(dirnames, filenames)