simplibs-path 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- simplibs/path/__init__.py +118 -0
- simplibs/path/actions/ChangeSuffix.py +23 -0
- simplibs/path/actions/CollectDirs.py +24 -0
- simplibs/path/actions/CollectFiles.py +25 -0
- simplibs/path/actions/CollectPaths.py +24 -0
- simplibs/path/actions/EnsureDir.py +22 -0
- simplibs/path/actions/EnsureFile.py +19 -0
- simplibs/path/actions/EnsureParentDir.py +20 -0
- simplibs/path/actions/FilterPaths.py +27 -0
- simplibs/path/actions/MakeUniquePath.py +22 -0
- simplibs/path/actions/NormalizeEntryPath.py +20 -0
- simplibs/path/actions/NormalizePath.py +23 -0
- simplibs/path/actions/RebasePath.py +24 -0
- simplibs/path/actions/RelativizePath.py +22 -0
- simplibs/path/actions/__init__.py +68 -0
- simplibs/path/annotations/PathLike.py +43 -0
- simplibs/path/annotations/PathPredicate.py +39 -0
- simplibs/path/annotations/RecursionDepth.py +33 -0
- simplibs/path/annotations/__init__.py +21 -0
- simplibs/path/logic/__init__.py +41 -0
- simplibs/path/logic/checks/__init__.py +29 -0
- simplibs/path/logic/checks/_helpers/__init__.py +21 -0
- simplibs/path/logic/checks/_helpers/_find_length_problem.py +48 -0
- simplibs/path/logic/checks/_helpers/_find_windows_name_problem.py +55 -0
- simplibs/path/logic/checks/_helpers/can_create_inside.py +39 -0
- simplibs/path/logic/checks/_helpers/constants/MAX_NAME_LENGTH.py +18 -0
- simplibs/path/logic/checks/_helpers/constants/WINDOWS_FORBIDDEN_CHARS.py +25 -0
- simplibs/path/logic/checks/_helpers/constants/WINDOWS_RESERVED_NAMES.py +27 -0
- simplibs/path/logic/checks/_helpers/constants/__init__.py +20 -0
- simplibs/path/logic/checks/_helpers/find_filename_problem.py +75 -0
- simplibs/path/logic/checks/_helpers/to_path.py +77 -0
- simplibs/path/logic/checks/is_empty_dir.py +59 -0
- simplibs/path/logic/checks/is_hidden_path.py +78 -0
- simplibs/path/logic/checks/is_path_readable.py +73 -0
- simplibs/path/logic/checks/is_path_writable.py +90 -0
- simplibs/path/logic/checks/is_valid_filename.py +47 -0
- simplibs/path/logic/checks/is_within.py +88 -0
- simplibs/path/logic/ensure/__init__.py +22 -0
- simplibs/path/logic/ensure/_validations/__init__.py +25 -0
- simplibs/path/logic/ensure/_validations/raise_ensure_dir_error.py +40 -0
- simplibs/path/logic/ensure/_validations/raise_ensure_dir_not_a_dir_error.py +36 -0
- simplibs/path/logic/ensure/_validations/raise_ensure_file_create_error.py +39 -0
- simplibs/path/logic/ensure/_validations/raise_ensure_file_not_a_file_error.py +36 -0
- simplibs/path/logic/ensure/_validations/raise_ensure_file_not_found_error.py +36 -0
- simplibs/path/logic/ensure/ensure_dir.py +83 -0
- simplibs/path/logic/ensure/ensure_file.py +103 -0
- simplibs/path/logic/ensure/ensure_parent_dir.py +58 -0
- simplibs/path/logic/scanners/__init__.py +35 -0
- simplibs/path/logic/scanners/_helpers/ScanEntry.py +36 -0
- simplibs/path/logic/scanners/_helpers/__init__.py +30 -0
- simplibs/path/logic/scanners/_helpers/as_path_predicate.py +38 -0
- simplibs/path/logic/scanners/_helpers/build_ignore_matcher.py +88 -0
- simplibs/path/logic/scanners/_helpers/generate_paths.py +74 -0
- simplibs/path/logic/scanners/_helpers/is_link_like.py +45 -0
- simplibs/path/logic/scanners/_helpers/resolve_scan_root.py +66 -0
- simplibs/path/logic/scanners/_helpers/scan_paths.py +92 -0
- simplibs/path/logic/scanners/_helpers/walk_tree.py +118 -0
- simplibs/path/logic/scanners/_validations/__init__.py +24 -0
- simplibs/path/logic/scanners/_validations/raise_collect_paths_not_a_dir_error.py +34 -0
- simplibs/path/logic/scanners/_validations/raise_collect_paths_not_found_error.py +38 -0
- simplibs/path/logic/scanners/_validations/raise_collect_paths_root_unreadable_error.py +38 -0
- simplibs/path/logic/scanners/_validations/raise_ignore_names_not_names_error.py +36 -0
- simplibs/path/logic/scanners/_validations/raise_nothing_to_collect_error.py +34 -0
- simplibs/path/logic/scanners/collect_dirs.py +62 -0
- simplibs/path/logic/scanners/collect_files.py +64 -0
- simplibs/path/logic/scanners/collect_paths.py +102 -0
- simplibs/path/logic/scanners/filter_paths.py +51 -0
- simplibs/path/logic/scanners/ignore_names/DEFAULT_IGNORE_NAMES.py +36 -0
- simplibs/path/logic/scanners/ignore_names/IGNORE_BUILD.py +48 -0
- simplibs/path/logic/scanners/ignore_names/IGNORE_ENV.py +31 -0
- simplibs/path/logic/scanners/ignore_names/IGNORE_GENERIC.py +30 -0
- simplibs/path/logic/scanners/ignore_names/IGNORE_IDE.py +20 -0
- simplibs/path/logic/scanners/ignore_names/IGNORE_SYSTEM.py +30 -0
- simplibs/path/logic/scanners/ignore_names/IGNORE_VCS.py +24 -0
- simplibs/path/logic/scanners/ignore_names/__init__.py +30 -0
- simplibs/path/logic/scanners/iter_paths.py +91 -0
- simplibs/path/logic/transformers/__init__.py +27 -0
- simplibs/path/logic/transformers/_helpers/__init__.py +17 -0
- simplibs/path/logic/transformers/_helpers/is_valid_suffix.py +37 -0
- simplibs/path/logic/transformers/_helpers/split_name.py +46 -0
- simplibs/path/logic/transformers/_validations/__init__.py +24 -0
- simplibs/path/logic/transformers/_validations/raise_invalid_suffix_error.py +36 -0
- simplibs/path/logic/transformers/_validations/raise_normalize_path_error.py +44 -0
- simplibs/path/logic/transformers/_validations/raise_path_has_no_name_error.py +36 -0
- simplibs/path/logic/transformers/_validations/raise_path_not_within_root_error.py +43 -0
- simplibs/path/logic/transformers/_validations/raise_unique_path_exhausted_error.py +39 -0
- simplibs/path/logic/transformers/change_suffix.py +77 -0
- simplibs/path/logic/transformers/make_unique_path.py +88 -0
- simplibs/path/logic/transformers/normalize_entry_path.py +67 -0
- simplibs/path/logic/transformers/normalize_path.py +85 -0
- simplibs/path/logic/transformers/rebase_path.py +78 -0
- simplibs/path/logic/transformers/relativize_path.py +56 -0
- simplibs/path/rules/__init__.py +66 -0
- simplibs/path/rules/_helpers/PathPartIs.py +152 -0
- simplibs/path/rules/_helpers/__init__.py +21 -0
- simplibs/path/rules/_helpers/build_not_a_path_exception.py +66 -0
- simplibs/path/rules/_helpers/build_path_rule_exception.py +71 -0
- simplibs/path/rules/_helpers/check_path.py +48 -0
- simplibs/path/rules/_validations/__init__.py +24 -0
- simplibs/path/rules/_validations/raise_has_suffix_invalid_suffix_error.py +35 -0
- simplibs/path/rules/_validations/raise_has_suffix_no_suffixes_error.py +31 -0
- simplibs/path/rules/_validations/raise_rule_param_not_bool_error.py +37 -0
- simplibs/path/rules/_validations/raise_rule_param_not_callable_error.py +36 -0
- simplibs/path/rules/_validations/raise_rule_param_not_path_error.py +37 -0
- simplibs/path/rules/filesystem/IsDir.py +70 -0
- simplibs/path/rules/filesystem/IsEmptyDir.py +76 -0
- simplibs/path/rules/filesystem/IsEmptyFile.py +76 -0
- simplibs/path/rules/filesystem/IsFile.py +76 -0
- simplibs/path/rules/filesystem/IsPathReadable.py +77 -0
- simplibs/path/rules/filesystem/IsPathWritable.py +82 -0
- simplibs/path/rules/filesystem/IsSymlink.py +71 -0
- simplibs/path/rules/filesystem/PathExists.py +75 -0
- simplibs/path/rules/filesystem/__init__.py +31 -0
- simplibs/path/rules/lexical/HasSuffix.py +146 -0
- simplibs/path/rules/lexical/IsAbsolutePath.py +74 -0
- simplibs/path/rules/lexical/IsHidden.py +92 -0
- simplibs/path/rules/lexical/IsRelativePath.py +74 -0
- simplibs/path/rules/lexical/IsValidFilename.py +136 -0
- simplibs/path/rules/lexical/IsWithin.py +116 -0
- simplibs/path/rules/lexical/NameIs.py +37 -0
- simplibs/path/rules/lexical/ParentNameIs.py +37 -0
- simplibs/path/rules/lexical/StemIs.py +36 -0
- simplibs/path/rules/lexical/__init__.py +33 -0
- simplibs_path-0.1.0.dist-info/METADATA +420 -0
- simplibs_path-0.1.0.dist-info/RECORD +128 -0
- simplibs_path-0.1.0.dist-info/WHEEL +5 -0
- simplibs_path-0.1.0.dist-info/licenses/LICENSE +21 -0
- simplibs_path-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from simplibs.validate import validate_call
|
|
3
|
+
# Outers
|
|
4
|
+
from ...annotations import PathLike
|
|
5
|
+
from ..transformers import normalize_path
|
|
6
|
+
# Inners
|
|
7
|
+
from .ensure_dir import ensure_dir
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@validate_call
|
|
11
|
+
def ensure_parent_dir(
|
|
12
|
+
path: PathLike,
|
|
13
|
+
) -> Path:
|
|
14
|
+
"""Guarantee that the directory a path lives in exists — without creating the path itself.
|
|
15
|
+
|
|
16
|
+
The step right before writing an output file: the file is not there
|
|
17
|
+
yet, but its folder must be.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
path: The path (usually of a file about to be written).
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
The normalized `Path` itself — not its parent — so the call can be
|
|
24
|
+
chained straight into the write.
|
|
25
|
+
|
|
26
|
+
Raises:
|
|
27
|
+
StateError: If a file occupies the place of a needed directory.
|
|
28
|
+
AccessError: If the parent directory cannot be created.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
# 1. Normalize the target path
|
|
32
|
+
normalized_path = normalize_path(path)
|
|
33
|
+
|
|
34
|
+
# 2. Guarantee its parent directory (and everything above it)
|
|
35
|
+
ensure_dir(normalized_path.parent)
|
|
36
|
+
|
|
37
|
+
# 3. Return the path itself, ready for writing
|
|
38
|
+
return normalized_path
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
_DESIGN_NOTES = """
|
|
42
|
+
# ensure_parent_dir — the Step Before Writing
|
|
43
|
+
|
|
44
|
+
## Purpose
|
|
45
|
+
Writing `out/pkg/a.md` fails when `out/pkg/` is missing, and creating the
|
|
46
|
+
*file* up front (what `ensure_file` does) is wrong when the writer wants to
|
|
47
|
+
create it itself — with its own content, mode or encoding. This function
|
|
48
|
+
prepares only the folder.
|
|
49
|
+
|
|
50
|
+
## Returns the Path, Not the Parent
|
|
51
|
+
In a pipeline the value flowing on is the file's path:
|
|
52
|
+
`RebasePath(...) >> EnsureParentDir() >> write_step`. Returning the parent
|
|
53
|
+
would force every caller to rebuild the path they started with.
|
|
54
|
+
|
|
55
|
+
## Pure Composition
|
|
56
|
+
No logic of its own: `normalize_path` and `ensure_dir` do the work, so the
|
|
57
|
+
two stay the single source of truth for what "normalized" and "exists" mean.
|
|
58
|
+
"""
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from .collect_dirs import collect_dirs
|
|
2
|
+
from .collect_files import collect_files
|
|
3
|
+
from .collect_paths import collect_paths
|
|
4
|
+
from .filter_paths import filter_paths
|
|
5
|
+
from .iter_paths import iter_paths
|
|
6
|
+
from .ignore_names import (
|
|
7
|
+
DEFAULT_IGNORE_NAMES,
|
|
8
|
+
IGNORE_BUILD,
|
|
9
|
+
IGNORE_ENV,
|
|
10
|
+
IGNORE_GENERIC,
|
|
11
|
+
IGNORE_IDE,
|
|
12
|
+
IGNORE_SYSTEM,
|
|
13
|
+
IGNORE_VCS,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
_DESIGN_NOTES = """
|
|
18
|
+
# Path Scanners Sub-Package
|
|
19
|
+
|
|
20
|
+
## Purpose
|
|
21
|
+
Everything that looks at a directory tree: scanning it for files and
|
|
22
|
+
directories, and filtering lists of paths. Nothing here changes the disk.
|
|
23
|
+
|
|
24
|
+
## Internal Components Registry
|
|
25
|
+
|
|
26
|
+
| Component | Type | Description |
|
|
27
|
+
| :--------------------- | :------- | :------------------------------------------------------------------- |
|
|
28
|
+
| `collect_dirs` | Function | Sorted list of the directories in a tree. |
|
|
29
|
+
| `collect_files` | Function | Sorted list of the files in a tree. |
|
|
30
|
+
| `collect_paths` | Function | Sorted list of files and/or directories in a tree. |
|
|
31
|
+
| `filter_paths` | Function | Keeps the paths of a list that satisfy a Rule or callable. |
|
|
32
|
+
| `iter_paths` | Function | Lazy, unsorted iterator over the paths of a tree. |
|
|
33
|
+
| `DEFAULT_IGNORE_NAMES` | Constant | The names every scan skips by default. |
|
|
34
|
+
| `IGNORE_*` | Constant | The named groups it is built from (`IGNORE_GENERIC` is opt-in). |
|
|
35
|
+
"""
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import NamedTuple
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class ScanEntry(NamedTuple):
|
|
6
|
+
"""One entry found by `walk_tree`: its path and what kind of entry it is.
|
|
7
|
+
|
|
8
|
+
Attributes:
|
|
9
|
+
path: The entry's full path (the scan root joined with its relative part).
|
|
10
|
+
is_file: True for a regular file, or a symlink to one.
|
|
11
|
+
is_dir: True for a directory, or a symlink to one.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
path: Path
|
|
15
|
+
is_file: bool
|
|
16
|
+
is_dir: bool
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
_DESIGN_NOTES = """
|
|
20
|
+
# ScanEntry — an Entry Plus Its Kind, Learned Once
|
|
21
|
+
|
|
22
|
+
## Purpose
|
|
23
|
+
`os.scandir` hands over the entry type as a by-product of listing (free on
|
|
24
|
+
Windows, cached from the directory read on POSIX). Passing it along lets the
|
|
25
|
+
collectors decide "file or directory?" without asking the disk again — the
|
|
26
|
+
old code made two extra `stat` calls per entry.
|
|
27
|
+
|
|
28
|
+
## Never Neither
|
|
29
|
+
`walk_tree` yields only entries that are a file or a directory, so exactly
|
|
30
|
+
one of the two flags is True. Broken symlinks, FIFOs, sockets and devices
|
|
31
|
+
are not yielded at all.
|
|
32
|
+
|
|
33
|
+
## Why a `NamedTuple`
|
|
34
|
+
Tens of thousands are created per scan; a tuple subclass is as cheap as
|
|
35
|
+
Python offers, is immutable, and still reads like an object (`entry.path`).
|
|
36
|
+
"""
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from .as_path_predicate import as_path_predicate
|
|
2
|
+
from .build_ignore_matcher import build_ignore_matcher
|
|
3
|
+
from .generate_paths import generate_paths
|
|
4
|
+
from .is_link_like import is_link_like
|
|
5
|
+
from .resolve_scan_root import resolve_scan_root
|
|
6
|
+
from .scan_paths import scan_paths
|
|
7
|
+
from .ScanEntry import ScanEntry
|
|
8
|
+
from .walk_tree import walk_tree
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
_DESIGN_NOTES = """
|
|
12
|
+
# Path Scanners Internal Helpers Sub-Package
|
|
13
|
+
|
|
14
|
+
## Purpose
|
|
15
|
+
The building blocks shared by `iter_paths`, `collect_paths`, `collect_files`,
|
|
16
|
+
`collect_dirs` and `filter_paths`.
|
|
17
|
+
|
|
18
|
+
## Internal Components Registry
|
|
19
|
+
|
|
20
|
+
| Component | Type | Description |
|
|
21
|
+
| :--------------------- | :--------- | :--------------------------------------------------------------------- |
|
|
22
|
+
| `as_path_predicate` | Function | Resolves a Rule or callable filter into one fast `(Path) -> bool`. |
|
|
23
|
+
| `build_ignore_matcher` | Function | Compiles ignore names and glob patterns into an `is_ignored(name)`. |
|
|
24
|
+
| `generate_paths` | Function | The type-selection and filter loop over `walk_tree`. |
|
|
25
|
+
| `is_link_like` | Function | Symlink or Windows junction — a directory not to be entered. |
|
|
26
|
+
| `resolve_scan_root` | Function | Normalizes a root and checks it exists, is a directory, is listable. |
|
|
27
|
+
| `scan_paths` | Function | Prepares a scan eagerly and returns the lazy result iterator. |
|
|
28
|
+
| `ScanEntry` | NamedTuple | A found path with its kind (file or directory). |
|
|
29
|
+
| `walk_tree` | Function | The `os.scandir`-based, iterative tree walker. |
|
|
30
|
+
"""
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from simplibs.rules import Rule
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def as_path_predicate(predicate: Callable[[Path], bool]) -> Callable[[Path], bool]:
|
|
7
|
+
"""Turn a Rule or a plain callable into the fastest `(Path) -> bool` call.
|
|
8
|
+
|
|
9
|
+
Args:
|
|
10
|
+
predicate: A `Rule` instance or any callable taking a `Path`.
|
|
11
|
+
|
|
12
|
+
Returns:
|
|
13
|
+
`predicate.is_valid` for a `Rule` (skipping the extra call of
|
|
14
|
+
`Rule.__call__`), the callable itself otherwise.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
# 1. A Rule — use its bound is_valid directly
|
|
18
|
+
if isinstance(predicate, Rule):
|
|
19
|
+
return predicate.is_valid
|
|
20
|
+
|
|
21
|
+
# 2. Any other callable is used as it is
|
|
22
|
+
return predicate
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
_DESIGN_NOTES = """
|
|
26
|
+
# as_path_predicate — Resolve a Filter Once
|
|
27
|
+
|
|
28
|
+
## Purpose
|
|
29
|
+
`collect_paths` and `filter_paths` call their filter once per entry, so the
|
|
30
|
+
choice "Rule or function?" is made a single time, before the loop. A `Rule`
|
|
31
|
+
is callable, but `Rule.__call__` only forwards to `is_valid` — calling
|
|
32
|
+
`is_valid` directly saves one Python call per entry.
|
|
33
|
+
|
|
34
|
+
## Replaces `getattr(rule, "is_valid", rule)`
|
|
35
|
+
The old duck-typing accepted any object that merely *had* an `is_valid`
|
|
36
|
+
attribute. An explicit `isinstance(..., Rule)` states what is meant, and the
|
|
37
|
+
`HasIsValid` protocol it served no longer exists.
|
|
38
|
+
"""
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from collections.abc import Callable, Collection
|
|
3
|
+
from fnmatch import fnmatchcase
|
|
4
|
+
# Inners
|
|
5
|
+
from .._validations import raise_ignore_names_not_names_error
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
_GLOB_CHARS = "*?["
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _ignore_nothing(name: str) -> bool:
|
|
12
|
+
"""The matcher used when no names are ignored."""
|
|
13
|
+
return False
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def build_ignore_matcher(
|
|
17
|
+
ignore_names: Collection[str] | None,
|
|
18
|
+
) -> Callable[[str], bool]:
|
|
19
|
+
"""Build a fast `is_ignored(name)` function from a collection of names and patterns.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
ignore_names: Entry names to skip. A name containing `*`, `?` or `[`
|
|
23
|
+
is a glob pattern (`*.egg-info`); every other name matches
|
|
24
|
+
exactly. `None` or an empty collection ignores nothing.
|
|
25
|
+
|
|
26
|
+
Returns:
|
|
27
|
+
A function answering True for a name that must be skipped. Matching
|
|
28
|
+
follows the platform's case rules (case-insensitive on Windows).
|
|
29
|
+
|
|
30
|
+
Raises:
|
|
31
|
+
ParamError: If `ignore_names` is a single string — it would be
|
|
32
|
+
split into characters.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
# 1. A bare string is iterable — reject it before it becomes a set of characters
|
|
36
|
+
if isinstance(ignore_names, str):
|
|
37
|
+
raise_ignore_names_not_names_error(ignore_names)
|
|
38
|
+
|
|
39
|
+
# 2. Nothing to ignore
|
|
40
|
+
if not ignore_names:
|
|
41
|
+
return _ignore_nothing
|
|
42
|
+
|
|
43
|
+
# 3. Split exact names from glob patterns, case-normalized for the platform
|
|
44
|
+
exact: set[str] = set()
|
|
45
|
+
patterns: list[str] = []
|
|
46
|
+
for name in ignore_names:
|
|
47
|
+
key = os.path.normcase(name)
|
|
48
|
+
if any(char in name for char in _GLOB_CHARS):
|
|
49
|
+
patterns.append(key)
|
|
50
|
+
else:
|
|
51
|
+
exact.add(key)
|
|
52
|
+
|
|
53
|
+
# 4. Exact names only — a single set lookup per entry
|
|
54
|
+
if not patterns:
|
|
55
|
+
return lambda name: os.path.normcase(name) in exact
|
|
56
|
+
|
|
57
|
+
# 5. Exact names first, patterns only for names that did not match
|
|
58
|
+
def is_ignored(name: str) -> bool:
|
|
59
|
+
key = os.path.normcase(name)
|
|
60
|
+
if key in exact:
|
|
61
|
+
return True
|
|
62
|
+
return any(fnmatchcase(key, pattern) for pattern in patterns)
|
|
63
|
+
|
|
64
|
+
return is_ignored
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
_DESIGN_NOTES = """
|
|
68
|
+
# build_ignore_matcher — Names and Patterns, Compiled Once
|
|
69
|
+
|
|
70
|
+
## Purpose
|
|
71
|
+
The walker asks "is this name ignored?" for every entry. This function
|
|
72
|
+
prepares the answer once: a set for exact names (one hash lookup) and a
|
|
73
|
+
short list for glob patterns, consulted only when the set did not match.
|
|
74
|
+
|
|
75
|
+
## Why `normcase` + `fnmatchcase`
|
|
76
|
+
`os.path.normcase` makes names lower-case on Windows and leaves them alone
|
|
77
|
+
elsewhere, and `fnmatchcase` then compares *without* a second, implicit
|
|
78
|
+
normalization — one consistent platform rule for exact names and patterns.
|
|
79
|
+
|
|
80
|
+
## The Single-String Guard Lives Here
|
|
81
|
+
It is the one place every scanner passes `ignore_names` through, so the
|
|
82
|
+
`str` trap is caught once, with the dedicated error card, and cannot be
|
|
83
|
+
forgotten by a new scanner.
|
|
84
|
+
|
|
85
|
+
## Returns a Function, Not a Class
|
|
86
|
+
A closure over two local containers is the lightest thing that holds
|
|
87
|
+
prepared state, and it is trivial to replace in a test.
|
|
88
|
+
"""
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from collections.abc import Callable, Iterator
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
# Inners
|
|
4
|
+
from .walk_tree import walk_tree
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def generate_paths(
|
|
8
|
+
root: Path,
|
|
9
|
+
*,
|
|
10
|
+
max_depth: float | int,
|
|
11
|
+
is_ignored: Callable[[str], bool],
|
|
12
|
+
include: Callable[[Path], bool] | None,
|
|
13
|
+
exclude: Callable[[Path], bool] | None,
|
|
14
|
+
collect_files: bool,
|
|
15
|
+
collect_dirs: bool,
|
|
16
|
+
) -> Iterator[Path]:
|
|
17
|
+
"""Lazily yield the paths of a tree that pass the type and filter checks.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
root: The validated, normalized scan root.
|
|
21
|
+
max_depth: See `walk_tree`.
|
|
22
|
+
is_ignored: See `walk_tree`.
|
|
23
|
+
include: If given, only paths it accepts are yielded.
|
|
24
|
+
exclude: If given, paths it accepts are skipped (checked first).
|
|
25
|
+
collect_files: Whether files are yielded.
|
|
26
|
+
collect_dirs: Whether directories are yielded.
|
|
27
|
+
|
|
28
|
+
Yields:
|
|
29
|
+
Matching `Path`s, in walk order (unsorted).
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
# 1. Walk the tree
|
|
33
|
+
for entry in walk_tree(root, max_depth=max_depth, is_ignored=is_ignored):
|
|
34
|
+
|
|
35
|
+
# 1.1 Apply the file/directory type selection (a directory symlink counts as a directory)
|
|
36
|
+
if entry.is_dir:
|
|
37
|
+
if not collect_dirs:
|
|
38
|
+
continue
|
|
39
|
+
elif not collect_files:
|
|
40
|
+
continue
|
|
41
|
+
|
|
42
|
+
# 1.2 Apply the exclude filter — it wins over include
|
|
43
|
+
if exclude is not None and exclude(entry.path):
|
|
44
|
+
continue
|
|
45
|
+
|
|
46
|
+
# 1.3 Apply the include filter
|
|
47
|
+
if include is not None and not include(entry.path):
|
|
48
|
+
continue
|
|
49
|
+
|
|
50
|
+
# 1.4 Yield the matching path
|
|
51
|
+
yield entry.path
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
_DESIGN_NOTES = """
|
|
55
|
+
# generate_paths — the Selection Loop, Written Once
|
|
56
|
+
|
|
57
|
+
## Purpose
|
|
58
|
+
The common core of `iter_paths` (yields it), `collect_paths` (sorts it) and
|
|
59
|
+
`collect_files`/`collect_dirs` (fix the flags): type selection, then
|
|
60
|
+
`exclude`, then `include`.
|
|
61
|
+
|
|
62
|
+
## Positive Type Selection
|
|
63
|
+
The old loop skipped "files when files are off" and "directories when
|
|
64
|
+
directories are off" separately, so an entry that was neither (a broken
|
|
65
|
+
symlink) slipped through *both* guards and was collected. `walk_tree` now
|
|
66
|
+
yields only files and directories, and the selection below handles both
|
|
67
|
+
cases explicitly.
|
|
68
|
+
|
|
69
|
+
## Filters Do Not Prune
|
|
70
|
+
`exclude` hides a directory *entry* but its contents are still walked and
|
|
71
|
+
judged on their own — only `ignore_names` prunes a subtree. This is
|
|
72
|
+
deliberate (a filter on a directory says nothing about its children) and
|
|
73
|
+
documented on the public functions.
|
|
74
|
+
"""
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import stat
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def is_link_like(entry: os.DirEntry) -> bool:
|
|
6
|
+
"""Check whether a directory entry is a symlink or a Windows junction.
|
|
7
|
+
|
|
8
|
+
Args:
|
|
9
|
+
entry: A directory entry from `os.scandir`.
|
|
10
|
+
|
|
11
|
+
Returns:
|
|
12
|
+
True for a symlink on any platform, and — on Windows — for a
|
|
13
|
+
directory junction (mount point). Entering such a directory could
|
|
14
|
+
loop forever or leave the scanned tree.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
# 1. A symlink, on any platform
|
|
18
|
+
if entry.is_symlink():
|
|
19
|
+
return True
|
|
20
|
+
|
|
21
|
+
# 2. Only Windows has junctions — nothing more to check elsewhere
|
|
22
|
+
if os.name != "nt":
|
|
23
|
+
return False
|
|
24
|
+
|
|
25
|
+
# 3. A junction reports itself as a mount-point reparse tag (cached by scandir, no extra syscall)
|
|
26
|
+
reparse_tag = getattr(entry.stat(follow_symlinks=False), "st_reparse_tag", 0)
|
|
27
|
+
return reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
_DESIGN_NOTES = """
|
|
31
|
+
# is_link_like — Do Not Follow This Directory
|
|
32
|
+
|
|
33
|
+
## Purpose
|
|
34
|
+
`walk_tree` follows real directories only. Python's `is_symlink()` misses
|
|
35
|
+
Windows **junctions** (created by `mklink /J`, common for relocated user
|
|
36
|
+
folders): `is_dir()` is True, `is_symlink()` is False, and a junction
|
|
37
|
+
pointing at a parent directory makes a naive walk run until the path is too
|
|
38
|
+
long.
|
|
39
|
+
|
|
40
|
+
## Why the Reparse Tag, Not "Any Reparse Point"
|
|
41
|
+
Cloud-sync placeholders (OneDrive Files On-Demand) are reparse points too,
|
|
42
|
+
but ordinary folders in every other respect — treating all reparse points
|
|
43
|
+
as links would silently skip real content. Only the mount-point tag
|
|
44
|
+
identifies a junction.
|
|
45
|
+
"""
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
# Outers
|
|
4
|
+
from ...transformers import normalize_path
|
|
5
|
+
# Inners
|
|
6
|
+
from .._validations import (
|
|
7
|
+
raise_collect_paths_not_a_dir_error,
|
|
8
|
+
raise_collect_paths_not_found_error,
|
|
9
|
+
raise_collect_paths_root_unreadable_error,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def resolve_scan_root(root_dir: str | Path) -> Path:
|
|
14
|
+
"""Normalize a scan root and make sure it can actually be scanned.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
root_dir: The directory the caller wants to scan.
|
|
18
|
+
|
|
19
|
+
Returns:
|
|
20
|
+
The normalized absolute directory `Path`.
|
|
21
|
+
|
|
22
|
+
Raises:
|
|
23
|
+
NotFoundError: If the root does not exist.
|
|
24
|
+
StateError: If the root is not a directory.
|
|
25
|
+
AccessError: If the root's entries cannot be listed.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
# 1. Normalize the root
|
|
29
|
+
normalized_root = normalize_path(root_dir)
|
|
30
|
+
|
|
31
|
+
# 2. It must exist
|
|
32
|
+
if not normalized_root.exists():
|
|
33
|
+
raise_collect_paths_not_found_error(normalized_root)
|
|
34
|
+
|
|
35
|
+
# 3. It must be a directory
|
|
36
|
+
if not normalized_root.is_dir():
|
|
37
|
+
raise_collect_paths_not_a_dir_error(normalized_root)
|
|
38
|
+
|
|
39
|
+
# 4. It must be listable — otherwise the walk would silently return nothing
|
|
40
|
+
try:
|
|
41
|
+
with os.scandir(normalized_root):
|
|
42
|
+
pass
|
|
43
|
+
except OSError as caught_exc:
|
|
44
|
+
raise_collect_paths_root_unreadable_error(normalized_root, caught_exc)
|
|
45
|
+
|
|
46
|
+
# 5. Return the checked root
|
|
47
|
+
return normalized_root
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
_DESIGN_NOTES = """
|
|
51
|
+
# resolve_scan_root — Fail Early, Before Any Walking
|
|
52
|
+
|
|
53
|
+
## Purpose
|
|
54
|
+
The root checks that used to sit inline in `collect_paths`, extracted so all
|
|
55
|
+
four scanners share them — and extended with the "unreadable" case.
|
|
56
|
+
|
|
57
|
+
## Why the Extra `scandir`
|
|
58
|
+
`walk_tree` skips what it cannot read. Without this probe, an unreadable root
|
|
59
|
+
would yield an empty list indistinguishable from an empty directory. One
|
|
60
|
+
`scandir` open on the root costs microseconds and turns a silent wrong answer
|
|
61
|
+
into an error.
|
|
62
|
+
|
|
63
|
+
## Eager by Design
|
|
64
|
+
It runs when the scanner is *called*. This matters for `iter_paths`: a bad
|
|
65
|
+
root must raise at the call, not at the first `next()` somewhere far away.
|
|
66
|
+
"""
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import math
|
|
2
|
+
from collections.abc import Callable, Collection, Iterator
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
# Outers
|
|
5
|
+
from ....annotations import PathLike
|
|
6
|
+
from .._validations import raise_nothing_to_collect_error
|
|
7
|
+
# Inners
|
|
8
|
+
from .as_path_predicate import as_path_predicate
|
|
9
|
+
from .build_ignore_matcher import build_ignore_matcher
|
|
10
|
+
from .generate_paths import generate_paths
|
|
11
|
+
from .resolve_scan_root import resolve_scan_root
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def scan_paths(
|
|
15
|
+
root_dir: PathLike,
|
|
16
|
+
*,
|
|
17
|
+
recursive: bool | int,
|
|
18
|
+
include_filter: Callable[[Path], bool] | None,
|
|
19
|
+
exclude_filter: Callable[[Path], bool] | None,
|
|
20
|
+
ignore_names: Collection[str] | None,
|
|
21
|
+
collect_files: bool,
|
|
22
|
+
collect_dirs: bool,
|
|
23
|
+
) -> Iterator[Path]:
|
|
24
|
+
"""Prepare a scan and return the lazy iterator over its results.
|
|
25
|
+
|
|
26
|
+
All checks and preparation happen now, when this is called; only the walk
|
|
27
|
+
itself is deferred to iteration. Shared by every public scanner.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
root_dir: The directory to scan.
|
|
31
|
+
recursive: `True` for unlimited depth, `False` for the root's own
|
|
32
|
+
entries only, or a non-negative `int` for a maximum depth.
|
|
33
|
+
include_filter: Only paths it accepts are kept.
|
|
34
|
+
exclude_filter: Paths it accepts are skipped.
|
|
35
|
+
ignore_names: Entry names or glob patterns to skip entirely.
|
|
36
|
+
collect_files: Whether files are collected.
|
|
37
|
+
collect_dirs: Whether directories are collected.
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
An iterator over the matching `Path`s (unsorted).
|
|
41
|
+
|
|
42
|
+
Raises:
|
|
43
|
+
ParamError: If both collect flags are False, or `ignore_names` is a string.
|
|
44
|
+
NotFoundError: If the root does not exist.
|
|
45
|
+
StateError: If the root is not a directory.
|
|
46
|
+
AccessError: If the root cannot be listed.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
# 1. Both flags off is a caller mistake, not an empty result
|
|
50
|
+
if not collect_files and not collect_dirs:
|
|
51
|
+
raise_nothing_to_collect_error()
|
|
52
|
+
|
|
53
|
+
# 2. Validate and normalize the root now, not at the first iteration
|
|
54
|
+
root = resolve_scan_root(root_dir)
|
|
55
|
+
|
|
56
|
+
# 3. Prepare depth, ignore matcher and filters once, outside the loop
|
|
57
|
+
max_depth = math.inf if recursive is True else int(recursive)
|
|
58
|
+
is_ignored = build_ignore_matcher(ignore_names)
|
|
59
|
+
include = None if include_filter is None else as_path_predicate(include_filter)
|
|
60
|
+
exclude = None if exclude_filter is None else as_path_predicate(exclude_filter)
|
|
61
|
+
|
|
62
|
+
# 4. Hand back the lazy generator
|
|
63
|
+
return generate_paths(
|
|
64
|
+
root,
|
|
65
|
+
max_depth=max_depth,
|
|
66
|
+
is_ignored=is_ignored,
|
|
67
|
+
include=include,
|
|
68
|
+
exclude=exclude,
|
|
69
|
+
collect_files=collect_files,
|
|
70
|
+
collect_dirs=collect_dirs,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
_DESIGN_NOTES = """
|
|
75
|
+
# scan_paths — Prepare Eagerly, Walk Lazily
|
|
76
|
+
|
|
77
|
+
## Purpose
|
|
78
|
+
Every public scanner differs only in its flags and what it does with the
|
|
79
|
+
result (return lazily, sort, ...). The preparation is identical, so it lives
|
|
80
|
+
here once, and each public function validates its own parameters
|
|
81
|
+
(`@validate_call`) and calls this.
|
|
82
|
+
|
|
83
|
+
## Why Not `collect_paths` Calling `iter_paths`
|
|
84
|
+
A chain of public functions would validate the same arguments two or three
|
|
85
|
+
times. Calling one internal, unvalidated function validates each public call
|
|
86
|
+
exactly once.
|
|
87
|
+
|
|
88
|
+
## `recursive` Semantics
|
|
89
|
+
`True` is unlimited, `False` is 0 (only the root's entries), an `int` is the
|
|
90
|
+
maximum depth below the root. `recursive is True` is an identity check on
|
|
91
|
+
purpose: `1 == True`, so equality would turn depth 1 into "unlimited".
|
|
92
|
+
"""
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from collections.abc import Callable, Iterator
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
# Inners
|
|
5
|
+
from .is_link_like import is_link_like
|
|
6
|
+
from .ScanEntry import ScanEntry
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def walk_tree(
|
|
10
|
+
root: Path,
|
|
11
|
+
*,
|
|
12
|
+
max_depth: float | int,
|
|
13
|
+
is_ignored: Callable[[str], bool],
|
|
14
|
+
) -> Iterator[ScanEntry]:
|
|
15
|
+
"""Walk a directory tree, yielding every file and directory it contains.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
root: The directory to walk (must already be validated and normalized).
|
|
19
|
+
max_depth: The deepest directory level whose entries are listed
|
|
20
|
+
(0 = only the entries of `root`; `math.inf` = unlimited).
|
|
21
|
+
is_ignored: Called with each entry's name — a True answer skips the
|
|
22
|
+
entry and, for a directory, everything below it.
|
|
23
|
+
|
|
24
|
+
Yields:
|
|
25
|
+
A `ScanEntry` for every non-ignored file and directory. Symlinks to
|
|
26
|
+
files or directories are yielded (as what they point to) but a
|
|
27
|
+
symlinked or junctioned directory is never entered. Broken symlinks
|
|
28
|
+
and special files (FIFOs, sockets, devices) are not yielded.
|
|
29
|
+
Directories that cannot be read are skipped silently.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
# 1. Directories still to be listed, each with its depth (iterative — no recursion limit)
|
|
33
|
+
pending: list[tuple[str, int]] = [(os.fspath(root), 0)]
|
|
34
|
+
|
|
35
|
+
# 2. List one directory at a time
|
|
36
|
+
while pending:
|
|
37
|
+
directory, depth = pending.pop()
|
|
38
|
+
|
|
39
|
+
# 2.1 A directory that cannot be opened is skipped
|
|
40
|
+
try:
|
|
41
|
+
entries = os.scandir(directory)
|
|
42
|
+
except OSError:
|
|
43
|
+
continue
|
|
44
|
+
|
|
45
|
+
# 2.2 Inspect its entries; a failure while listing ends only this directory
|
|
46
|
+
with entries:
|
|
47
|
+
try:
|
|
48
|
+
for entry in entries:
|
|
49
|
+
|
|
50
|
+
# 2.2.1 Ignored names are skipped entirely — no yield, no descent
|
|
51
|
+
if is_ignored(entry.name):
|
|
52
|
+
continue
|
|
53
|
+
|
|
54
|
+
# 2.2.2 Learn the entry's kind (symlinks are followed for the answer)
|
|
55
|
+
try:
|
|
56
|
+
is_dir = entry.is_dir()
|
|
57
|
+
is_file = not is_dir and entry.is_file()
|
|
58
|
+
enter = is_dir and depth < max_depth and not is_link_like(entry)
|
|
59
|
+
except OSError:
|
|
60
|
+
continue
|
|
61
|
+
|
|
62
|
+
# 2.2.3 Neither a file nor a directory — broken symlink, FIFO, device, ...
|
|
63
|
+
if not (is_dir or is_file):
|
|
64
|
+
continue
|
|
65
|
+
|
|
66
|
+
# 2.2.4 Yield the entry, then queue real directories within the depth limit
|
|
67
|
+
yield ScanEntry(Path(entry.path), is_file, is_dir)
|
|
68
|
+
if enter:
|
|
69
|
+
pending.append((entry.path, depth + 1))
|
|
70
|
+
|
|
71
|
+
except OSError:
|
|
72
|
+
continue
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
_DESIGN_NOTES = """
|
|
76
|
+
# walk_tree — the Shared Walker Behind Every Scanner
|
|
77
|
+
|
|
78
|
+
## Purpose
|
|
79
|
+
One place that knows how to traverse a directory tree safely. `iter_paths`,
|
|
80
|
+
`collect_paths`, `collect_files` and `collect_dirs` all sit on it.
|
|
81
|
+
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
## 1. Rewritten on `os.scandir`, Iterative
|
|
85
|
+
The old version used recursive `Path.iterdir()` generators plus separate
|
|
86
|
+
`is_dir()`/`is_symlink()` calls per entry, and `collect_paths` asked the disk
|
|
87
|
+
twice more. `os.scandir` returns the entry type with the listing itself, so
|
|
88
|
+
each entry costs no extra syscall for its kind (on Windows none at all), and
|
|
89
|
+
the `ScanEntry` carries that answer to the caller. A work list instead of
|
|
90
|
+
recursion means a very deep tree cannot hit Python's recursion limit.
|
|
91
|
+
|
|
92
|
+
## 2. Errors: Skipped Here, Reported at the Root
|
|
93
|
+
A normal tree contains unreadable folders (permissions, sync placeholders);
|
|
94
|
+
dying on the first one would make scanning home directories impractical, so
|
|
95
|
+
`walk_tree` skips them. The old version also lost the *rest* of a directory
|
|
96
|
+
when one entry failed; now only the failing entry is skipped. An unreadable
|
|
97
|
+
**root** is caught before the walk (`resolve_scan_root`) so it cannot masquerade
|
|
98
|
+
as an empty result.
|
|
99
|
+
|
|
100
|
+
## 3. Ignoring Applies to Files and Directories
|
|
101
|
+
`is_ignored(name)` is asked for every entry. An ignored directory is neither
|
|
102
|
+
yielded nor entered, which is what makes `.git` and `node_modules` cheap.
|
|
103
|
+
|
|
104
|
+
## 4. Symlinks and Junctions
|
|
105
|
+
A symlink to a directory is reported (it *is* a directory to the caller) but
|
|
106
|
+
not entered, so link cycles and escapes from the tree are impossible.
|
|
107
|
+
Windows junctions get the same treatment via `is_link_like`.
|
|
108
|
+
|
|
109
|
+
## 5. Pre-Order Within a Directory, No Global Order
|
|
110
|
+
Entries of a directory are yielded in the order the OS lists them, and
|
|
111
|
+
directories are processed last-queued-first. No order is promised —
|
|
112
|
+
`collect_paths` sorts; `iter_paths` documents that it does not.
|
|
113
|
+
|
|
114
|
+
## 6. No `@validate_call`
|
|
115
|
+
Internal helper, called with arguments the scanners have already validated.
|
|
116
|
+
Re-validating a set of names on every directory (as the old version did)
|
|
117
|
+
was pure overhead on the hottest path.
|
|
118
|
+
"""
|