fss-cli 0.11.0__py3-none-any.whl

fss_cli/_recursive_rm.py

@@ -0,0 +1,335 @@
+"""Capability-gated, manifest-verified recursive ``rm``."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from functools import partial
+from typing import TYPE_CHECKING, cast
+
+from ._command import (
+    _backend_category,
+    _call,
+    _render_operand_diagnostic,
+)
+from ._concurrent import _gather_bounded, _outcome_value, _run_bounded
+from ._metadata import snapshot_mapping
+from ._path import (
+    _has_dot_segment,
+    _is_same_or_descendant,
+    _lexical_basename,
+    _lexical_join,
+    _lexical_parent,
+    _same_lexical_path,
+)
+
+if TYPE_CHECKING:
+    from fsspec.asyn import AsyncFileSystem
+
+    from ._command import _MappedOperand
+
+_REQUIRED_HOOKS = ("_info", "_ls", "_rm_file", "_rmdir")
+
+
+class _IncompatibleManifestError(Exception):
+    pass
+
+
+class _UnsupportedEntryError(Exception):
+    pass
+
+
+@dataclass(frozen=True)
+class _ManifestEntry:
+    path: str
+    kind: str
+
+
+@dataclass(frozen=True)
+class _Manifest:
+    root: str
+    entries: tuple[_ManifestEntry, ...]
+
+
+@dataclass(frozen=True)
+class _RecursiveRmFailure:
+    operand: _MappedOperand
+    category: str
+    backend_error: Exception | None = None
+    root_missing: bool = False
+
+
+def _has_required_hooks(filesystem: AsyncFileSystem) -> bool:
+    return all(callable(getattr(filesystem, name, None)) for name in _REQUIRED_HOOKS)
+
+
+def _freeze_mapping(value: object) -> Mapping[object, object]:
+    if not isinstance(value, Mapping):
+        raise _IncompatibleManifestError
+    try:
+        return cast("Mapping[object, object]", snapshot_mapping(value))
+    except Exception as error:
+        raise _IncompatibleManifestError from error
+
+
+def _entry_fields(value: object) -> tuple[str, str]:
+    info = _freeze_mapping(value)
+    name = info.get("name")
+    if type(name) is not str:
+        raise _IncompatibleManifestError
+    islink = info.get("islink", False)
+    if type(islink) is not bool:
+        raise _IncompatibleManifestError
+    if islink:
+        raise _UnsupportedEntryError
+    kind = info.get("type")
+    if type(kind) is not str:
+        raise _IncompatibleManifestError
+    return name, kind
+
+
+def _root_entry(path: str, value: object) -> _ManifestEntry:
+    name, kind = _entry_fields(value)
+    if not _same_lexical_path(name, path):
+        raise _IncompatibleManifestError
+    if kind != "directory":
+        raise NotADirectoryError(path)
+    return _ManifestEntry(path, kind)
+
+
+def _listed_entry(parent: str, root: str, value: object) -> _ManifestEntry:
+    name, kind = _entry_fields(value)
+    if kind not in {"file", "directory"}:
+        raise _UnsupportedEntryError
+    if (
+        not name
+        or name.rstrip("/") != name
+        or "\0" in name
+        or "\n" in name
+        or "\r" in name
+        or _has_dot_segment(name)
+        or not _is_same_or_descendant(root, name)
+        or not _same_lexical_path(_lexical_parent(name), parent)
+    ):
+        raise _IncompatibleManifestError
+    return _ManifestEntry(_lexical_join(parent, _lexical_basename(name)), kind)
+
+
+def _frozen_listing(
+    directory: str,
+    root: str,
+    result: object,
+    seen: set[str],
+) -> list[_ManifestEntry]:
+    """Validate one directory's ``_ls(detail=True)`` result into child entries."""
+    if not isinstance(result, list):
+        raise _IncompatibleManifestError
+    frozen: list[_ManifestEntry] = []
+    expected_length = len(result)
+    try:
+        for value in result:
+            entry = _listed_entry(directory, root, value)
+            if entry.path in seen:
+                raise _IncompatibleManifestError  # noqa: TRY301
+            seen.add(entry.path)
+            frozen.append(entry)
+    except (_IncompatibleManifestError, _UnsupportedEntryError):
+        raise
+    except Exception as error:
+        raise _IncompatibleManifestError from error
+    if len(result) != expected_length or len(frozen) != expected_length:
+        raise _IncompatibleManifestError
+    return frozen
+
+
+async def _manifest(
+    filesystem: AsyncFileSystem,
+    root: str,
+    root_info: object,
+) -> _Manifest:
+    """List the tree breadth-first, then freeze it in leaves-first order.
+
+    Every directory of one depth is listed concurrently under the shared
+    bound; a listing failure at any depth fails the plan. Entries are emitted
+    depth-first in path order with each directory after its children.
+    """
+    root_entry = _root_entry(root, root_info)
+    seen = {root}
+    children: dict[str, list[_ManifestEntry]] = {}
+    level = [root_entry]
+    while level:
+        listings = await _gather_bounded(
+            [partial(_call, filesystem, "_ls", directory.path, detail=True) for directory in level]
+        )
+        next_level: list[_ManifestEntry] = []
+        for directory, result in zip(level, listings, strict=True):
+            frozen = _frozen_listing(directory.path, root, result, seen)
+            children[directory.path] = frozen
+            next_level.extend(entry for entry in frozen if entry.kind == "directory")
+        level = next_level
+
+    entries: list[_ManifestEntry] = []
+    stack = [(root_entry, False)]
+    while stack:
+        candidate, visited = stack.pop()
+        if candidate.kind == "file" or visited:
+            entries.append(candidate)
+            continue
+        stack.append((candidate, True))
+        stack.extend(
+            (entry, False)
+            for entry in sorted(
+                children[candidate.path],
+                key=lambda item: item.path,
+                reverse=True,
+            )
+        )
+
+    manifest = _Manifest(root, tuple(entries))
+    _revalidate_manifest(manifest)
+    return manifest
+
+
+def _revalidate_manifest(manifest: _Manifest) -> None:
+    if not manifest.entries or manifest.entries[-1] != _ManifestEntry(manifest.root, "directory"):
+        raise _IncompatibleManifestError
+    seen: set[str] = set()
+    directories = {manifest.root}
+    for entry in manifest.entries:
+        if entry.path in seen or not _is_same_or_descendant(manifest.root, entry.path):
+            raise _IncompatibleManifestError
+        seen.add(entry.path)
+        if entry.kind == "directory":
+            directories.add(entry.path)
+    for entry in manifest.entries[:-1]:
+        if _lexical_parent(entry.path) not in directories:
+            raise _IncompatibleManifestError
+
+
+def _read_failure(
+    operand: _MappedOperand,
+    error: Exception,
+    *,
+    root_missing: bool = False,
+) -> _RecursiveRmFailure:
+    return _RecursiveRmFailure(
+        operand,
+        _backend_category(error),
+        backend_error=error,
+        root_missing=root_missing,
+    )
+
+
+async def _plan(
+    operand: _MappedOperand,
+    filesystem: AsyncFileSystem,
+) -> _Manifest | _RecursiveRmFailure:
+    if not _has_required_hooks(filesystem):
+        return _RecursiveRmFailure(operand, "unsupported operation")
+    root = operand.path.rstrip("/")
+    try:
+        root_info = await _call(filesystem, "_info", root)
+    except Exception as error:  # noqa: BLE001 - classify read boundary.
+        return _read_failure(
+            operand,
+            error,
+            root_missing=isinstance(error, FileNotFoundError),
+        )
+    try:
+        return await _manifest(filesystem, root, root_info)
+    except _UnsupportedEntryError:
+        return _RecursiveRmFailure(operand, "unsupported operation")
+    except _IncompatibleManifestError:
+        return _RecursiveRmFailure(operand, "incompatible result")
+    except Exception as error:  # noqa: BLE001 - classify planning boundary.
+        return _read_failure(operand, error)
+
+
+async def _remove_entry(
+    operand: _MappedOperand,
+    filesystem: AsyncFileSystem,
+    manifest: _Manifest,
+    entry: _ManifestEntry,
+) -> _RecursiveRmFailure | None:
+    """Remove one entry, then prove its absence."""
+    if not _is_same_or_descendant(manifest.root, entry.path):
+        return _RecursiveRmFailure(operand, "incompatible result")
+    operation = "_rm_file" if entry.kind == "file" else "_rmdir"
+    try:
+        await _call(filesystem, operation, entry.path)
+    except Exception as error:  # noqa: BLE001 - mutation may be partial.
+        return _RecursiveRmFailure(
+            operand,
+            "recursive removal incomplete; residue possible",
+            backend_error=error,
+        )
+    try:
+        await _call(filesystem, "_info", entry.path)
+    except FileNotFoundError:
+        return None
+    except Exception as error:  # noqa: BLE001 - absence remains uncertain.
+        return _RecursiveRmFailure(
+            operand,
+            "recursive removal incomplete; residue possible",
+            backend_error=error,
+        )
+    return _RecursiveRmFailure(
+        operand,
+        "recursive removal incomplete; residue possible",
+    )
+
+
+def _depth(manifest: _Manifest, entry: _ManifestEntry) -> int:
+    return entry.path[len(manifest.root) :].count("/")
+
+
+async def _mutate(
+    operand: _MappedOperand,
+    filesystem: AsyncFileSystem,
+    manifest: _Manifest,
+) -> _RecursiveRmFailure | None:
+    """Remove the manifest leaves-first, one depth at a time.
+
+    Every entry of the deepest remaining depth is removed and proven absent
+    concurrently under the shared bound, so a directory is removed only after
+    all of its children are confirmed gone. After the first failure no new
+    removal starts; in-flight ones finish and the first failure in manifest
+    order is reported.
+    """
+    try:
+        _revalidate_manifest(manifest)
+    except _IncompatibleManifestError:
+        return _RecursiveRmFailure(operand, "incompatible result")
+
+    levels: dict[int, list[_ManifestEntry]] = {}
+    for entry in manifest.entries:
+        levels.setdefault(_depth(manifest, entry), []).append(entry)
+    for depth in sorted(levels, reverse=True):
+        outcomes = await _run_bounded(
+            [partial(_remove_entry, operand, filesystem, manifest, entry) for entry in levels[depth]],
+            stop=lambda failure: failure is not None,
+        )
+        for outcome in outcomes:
+            if outcome is None:
+                continue
+            failure = _outcome_value(outcome)
+            if failure is not None:
+                return failure
+    return None
+
+
+async def _remove_recursive(
+    operand: _MappedOperand,
+    filesystem: AsyncFileSystem,
+) -> _RecursiveRmFailure | None:
+    plan = await _plan(operand, filesystem)
+    if isinstance(plan, _RecursiveRmFailure):
+        return plan
+    return await _mutate(operand, filesystem, plan)
+
+
+def _render_recursive_failure(
+    command: str,
+    failure: _RecursiveRmFailure,
+) -> None:
+    _render_operand_diagnostic(command, failure.operand, failure.category)

fss_cli/_results.py

@@ -0,0 +1,115 @@
+"""Validate untrusted fsspec results once, at the hook boundary.
+
+Integers stay exact (``bool`` is not an integer). ``str``, ``list`` and
+``Mapping`` accept subclasses, because backends may return specialised
+strings and lists. A mapping is snapshotted once.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from functools import partial
+from typing import TYPE_CHECKING, cast
+
+from ._command import _call
+from ._concurrent import _gather_bounded
+from ._metadata import snapshot_mapping, valid_display_text
+from ._path import _has_dot_segment, _lexical_components, _lexical_parent, _same_lexical_path
+
+if TYPE_CHECKING:
+    from fsspec.asyn import AsyncFileSystem
+
+
+class _BadResultError(Exception):
+    """A hook result does not have the shape the command can use."""
+
+
+def snapshot_fields(value: object) -> tuple[Mapping[str, object], str, str]:
+    """Snapshot one metadata mapping and return it with its name and type.
+
+    ``str`` subclasses are accepted for both fields. The size is not checked.
+    """
+    if not isinstance(value, Mapping):
+        raise _BadResultError
+    try:
+        raw = snapshot_mapping(cast("Mapping[object, object]", value))
+    except Exception as error:
+        raise _BadResultError from error
+    name = raw.get("name")
+    kind = raw.get("type")
+    if not isinstance(name, str) or not isinstance(kind, str):
+        raise _BadResultError
+    return cast("Mapping[str, object]", raw), name, kind
+
+
+def snapshot_listing(root: str, listing: object) -> dict[tuple[str, ...], Mapping[str, object]]:
+    """Index a complete detail listing, refusing invalid or duplicated paths."""
+    if not isinstance(listing, list):
+        raise _BadResultError
+    expected = len(listing)
+    indexed: dict[tuple[str, ...], Mapping[str, object]] = {}
+    for value in listing:
+        info, name, _kind = snapshot_fields(value)
+        path = name.rstrip("/")
+        identity = _lexical_components(path)
+        if (
+            not name
+            or not valid_display_text(name)
+            or _has_dot_segment(name)
+            or (not _same_lexical_path(path, root) and not _same_lexical_path(_lexical_parent(path), root))
+            or identity in indexed
+        ):
+            raise _BadResultError
+        indexed[identity] = info
+    if len(listing) != expected or len(indexed) != expected:
+        raise _BadResultError
+    return indexed
+
+
+async def _info(filesystem: AsyncFileSystem, path: str, *, missing_ok: bool) -> object | None:
+    try:
+        return await _call(filesystem, "_info", path)
+    except FileNotFoundError:
+        if not missing_ok:
+            raise
+        return None
+
+
+async def info_or_missing(filesystem: AsyncFileSystem, path: str) -> tuple[object | None, Exception | None]:
+    """Read optional metadata while preserving the command's error classification."""
+    try:
+        return await _info(filesystem, path, missing_ok=True), None
+    except Exception as error:  # noqa: BLE001 - metadata acquisition boundary.
+        return None, error
+
+
+async def infos_for_paths(
+    filesystem: AsyncFileSystem, paths: list[str], *, missing_ok: bool
+) -> tuple[dict[str, object | None], Exception | None]:
+    """Use validated parent listings, then bounded direct lookup for every gap."""
+    parents = list(dict.fromkeys(_lexical_parent(path) for path in paths))
+
+    async def listing(parent: str) -> dict[tuple[str, ...], Mapping[str, object]]:
+        try:
+            result = await _call(filesystem, "_ls", parent, detail=True)
+        except (FileNotFoundError, PermissionError, NotImplementedError):
+            return {}
+        return snapshot_listing(parent, result)
+
+    try:
+        listings = dict(
+            zip(parents, await _gather_bounded([partial(listing, parent) for parent in parents]), strict=True)
+        )
+        found: dict[str, object | None] = {}
+        gaps: list[str] = []
+        for path in paths:
+            row = listings[_lexical_parent(path)].get(_lexical_components(path))
+            if row is None or (row["type"] == "file" and row.get("size") is None):
+                gaps.append(path)
+            else:
+                found[path] = row
+        values = await _gather_bounded([partial(_info, filesystem, path, missing_ok=missing_ok) for path in gaps])
+        found.update(zip(gaps, values, strict=True))
+    except Exception as error:  # noqa: BLE001 - command owns diagnostics and mutation policy.
+        return {}, error
+    return found, None

fss_cli/_rm.py

@@ -0,0 +1,149 @@
+"""Typed async execution for ``rm`` profiles."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from ._command import (
+    _binary_stdout,
+    _CommandFailureError,
+    _first_backend_error,
+    _MappedOperand,
+    _render_output_failure,
+    _run_mapped_command,
+    _write_binary,
+)
+from ._recursive_rm import (
+    _RecursiveRmFailure,
+    _remove_recursive,
+    _render_recursive_failure,
+)
+from ._rmdir import _remove_empty_directory, _RmdirFailure
+from ._rmdir import _render_failure as _render_rmdir_failure
+from ._unlink import _confirmed_rm_file, _render_failure, _UnlinkFailure
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from fsspec.asyn import AsyncFileSystem
+
+    from ._app import AsyncFilesystemSource
+
+
+@dataclass(frozen=True)
+class _RmRequest:
+    force: bool
+    directory: bool
+    recursive: bool
+    verbose: bool
+    operands: tuple[_MappedOperand, ...]
+
+
+def _write_verbose_line(spelling: str) -> None:
+    chunk = f"{spelling}\n".encode()
+    stdout = _binary_stdout()
+    _write_binary(stdout, chunk)
+    stdout.flush()
+
+
+def _render_rm_failure(
+    command: str,
+    failure: _UnlinkFailure | _RmdirFailure | _RecursiveRmFailure,
+) -> None:
+    if isinstance(failure, _RecursiveRmFailure):
+        _render_recursive_failure(command, failure)
+    elif isinstance(failure, _RmdirFailure):
+        _render_rmdir_failure(command, failure)
+    else:
+        _render_failure(command, failure)
+
+
+def _render_rm_failure_or_raise(
+    command: str,
+    failure: _UnlinkFailure | _RmdirFailure | _RecursiveRmFailure,
+) -> None:
+    try:
+        _render_rm_failure(command, failure)
+    except Exception as error:
+        raise _CommandFailureError(
+            error=failure.backend_error,
+            render=False,
+            propagate=error,
+        ) from error
+
+
+async def _remove_directory_entry(
+    operand: _MappedOperand,
+    filesystem: AsyncFileSystem,
+) -> _UnlinkFailure | _RmdirFailure | None:
+    failure = await _confirmed_rm_file(operand, filesystem)
+    if isinstance(failure, _UnlinkFailure) and failure.incompatible == "directory":
+        return await _remove_empty_directory(operand, filesystem)
+    return failure
+
+
+async def _trace_operands(
+    command: str,
+    request: _RmRequest,
+    filesystems: Mapping[str, AsyncFileSystem],
+    failures: list[_UnlinkFailure | _RmdirFailure | _RecursiveRmFailure],
+) -> Exception | None:
+    for operand in request.operands:
+        filesystem = filesystems[operand.name]
+        if request.recursive:
+            result = await _remove_recursive(operand, filesystem)
+        elif request.directory:
+            result = await _remove_directory_entry(operand, filesystem)
+        else:
+            result = await _confirmed_rm_file(operand, filesystem)
+        force_missing = request.force and (
+            (
+                isinstance(result, _UnlinkFailure)
+                and not result.uncertain
+                and isinstance(result.backend_error, FileNotFoundError)
+            )
+            or (isinstance(result, _RecursiveRmFailure) and result.root_missing)
+        )
+        if result is not None and not force_missing:
+            failures.append(result)
+            if request.verbose:
+                _render_rm_failure_or_raise(command, result)
+            continue
+        if request.verbose and result is None:
+            try:
+                _write_verbose_line(operand.spelling)
+            except BrokenPipeError as error:
+                return error
+            except Exception as error:  # noqa: BLE001 - stdout boundary.
+                try:
+                    _render_output_failure(command, error)
+                except Exception as render_error:
+                    raise _CommandFailureError(
+                        error=error,
+                        render=False,
+                        propagate=render_error,
+                    ) from render_error
+                return error
+    return None
+
+
+async def _run_rm(
+    command: str,
+    request: _RmRequest,
+    sources: Mapping[str, AsyncFilesystemSource],
+) -> None:
+    async def operation(filesystems: Mapping[str, AsyncFileSystem]) -> None:
+        failures: list[_UnlinkFailure | _RmdirFailure | _RecursiveRmFailure] = []
+        output_error = await _trace_operands(command, request, filesystems, failures)
+        backend_error = _first_backend_error(failures)
+        if not request.verbose:
+            for failure in failures:
+                _render_rm_failure_or_raise(command, failure)
+        if failures or output_error is not None:
+            raise _CommandFailureError(
+                error=backend_error if backend_error is not None else output_error,
+                render=False,
+            )
+
+    await _run_mapped_command(command, request.operands, sources, operation)