fss-cli 0.11.0__py3-none-any.whl

fss_cli/_metadata.py

@@ -0,0 +1,70 @@
+"""Snapshot backend mappings through their authoritative iteration interface."""
+
+import math
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from typing import TypeVar
+
+_Key = TypeVar("_Key")
+_Value = TypeVar("_Value")
+
+
+def valid_display_text(value: str) -> bool:
+    """Reject separators that can split or overwrite output records."""
+    return not any(character in value for character in ("\0", "\r", "\n"))
+
+
+def snapshot_mapping(value: Mapping[_Key, _Value]) -> dict[_Key, _Value]:
+    """Freeze indexed values, rejecting incomplete or changing enumeration."""
+    expected = len(value)
+    result: dict[_Key, _Value] = {}
+    count = 0
+    for key in value:
+        if count >= expected:
+            message = "mapping enumerates more entries than its length"
+            raise ValueError(message)
+        result[key] = value[key]
+        count += 1
+    if count != expected or len(result) != expected or len(value) != expected:
+        message = "mapping enumeration does not match its length"
+        raise ValueError(message)
+    return result
+
+
+def _supported_time(normalized: float) -> float | None:
+    if not math.isfinite(normalized):
+        return None
+    try:
+        datetime.fromtimestamp(normalized, tz=timezone.utc)
+    except (OverflowError, OSError, ValueError):
+        return None
+    return normalized
+
+
+def _datetime_time(value: datetime) -> float | None:
+    if value.tzinfo is None or value.utcoffset() is None:
+        value = value.replace(tzinfo=timezone.utc)
+    try:
+        return _supported_time(value.timestamp())
+    except (OverflowError, OSError, ValueError):
+        return None
+
+
+def normalize_time(value: object) -> float | None:
+    if type(value) is int or type(value) is float:
+        try:
+            normalized = float(value)
+        except OverflowError:
+            return None
+        return _supported_time(normalized)
+    if isinstance(value, datetime):
+        return _datetime_time(value)
+    if not isinstance(value, str):
+        return None
+
+    spelling = value[:-1] + "+00:00" if value.endswith(("Z", "z")) else value
+    try:
+        parsed = datetime.fromisoformat(spelling)
+    except ValueError:
+        return None
+    return _datetime_time(parsed)

fss_cli/_mkdir.py

@@ -0,0 +1,97 @@
+"""Typed async execution for ``mkdir``."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from ._command import (
+    _backend_category,
+    _drain_current_operation,
+    _Failure,
+    _MappedOperand,
+    _raise_operand_failures,
+    _render_operand_diagnostic,
+    _run_mapped_command,
+)
+from ._command import _render_failure as _render_base_failure
+
+if TYPE_CHECKING:
+    from fsspec.asyn import AsyncFileSystem
+
+    from ._app import AsyncFilesystemSource
+
+
+@dataclass(frozen=True)
+class _MkdirRequest:
+    create_parents: bool
+    operands: tuple[_MappedOperand, ...]
+
+
+async def _run_mkdir(
+    command: str,
+    request: _MkdirRequest,
+    sources: Mapping[str, AsyncFilesystemSource],
+) -> None:
+    async def operation(filesystems: Mapping[str, AsyncFileSystem]) -> None:
+        failures = await _trace_operands(request, filesystems)
+        if failures:
+            _raise_operand_failures(command, failures, _render_failure)
+
+    await _run_mapped_command(command, request.operands, sources, operation)
+
+
+async def _trace_operands(
+    request: _MkdirRequest,
+    filesystems: Mapping[str, AsyncFileSystem],
+) -> tuple[_Failure, ...]:
+    failures = []
+    for operand in request.operands:
+        result = await _create_operand(
+            operand,
+            filesystems[operand.name],
+            create_parents=request.create_parents,
+        )
+        if isinstance(result, _Failure):
+            failures.append(result)
+    return tuple(failures)
+
+
+async def _create_operand(
+    operand: _MappedOperand,
+    filesystem: AsyncFileSystem,
+    *,
+    create_parents: bool,
+) -> _Failure | None:
+    try:
+        if create_parents:
+            await _drain_current_operation(filesystem._makedirs(operand.path, exist_ok=True))
+        else:
+            await _drain_current_operation(filesystem._mkdir(operand.path, create_parents=False))
+    except Exception as error:  # noqa: BLE001 - classify awaited backend failure.
+        return _Failure(operand, backend_error=error)
+
+    try:
+        info = await _drain_current_operation(filesystem._info(operand.path))
+    except Exception as error:  # noqa: BLE001 - post-mutation verify is uncertain.
+        return _Failure(operand, backend_error=error, uncertain=True)
+
+    if not isinstance(info, Mapping) or not isinstance(info.get("type"), str):
+        return _Failure(operand, uncertain=True)
+
+    if info["type"] != "directory":
+        return _Failure(operand, uncertain=True)
+
+    return None
+
+
+def _render_failure(command: str, failure: _Failure) -> None:
+    if not failure.uncertain:
+        _render_base_failure(command, failure)
+        return
+    if failure.backend_error is None:
+        category = "uncertain state (incompatible result)"
+    else:
+        category = f"uncertain state ({_backend_category(failure.backend_error)})"
+    _render_operand_diagnostic(command, failure.operand, category)

fss_cli/_mv.py

@@ -0,0 +1,146 @@
+"""Typed async execution for same-source file ``mv``."""
+
+from __future__ import annotations
+
+import inspect
+import time
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from fsspec.asyn import AsyncFileSystem
+
+from ._command import (
+    _call,
+    _CommandFailureError,
+    _MappedOperand,
+    _run_mapped_command,
+    _usage_error,
+)
+from ._cp import (
+    _CpFailure,
+    _CpRequest,
+    _log_verified_transfer,
+    _prepare_transfer,
+    _render_failure,
+    _require_directory,
+    _verify_transfer,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping as MappingType
+
+    from ._app import AsyncFilesystemSource
+
+
+@dataclass(frozen=True)
+class _MvPlan:
+    operands: tuple[_MappedOperand, ...]
+    requests: tuple[_CpRequest, ...]
+    require_directory: bool
+
+
+def _plan_mv(
+    command: str,
+    operands: tuple[_MappedOperand, ...],
+) -> _MvPlan:
+    sources = operands[:-1]
+    destination = operands[-1]
+    if any(source.name != destination.name for source in sources):
+        _usage_error(command, "cross-source move unsupported")
+    return _MvPlan(
+        operands=operands,
+        requests=tuple(_CpRequest(source, destination) for source in sources),
+        require_directory=len(sources) > 1,
+    )
+
+
+def _declared_move(filesystem: AsyncFileSystem) -> str | None:
+    """Return the async move hook this filesystem actually implements.
+
+    ``_mv_file`` is fsspec's hook. ``getattr`` sees an override inherited from
+    a parent. The stock ``AsyncFileSystem._mv_file`` (copy, then delete) does
+    not count: a class that declares ``_mv`` keeps that operation, and a
+    filesystem with neither stays refused.
+    """
+    mv_file = getattr(filesystem, "_mv_file", None)
+    implementation = getattr(mv_file, "__func__", mv_file)
+    if inspect.iscoroutinefunction(mv_file) and implementation is not AsyncFileSystem._mv_file:
+        return "_mv_file"
+    declared = type(filesystem).__dict__.get("_mv")
+    if inspect.iscoroutinefunction(declared):
+        return "_mv"
+    return None
+
+
+async def _confirmed_mv_file(request: _CpRequest, filesystem: AsyncFileSystem) -> _CpFailure | None:
+    started = time.monotonic()
+    prepared = await _prepare_transfer(request, filesystem, filesystem)
+    if isinstance(prepared, _CpFailure):
+        return prepared
+    proof, resolved = prepared.proof, prepared.resolved
+    if request.source.path == resolved:
+        return None
+    operation = _declared_move(filesystem)
+    if operation is None:
+        return _CpFailure(
+            request.destination,
+            backend_error=NotImplementedError("_mv_file or _mv must be configured by source form"),
+        )
+
+    try:
+        await _call(filesystem, operation, request.source.path, resolved)
+    except Exception as operation_error:  # noqa: BLE001
+        return _CpFailure(
+            request.destination,
+            backend_error=operation_error,
+            uncertain=True,
+            residue=True,
+        )
+
+    failure = await _verify_transfer(
+        filesystem,
+        filesystem,
+        request.source.path,
+        resolved,
+        proof,
+        request.destination,
+        require_source_absent=True,
+    )
+    if failure is None:
+        _log_verified_transfer("moved", request, resolved, proof, started)
+    return failure
+
+
+async def _run_mv(
+    command: str,
+    plan: _MvPlan,
+    sources: MappingType[str, AsyncFilesystemSource],
+) -> None:
+    async def operation(filesystems: MappingType[str, AsyncFileSystem]) -> None:
+        filesystem = filesystems[plan.requests[0].source.name]
+        failure = None
+        if plan.require_directory:
+            failure = await _require_directory(
+                plan.requests[0].destination,
+                filesystem,
+            )
+        if failure is None:
+            for request in plan.requests:
+                failure = await _confirmed_mv_file(request, filesystem)
+                if failure is not None:
+                    break
+        if failure is not None:
+            try:
+                _render_failure(command, failure)
+            except Exception as error:
+                raise _CommandFailureError(
+                    error=failure.backend_error,
+                    render=False,
+                    propagate=error,
+                ) from error
+            raise _CommandFailureError(
+                error=failure.backend_error,
+                render=False,
+            )
+
+    await _run_mapped_command(command, plan.operands, sources, operation)

fss_cli/_path.py

@@ -0,0 +1,62 @@
+"""Pure lexical path helpers."""
+
+
+def _is_root(path: str) -> bool:
+    return bool(path) and not path.rstrip("/")
+
+
+def _lexical_root(path: str) -> str:
+    return path.rstrip("/") or "/"
+
+
+def _has_dot_segment(path: str) -> bool:
+    return any(component in {".", ".."} for component in path.split("/"))
+
+
+def _lexical_components(path: str) -> tuple[str, ...]:
+    return tuple(component for component in path.split("/") if component)
+
+
+def _same_lexical_path(left: str, right: str) -> bool:
+    return _lexical_components(left) == _lexical_components(right)
+
+
+def _is_same_or_descendant(parent: str, candidate: str) -> bool:
+    parent_components = _lexical_components(parent)
+    candidate_components = _lexical_components(candidate)
+    return candidate_components[: len(parent_components)] == parent_components
+
+
+def _lexical_relative(parent: str, candidate: str) -> str | None:
+    if not _is_same_or_descendant(parent, candidate):
+        return None
+    return "/".join(_lexical_components(candidate)[len(_lexical_components(parent)) :])
+
+
+def _lexical_basename(path: str) -> str:
+    if _is_root(path):
+        return "/"
+    path = path.rstrip("/")
+    return path.rsplit("/", 1)[-1]
+
+
+def _has_final_dot_segment(path: str) -> bool:
+    return _lexical_basename(path) in {".", ".."}
+
+
+def _lexical_parent(path: str) -> str:
+    if _is_root(path):
+        return "/"
+    path = path.rstrip("/")
+    if "/" not in path:
+        return "."
+    parent = path.rpartition("/")[0]
+    return parent or "/"
+
+
+def _lexical_join(parent: str, child: str) -> str:
+    parent = parent.rstrip("/")
+    child = "" if child == "/" else child
+    if parent in {"", "/"}:
+        return f"/{child}"
+    return f"{parent}/{child}"

fss_cli/_post_copy.py

@@ -0,0 +1,191 @@
+"""Post-copy destination verification shared by file and recursive ``cp``."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import Mapping
+from contextvars import ContextVar
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from ._command import (
+    _CommandFailureError,
+    _drain_current_operation,
+    _MappedOperand,
+    _render_operand_diagnostic,
+)
+from ._diagnostics import _render_diagnostic_value
+from ._manifest import _TOKEN_ALIASES
+
+if TYPE_CHECKING:
+    from fsspec.asyn import AsyncFileSystem
+
+_DEFAULT_METADATA_WAIT_SECONDS = 60
+_METADATA_POLL_SECONDS = 5
+_report_gate: ContextVar[asyncio.Event | None] = ContextVar("fss_cli_report_gate", default=None)
+_report_ready: ContextVar[bool] = ContextVar("fss_cli_report_ready", default=False)
+
+
+@dataclass(frozen=True)
+class _TransferProof:
+    expected_size: int
+    tokens: tuple[tuple[str, frozenset[str | bytes]], ...]
+
+
+@dataclass(frozen=True)
+class _PreviousFile:
+    """The file a resolved destination held before the copy, from resolution ``_info``."""
+
+    size: int | None
+    tokens: Mapping[str, frozenset[str | bytes]]
+
+
+@dataclass(frozen=True)
+class _Verdict:
+    """One classified post-copy ``_info``; ``text`` is ``None`` once verified."""
+
+    text: str | None
+    stale: bool = False
+
+
+@dataclass(frozen=True)
+class _Unverified:
+    """Why a copied file did not verify; ``error`` is a raised ``_info`` failure."""
+
+    text: str
+    error: Exception | None = None
+
+
+def _require_file_size(info: object) -> int | None:
+    if not isinstance(info, Mapping):
+        return None
+    result_type = info.get("type")
+    if not isinstance(result_type, str) or result_type != "file":
+        return None
+    size = info.get("size")
+    if not isinstance(size, int) or isinstance(size, bool) or size < 0:
+        return None
+    return size
+
+
+def _verification_tokens(info: object) -> dict[str, frozenset[str | bytes]]:
+    if not isinstance(info, Mapping):
+        return {}
+    tokens: dict[str, frozenset[str | bytes]] = {}
+    for normalized, aliases in _TOKEN_ALIASES:
+        values = frozenset(
+            value for alias in aliases if (type(value := info.get(alias)) is str or type(value) is bytes)
+        )
+        if values:
+            tokens[normalized] = values
+    return tokens
+
+
+def _post_copy_verdict(  # noqa: PLR0911 - one return per verification outcome.
+    info: object,
+    proof: _TransferProof,
+    previous: _PreviousFile | None,
+    mutation: str,
+) -> _Verdict:
+    """Classify one post-copy destination ``_info`` against the frozen proof.
+
+    Stale means the reported metadata still looks like no size, or like the
+    file that was there before this copy; only a stale verdict is polled.
+    """
+    if not isinstance(info, Mapping) or not isinstance(info.get("type"), str):
+        return _Verdict("verification failure")
+    if info["type"] != "file":
+        return _Verdict(f"not a file after {mutation}")
+    observed = _require_file_size(info)
+    if observed is None:
+        return _Verdict("verification failure")
+    expected = proof.expected_size
+    if observed != expected:
+        if observed == 0 or (previous is not None and observed == previous.size):
+            return _Verdict(f"not yet verified (size {observed}, expected {expected})", stale=True)
+        return _Verdict(f"size mismatch (expected {expected}, observed {observed})")
+    tokens = _verification_tokens(info)
+    differing = [name for name, values in proof.tokens if name in tokens and tokens[name] != values]
+    if not differing:
+        return _Verdict(None)
+    changed = [name for name in differing if previous is None or previous.tokens.get(name) != tokens[name]]
+    if changed:
+        return _Verdict(f"metadata mismatch ({changed[0]})")
+    return _Verdict(f"not yet verified ({differing[0]} unchanged)", stale=True)
+
+
+async def _await_verified(  # noqa: PLR0913 - one explicit post-copy proof boundary.
+    filesystem: AsyncFileSystem,
+    path: str,
+    proof: _TransferProof,
+    previous: _PreviousFile | None,
+    *,
+    mutation: str,
+    wait: int,
+) -> _Unverified | None:
+    """Read destination ``_info`` until it matches the proof, stops being stale, or ``wait`` ends."""
+    deadline = time.monotonic() + wait
+    while True:
+        try:
+            info = await _drain_current_operation(filesystem._info(path))
+        except Exception as error:  # noqa: BLE001 - post-copy verify is residue-bearing.
+            rendered_class = _render_diagnostic_value(type(error).__name__)
+            return _Unverified(f"verification failure ({rendered_class})", error)
+        verdict = _post_copy_verdict(info, proof, previous, mutation)
+        if verdict.text is None:
+            return None
+        remaining = deadline - time.monotonic()
+        if not verdict.stale or remaining <= 0:
+            return _Unverified(verdict.text)
+        await asyncio.sleep(min(_METADATA_POLL_SECONDS, remaining))
+
+
+async def report_verified(  # noqa: PLR0913 - shared per-file outcome and verification contract.
+    command: str,
+    filesystem: AsyncFileSystem,
+    target: _MappedOperand,
+    proof: _TransferProof,
+    previous: _PreviousFile | None,
+    *,
+    mutation: str,
+    wait: int,
+) -> _Unverified | None:
+    """Report a completed transfer and its proof in this operand's output turn."""
+    await _await_report_turn()
+    verb = "copied" if mutation == "copy" else "uploaded"
+    _report(command, target, f"{verb} {proof.expected_size} bytes")
+    unverified = await _await_verified(filesystem, target.path, proof, previous, mutation=mutation, wait=wait)
+    if unverified is None:
+        _report(command, target, f"verified {proof.expected_size} bytes")
+    return unverified
+
+
+def _resolved_operand(destination: _MappedOperand, resolved: str) -> _MappedOperand:
+    return _MappedOperand(
+        spelling=f"{destination.name}:{resolved}",
+        name=destination.name,
+        path=resolved,
+    )
+
+
+async def _await_report_turn() -> None:
+    """Let earlier multi-source copies finish printing before this one does.
+
+    The wait happens once per copy, immediately before its first outcome line,
+    so the byte transfer itself still overlaps and a failed write still aborts
+    the rest of that copy.
+    """
+    gate = _report_gate.get()
+    if gate is None or _report_ready.get():
+        return
+    await gate.wait()
+    _report_ready.set(True)
+
+
+def _report(command: str, operand: _MappedOperand, outcome: str) -> None:
+    """Write one copy outcome line to stderr; a failed write ends the invocation."""
+    try:
+        _render_operand_diagnostic(command, operand, outcome)
+    except Exception as error:
+        raise _CommandFailureError(error=error, render=False, propagate=error) from error

fss_cli/_progress.py

@@ -0,0 +1,132 @@
+"""One shared Rich progress display for a ``cp`` invocation."""
+
+from __future__ import annotations
+
+import os
+import sys
+from contextlib import contextmanager, redirect_stderr, suppress
+from typing import TYPE_CHECKING
+
+from fsspec.callbacks import DEFAULT_CALLBACK, Callback
+from rich import progress as rich_progress
+from rich.console import Console
+from rich.file_proxy import FileProxy
+from rich.markup import escape
+
+from ._diagnostics import _render_diagnostic_value
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+
+def _draws_progress() -> bool:
+    """Draw only on a terminal stderr, and never when ``NO_COLOR`` is set to a value.
+
+    Rich's own terminal detection is not used: ``FORCE_COLOR`` makes it treat
+    a pipe as a terminal, and a pipe must never receive a bar.
+    """
+    if os.environ.get("NO_COLOR"):
+        return False
+    try:
+        return sys.stderr.isatty()
+    except (AttributeError, ValueError):
+        return False
+
+
+class _StderrProxy(FileProxy):
+    """Rich's line proxy, reporting the text attributes click checks before writing to a stream.
+
+    Without them click treats the proxy as incompatible and writes to the
+    proxied file's binary buffer, through the live bars.
+    """
+
+    @property
+    def encoding(self) -> str:
+        return getattr(self.rich_proxied_file, "encoding", None) or "utf-8"
+
+    @property
+    def errors(self) -> str:
+        return getattr(self.rich_proxied_file, "errors", None) or "strict"
+
+
+class _TaskCallback(Callback):
+    """Mirror one fsspec transfer's byte counts onto a Rich task."""
+
+    def __init__(self, progress: rich_progress.Progress, task: rich_progress.TaskID) -> None:
+        super().__init__()
+        self._progress = progress
+        self._task = task
+
+    def call(self, hook_name: str | None = None, **kwargs: object) -> None:
+        del hook_name, kwargs
+        with suppress(KeyError):  # A backend reporting after its hook returned.
+            self._progress.update(self._task, total=self.size, completed=self.value, visible=True)
+
+
+class _Transfer:
+    """One file's bar: hidden until its backend reports bytes, removed by ``finish``."""
+
+    def __init__(self, progress: rich_progress.Progress | None) -> None:
+        self._progress = progress
+        self._task: rich_progress.TaskID | None = None
+
+    def phase(self, verb: str, label: str) -> Callback:
+        """Point the bar at the next hook call and return that call's callback."""
+        progress = self._progress
+        if progress is None:
+            return DEFAULT_CALLBACK
+        description = f"{verb} {escape(_render_diagnostic_value(label))}"
+        if self._task is None:
+            self._task = progress.add_task(description, total=None, visible=False)
+        else:
+            progress.reset(self._task, total=None, completed=0, visible=False, description=description)
+        return _TaskCallback(progress, self._task)
+
+    def finish(self) -> None:
+        if self._progress is not None and self._task is not None:
+            self._progress.remove_task(self._task)
+            # Lines printed next redraw the last rendered frame, which still holds this bar.
+            self._progress.refresh()
+        self._task = None
+
+
+class _TransferDisplay:
+    """The invocation's bars, or none when stderr is not a terminal."""
+
+    def __init__(self, progress: rich_progress.Progress | None = None) -> None:
+        self._progress = progress
+
+    @contextmanager
+    def transfer(self) -> Generator[_Transfer, None, None]:
+        transfer = _Transfer(self._progress)
+        try:
+            yield transfer
+        finally:
+            transfer.finish()
+
+
+_NO_DISPLAY = _TransferDisplay()
+
+
+@contextmanager
+def _transfer_display() -> Generator[_TransferDisplay, None, None]:
+    """Run one ``cp`` invocation under a shared stderr ``Progress`` when bars are drawn.
+
+    Stderr writes during the invocation go through Rich so outcome lines print
+    above the live bars instead of through them.
+    """
+    if not _draws_progress():
+        yield _NO_DISPLAY
+        return
+    console = Console(file=sys.stderr)
+    progress = rich_progress.Progress(
+        rich_progress.TextColumn("[bold blue]{task.description}"),
+        rich_progress.BarColumn(bar_width=None),
+        rich_progress.TimeRemainingColumn(),
+        console=console,
+        transient=True,
+        redirect_stdout=False,
+        redirect_stderr=False,
+    )
+    with progress, redirect_stderr(_StderrProxy(console, sys.stderr)):
+        yield _TransferDisplay(progress)