fss-cli 0.11.0__py3-none-any.whl

fss_cli/_command.py

@@ -0,0 +1,368 @@
+"""Shared execution support for mapped-source command modules."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import locale
+import sys
+from contextlib import suppress
+from dataclasses import dataclass
+from typing import (
+    TYPE_CHECKING,
+    Literal,
+    NoReturn,
+    Protocol,
+    TypeAlias,
+    TypeGuard,
+    TypeVar,
+    cast,
+)
+
+import typer
+
+from ._diagnostics import _render_diagnostic_prefix, _render_diagnostic_value
+from ._sources import _SourceInvocation
+
+if TYPE_CHECKING:
+    from collections.abc import (
+        Awaitable,
+        Callable,
+        Collection,
+        Iterable,
+        Mapping,
+        Sequence,
+    )
+
+    from fsspec.asyn import AsyncFileSystem
+
+    from ._app import AsyncFilesystemSource
+    from ._recursive_rm import _RecursiveRmFailure
+
+# 128 + SIGPIPE (13): lets pipeline consumers distinguish a closed reader from
+# an ordinary command failure when the broken pipe is the sole failure.
+_BROKEN_PIPE_EXIT_CODE = 141
+_ResultT = TypeVar("_ResultT")
+
+
+def _usage_error(command: str, diagnostic: str) -> NoReturn:
+    """Emit one stable usage diagnostic on stderr and exit ``2``."""
+    prefix = _render_diagnostic_prefix(command)
+    typer.echo(f"{prefix} {diagnostic}", err=True, color=True)
+    raise typer.Exit(2)
+
+
+@dataclass(frozen=True)
+class _MappedOperand:
+    """A parsed ``name:/path`` operand selecting one mapped source."""
+
+    spelling: str
+    name: str
+    path: str
+
+
+class _BinaryWriter(Protocol):
+    """The write/flush surface of a binary stdout stream."""
+
+    def write(self, data: bytes) -> int: ...
+
+    def flush(self) -> None: ...
+
+
+def _binary_stdout() -> _BinaryWriter:
+    """Return the process binary stdout buffer, or raise if unavailable."""
+    buffer = getattr(sys.stdout, "buffer", None)
+    if buffer is None:
+        message = "stdout has no binary buffer"
+        raise OSError(message)
+    return buffer
+
+
+def _write_binary(stdout: _BinaryWriter, payload: bytes) -> None:
+    """Write one complete byte payload or reject a short write."""
+    written = stdout.write(payload)
+    if written != len(payload):
+        message = "short write"
+        raise OSError(message)
+
+
+# The result shapes a command can reject without a backend exception. Commands
+# translate these into their own diagnostic vocabulary when rendering.
+_Incompatible: TypeAlias = Literal["directory", "result", "same_path"]
+
+
+@dataclass(frozen=True)
+class _Failure:
+    """One operand-scoped command failure.
+
+    ``backend_error`` carries an exception raised by a filesystem hook;
+    ``incompatible`` marks a hook that returned successfully with a result the
+    command's contract rejects. ``uncertain`` marks a failure observed after a
+    mutation was dispatched, so the remote state is not known to be unchanged.
+    """
+
+    operand: _MappedOperand
+    backend_error: Exception | None = None
+    incompatible: _Incompatible | None = None
+    uncertain: bool = False
+
+
+# Commands that need to tell apart the code path a failure came from subclass
+# `_Failure` as a tag; the shared helpers stay usable through this bound.
+_FailureT = TypeVar("_FailureT", bound=_Failure)
+
+
+class _CommandFailureError(Exception):
+    """One expected command or output failure crossing the invocation boundary."""
+
+    def __init__(
+        self,
+        operand: _MappedOperand | None = None,
+        error: Exception | None = None,
+        *,
+        render: bool = True,
+        propagate: Exception | None = None,
+    ) -> None:
+        self.operand = operand
+        self.error = error
+        self.render = render
+        self.propagate = propagate
+
+
+async def _drain_current_operation(operation: Awaitable[_ResultT]) -> _ResultT:
+    """Drain one started operation before propagating caller control flow."""
+
+    async def capture() -> tuple[BaseException | None, _ResultT | None]:
+        try:
+            return None, await operation
+        except BaseException as error:  # noqa: BLE001 - preserve exact control flow.
+            return error, None
+
+    task = asyncio.create_task(capture())
+    try:
+        error, result = await asyncio.shield(task)
+    except BaseException:
+        while not task.done():
+            with suppress(BaseException):
+                await asyncio.shield(task)
+        with suppress(BaseException):
+            task.result()
+        raise
+    if error is not None:
+        raise error
+    return cast("_ResultT", result)
+
+
+async def _call(
+    filesystem: AsyncFileSystem,
+    operation: str,
+    *args: object,
+    **kwargs: object,
+) -> object:
+    """Await one named async hook, refusing absent or synchronous forms."""
+    method = getattr(filesystem, operation, None)
+    if not callable(method):
+        raise NotImplementedError
+    result = method(*args, **kwargs)
+    if not inspect.isawaitable(result):
+        raise NotImplementedError
+    return await _drain_current_operation(result)
+
+
+def _render_operand_diagnostic(
+    command: str,
+    operand: _MappedOperand,
+    category: str,
+) -> None:
+    prefix = _render_diagnostic_prefix(command)
+    rendered_operand = _render_diagnostic_value(operand.spelling)
+    typer.echo(f"{prefix} {rendered_operand}: {category}", err=True, color=True)
+
+
+def _render_failure(command: str, failure: _Failure) -> None:
+    if failure.backend_error is None:
+        _render_operand_diagnostic(command, failure.operand, "incompatible result")
+    else:
+        _render_backend_failure(command, failure.operand, failure.backend_error)
+
+
+def _render_backend_failure(
+    command: str,
+    operand: _MappedOperand,
+    error: Exception,
+) -> None:
+    _render_operand_diagnostic(command, operand, _backend_category(error))
+
+
+def _backend_category(error: Exception) -> str:
+    for error_type, category in (
+        (FileNotFoundError, "not found"),
+        (FileExistsError, "file exists"),
+        (PermissionError, "permission denied"),
+        (IsADirectoryError, "is a directory"),
+        (NotADirectoryError, "not a directory"),
+        (NotImplementedError, "unsupported operation"),
+    ):
+        if isinstance(error, error_type):
+            return category
+    rendered_class = _render_diagnostic_value(type(error).__name__)
+    rendered_message = _render_diagnostic_value(str(error))
+    return f"backend failure ({rendered_class}): {rendered_message}"
+
+
+def _first_backend_error(
+    failures: Iterable[_Failure | _RecursiveRmFailure],
+) -> Exception | None:
+    """Return the first backend exception among ``failures``, if any."""
+    return next(
+        (failure.backend_error for failure in failures if failure.backend_error is not None),
+        None,
+    )
+
+
+def _raise_operand_failures(
+    command: str,
+    failures: Sequence[_FailureT],
+    render: Callable[[str, _FailureT], None],
+) -> NoReturn:
+    """Render every operand failure, then fail the invocation exactly once.
+
+    An exception escaping ``render`` (an output failure) propagates instead of
+    the command failure, but the invocation still reports the backend error it
+    had already observed.
+    """
+    backend_error = _first_backend_error(failures)
+    try:
+        for failure in failures:
+            render(command, failure)
+    except Exception as error:
+        raise _CommandFailureError(
+            error=backend_error,
+            render=False,
+            propagate=error,
+        ) from error
+    raise _CommandFailureError(error=backend_error, render=False)
+
+
+def _render_output_failure(command: str, error: Exception) -> None:
+    prefix = _render_diagnostic_prefix(command)
+    rendered_class = _render_diagnostic_value(type(error).__name__)
+    rendered_message = _render_diagnostic_value(str(error))
+    typer.echo(
+        f"{prefix} output: output failure ({rendered_class}): {rendered_message}",
+        err=True,
+        color=True,
+    )
+
+
+async def _run_mapped_command(
+    command: str,
+    operands: tuple[_MappedOperand, ...],
+    sources: Mapping[str, AsyncFilesystemSource],
+    operation: Callable[[Mapping[str, AsyncFileSystem]], Awaitable[None]],
+    *,
+    broken_pipe_exit_code: int = _BROKEN_PIPE_EXIT_CODE,
+) -> None:
+    """Acquire referenced sources, run one command, and own final status."""
+    invocation = _SourceInvocation(command, sources)
+    acquired = False
+    failure: _CommandFailureError | None = None
+    try:
+        filesystems = await invocation.acquire(tuple(dict.fromkeys(operand.name for operand in operands)))
+        acquired = filesystems is not None
+        if filesystems is not None:
+            await operation(filesystems)
+    except _CommandFailureError as error:
+        failure = error
+        if error.render and error.operand is not None:
+            _render_failure(command, _Failure(error.operand, error.error))
+        elif (
+            error.render
+            and error.error is not None
+            and not isinstance(
+                error.error,
+                BrokenPipeError,
+            )
+        ):
+            _render_output_failure(command, error.error)
+    finally:
+        cleanup_failed = await invocation.close_with_command_error(failure.error if failure is not None else None)
+
+    if failure is not None and failure.propagate is not None:
+        raise failure.propagate
+    if not acquired or failure is not None or cleanup_failed:
+        if failure is not None and isinstance(failure.error, BrokenPipeError) and not cleanup_failed:
+            raise typer.Exit(broken_pipe_exit_code)
+        raise typer.Exit(1)
+
+
+async def _run_single_operand_text(
+    command: str,
+    operand: _MappedOperand,
+    sources: Mapping[str, AsyncFilesystemSource],
+    operation: Callable[[AsyncFileSystem], Awaitable[str | _Failure]],
+) -> None:
+    """Run one mapped async operation with buffered text output and cleanup."""
+
+    async def execute(filesystems: Mapping[str, AsyncFileSystem]) -> None:
+        result = await operation(filesystems[operand.name])
+        if isinstance(result, _Failure):
+            raise _CommandFailureError(operand, result.backend_error)
+        if not result:
+            return
+        try:
+            typer.echo(result, nl=False, color=True)
+        except BrokenPipeError as error:
+            raise _CommandFailureError(error=error, render=False) from error
+        except Exception as error:
+            raise _CommandFailureError(error=error) from error
+
+    await _run_mapped_command(
+        command,
+        (operand,),
+        sources,
+        execute,
+        broken_pipe_exit_code=1,
+    )
+
+
+def _collate(value: str) -> tuple[str, str]:
+    """Return the locale-aware sort key used for every ordered CLI listing.
+
+    The raw value breaks ties so that strings the current locale considers
+    equal still order deterministically.
+    """
+    return locale.strxfrm(value), value
+
+
+def _valid_size(value: object) -> TypeGuard[int]:
+    """Accept only an exact non-negative ``int`` byte count.
+
+    ``type(...) is int`` rather than ``isinstance``: ``bool`` subclasses
+    ``int``, so a backend returning ``True`` would otherwise pass as size 1.
+    """
+    return type(value) is int and value >= 0
+
+
+def _parse_mapped_operand(
+    command: str,
+    argument: str,
+    known_names: Collection[str],
+) -> _MappedOperand:
+    """Parse and validate one ``name:/path`` operand against the known sources."""
+    name, separator, path = argument.partition(":")
+    if not name or not separator or not path.startswith("/") or "\0" in argument or "\n" in argument:
+        rendered = _render_diagnostic_value(argument)
+        _usage_error(command, f"{rendered}: invalid mapped filesystem operand")
+
+    if name not in known_names:
+        rendered_operand = _render_diagnostic_value(argument)
+        rendered_names = ", ".join(
+            _render_diagnostic_value(candidate) for candidate in sorted(known_names, key=_collate)
+        )
+        _usage_error(
+            command,
+            f"{rendered_operand}: unknown filesystem (known: {rendered_names})",
+        )
+
+    return _MappedOperand(spelling=argument, name=name, path=path)

fss_cli/_concurrent.py

@@ -0,0 +1,181 @@
+"""Bounded, order-preserving concurrency for independent backend operations.
+
+Remote backends pay one round trip per hook call, so commands that issue many
+independent calls run them concurrently under one fixed bound. Results are
+always returned in submission order, so output and the reported failure stay
+deterministic regardless of completion order.
+
+After the first failure, no new operation starts. Every operation already in
+flight finishes before control flow propagates.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from contextlib import suppress
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Generic, TypeVar, cast
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable, Sequence
+
+_CONCURRENCY = 16
+_ResultT = TypeVar("_ResultT")
+
+
+@dataclass(frozen=True)
+class _Outcome(Generic[_ResultT]):
+    """The observed result of one started operation.
+
+    Exactly one of ``value`` or ``error`` is meaningful: ``error`` is set when
+    the operation raised an ordinary exception.
+    """
+
+    value: _ResultT | None = None
+    error: Exception | None = None
+
+
+async def _capture(
+    operation: Callable[[], Awaitable[_ResultT]],
+) -> _Outcome[_ResultT] | BaseException:
+    """Run one operation, returning control-flow exceptions as values.
+
+    A task must never raise ``KeyboardInterrupt`` or ``SystemExit`` itself:
+    asyncio re-raises those out of the event loop, bypassing the drain below.
+    """
+    try:
+        return _Outcome(value=await operation())
+    except Exception as error:  # noqa: BLE001 - reported in submission order.
+        return _Outcome(error=error)
+    except BaseException as error:  # noqa: BLE001 - preserve exact control flow.
+        return error
+
+
+async def _drain(tasks: Sequence[asyncio.Task[object]]) -> None:
+    """Wait for every task to finish, even while this waiter is cancelled."""
+    if not tasks:
+        return
+    aggregate = asyncio.gather(*tasks, return_exceptions=True)
+    while not aggregate.done():
+        with suppress(BaseException):
+            await asyncio.shield(aggregate)
+
+
+class _BoundedRun(Generic[_ResultT]):
+    """Scheduler state for one :func:`_run_bounded` call."""
+
+    def __init__(
+        self,
+        operations: Sequence[Callable[[], Awaitable[_ResultT]]],
+        stop: Callable[[_ResultT], bool] | None,
+        limit: int,
+    ) -> None:
+        self.operations = operations
+        self.stop = stop
+        self.limit = limit
+        self.outcomes: list[_Outcome[_ResultT] | None] = [None] * len(operations)
+        self.pending: dict[asyncio.Task[_Outcome[_ResultT] | BaseException], int] = {}
+        self.next_index = 0
+        self.stopped = False
+        self.control: BaseException | None = None
+
+    def launch(self) -> None:
+        while not self.stopped and len(self.pending) < self.limit and self.next_index < len(self.operations):
+            operation = self.operations[self.next_index]
+            self.pending[asyncio.create_task(_capture(operation))] = self.next_index
+            self.next_index += 1
+
+    def interrupt(self) -> None:
+        self.stopped = True
+        for task in self.pending:
+            task.cancel()
+
+    def record(self, task: asyncio.Task[_Outcome[_ResultT] | BaseException]) -> None:
+        index = self.pending.pop(task)
+        result = asyncio.CancelledError() if task.cancelled() else task.result()
+        if isinstance(result, BaseException):
+            if self.control is None:
+                self.control = result
+            self.interrupt()
+            return
+        self.outcomes[index] = result
+        if result.error is not None or (self.stop is not None and self.stop(cast("_ResultT", result.value))):
+            self.stopped = True
+
+    async def run(self) -> list[_Outcome[_ResultT] | None]:
+        try:
+            self.launch()
+            while self.pending:
+                done, _ = await asyncio.wait(self.pending, return_when=asyncio.FIRST_COMPLETED)
+                for task in done:
+                    self.record(task)
+                self.launch()
+        except BaseException:
+            self.interrupt()
+            await _drain(tuple(self.pending))
+            raise
+        if self.control is not None:
+            raise self.control
+        return self.outcomes
+
+
+async def _run_bounded(
+    operations: Sequence[Callable[[], Awaitable[_ResultT]]],
+    *,
+    stop: Callable[[_ResultT], bool] | None = None,
+    limit: int = _CONCURRENCY,
+) -> list[_Outcome[_ResultT] | None]:
+    """Run operations with at most ``limit`` in flight, in submission order.
+
+    Operations start in submission order. Once one raises an ordinary
+    exception, or returns a value for which ``stop`` is true, no further
+    operation starts; every operation already started is awaited. The result
+    holds one outcome per started operation and ``None`` for each operation
+    that never started. Because operations start in order, every operation
+    before a stopping one has an outcome, so a caller scanning the result in
+    order meets the first failure in submission order before any ``None``.
+
+    A control-flow exception cancels the other started operations. This
+    includes cancellation and ``KeyboardInterrupt``, whether raised by an
+    operation or delivered to the caller. Each operation drains its in-flight
+    hook call through ``_drain_current_operation``. The exception propagates
+    only after every started operation has finished.
+
+    Args:
+        operations: Zero-argument callables, each starting one operation.
+        stop: Optional predicate marking a successful value as a failure that
+            stops further starts.
+        limit: Maximum number of operations in flight.
+
+    Returns:
+        Outcomes in submission order, ``None`` for unstarted operations.
+    """
+    return await _BoundedRun(operations, stop, limit).run()
+
+
+def _outcome_value(outcome: _Outcome[_ResultT] | None) -> _ResultT:
+    """Return a started operation's value, re-raising its failure."""
+    if outcome is None:
+        message = "operation never started"
+        raise RuntimeError(message)
+    if outcome.error is not None:
+        raise outcome.error
+    return cast("_ResultT", outcome.value)
+
+
+async def _gather_bounded(
+    operations: Sequence[Callable[[], Awaitable[_ResultT]]],
+    *,
+    limit: int = _CONCURRENCY,
+) -> list[_ResultT]:
+    """Run operations under the bound and return their values in order.
+
+    Raises:
+        Exception: The failure of the first failing operation in submission
+            order, after every started operation has finished.
+    """
+    outcomes = await _run_bounded(operations, limit=limit)
+    for outcome in outcomes:
+        if outcome is not None and outcome.error is not None:
+            raise outcome.error
+    return [_outcome_value(outcome) for outcome in outcomes]

fss_cli/_content.py

@@ -0,0 +1,94 @@
+"""Bounded content comparison for resumable recursive copies."""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import binascii
+import hashlib
+import re
+import tempfile
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from ._command import _call, _drain_current_operation
+
+if TYPE_CHECKING:
+    from fsspec.asyn import AsyncFileSystem
+
+_HASH_CHUNK = 1024 * 1024
+_MD5_BYTES = 16
+
+
+def _md5(tokens: tuple[tuple[str, object], ...]) -> bytes | None:
+    values = dict(tokens)
+    value = values.get("md5")
+    if isinstance(value, str) and re.fullmatch(r"[0-9a-fA-F]{32}", value):
+        return bytes.fromhex(value)
+    value = values.get("content-md5")
+    if isinstance(value, (str, bytes)):
+        try:
+            digest = base64.b64decode(value, validate=True)
+        except (ValueError, binascii.Error):
+            return None
+        if len(digest) == _MD5_BYTES:
+            return digest
+    return None
+
+
+def checksum_relation(
+    source: tuple[tuple[str, object], ...],
+    destination: tuple[tuple[str, object], ...],
+) -> bool | None:
+    """Return whether two MD5s match.
+
+    ``None`` means at least one side has no usable MD5, so the caller must
+    compare bytes. ``False`` means both sides have an MD5 and they differ, so
+    a byte download would not change the decision.
+    """
+    left = _md5(source)
+    right = _md5(destination)
+    if left is None or right is None:
+        return None
+    return left == right
+
+
+def matching_checksum(source: tuple[tuple[str, object], ...], destination: tuple[tuple[str, object], ...]) -> bool:
+    """Accept compatible explicit MD5 values, never opaque ETags or sizes."""
+    digest = _md5(source)
+    return digest is not None and digest == _md5(destination)
+
+
+def describe_checksums(source: tuple[tuple[str, object], ...], destination: tuple[tuple[str, object], ...]) -> str:
+    """Describe the content-identity decision for a debug record, without values."""
+    source_digest = _md5(source)
+    destination_digest = _md5(destination)
+    if source_digest is None or destination_digest is None:
+        return "no compatible MD5 checksum on both sides; comparing SHA-256"
+    if source_digest == destination_digest:
+        return "MD5 checksums equal"
+    return "MD5 checksums differ; comparing SHA-256"
+
+
+def _digest(path: str) -> bytes:
+    digest = hashlib.sha256()
+    with Path(path).open("rb") as stream:
+        while chunk := stream.read(_HASH_CHUNK):
+            digest.update(chunk)
+    return digest.digest()
+
+
+async def same_contents(filesystem: AsyncFileSystem, destination: str, staged_source: str) -> bool:
+    """Compare staged bytes without keeping object contents in memory."""
+    directory = tempfile.TemporaryDirectory(prefix="fss-cli-compare-")
+    try:
+        local = str(Path(directory.name) / "destination")
+        await _call(filesystem, "_get_file", destination, local)
+        identical = await _drain_current_operation(asyncio.to_thread(lambda: _digest(staged_source) == _digest(local)))
+    except BaseException:
+        with suppress(BaseException):
+            directory.cleanup()
+        raise
+    directory.cleanup()
+    return identical

fss_cli/_copy_file.py

@@ -0,0 +1,65 @@
+"""Backend-neutral copy dispatch and owned staging shared by both copy commands."""
+
+from __future__ import annotations
+
+import inspect
+import os
+import tempfile
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from ._command import _call
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from fsspec.asyn import AsyncFileSystem
+    from fsspec.callbacks import Callback
+
+
+async def copy_on_backend(
+    filesystem: AsyncFileSystem, source: str, destination: str, callback: Callable[[], Callback]
+) -> bool:
+    """Try the copy hook; only an explicitly unsupported operation permits staging."""
+    try:
+        parameters = inspect.signature(filesystem._cp_file).parameters
+    except (TypeError, ValueError):
+        parameters = {}
+    kwargs = {"callback": callback()} if "callback" in parameters else {}
+    try:
+        await _call(filesystem, "_cp_file", source, destination, **kwargs)
+    except NotImplementedError:
+        return False
+    return True
+
+
+def create_staging(prefix: str) -> str:
+    """Create an owned staging file, cleaning up if descriptor handoff fails."""
+    descriptor, path = tempfile.mkstemp(prefix=prefix)
+    try:
+        os.close(descriptor)
+    except Exception:
+        remove_staging(path)
+        raise
+    except BaseException:
+        discard_staging(path)
+        raise
+    return path
+
+
+def remove_staging(path: str) -> Exception | None:
+    """Return an ordinary cleanup failure without masking the transfer result."""
+    try:
+        Path(path).unlink()
+    except FileNotFoundError:
+        return None
+    except Exception as error:  # noqa: BLE001 - caller owns cleanup diagnostics.
+        return error
+    return None
+
+
+def discard_staging(path: str) -> None:
+    """Clean up while preserving the caller's active control-flow exception."""
+    with suppress(BaseException):
+        remove_staging(path)