xrdcore 0.4.0__py3-none-any.whl

xrdcore/__init__.py

@@ -0,0 +1,87 @@
+"""xrdcore: the small, dependency-free core every grid storage client shares.
+
+What lives here is what xrdclient, xgfalclient and xrdarc each needed and
+none of them owns - nothing protocol-specific, nothing that needs a third-party
+package to import:
+
+* :mod:`.safexml` - XML parsing that refuses every ``<!DOCTYPE``, so no
+  entity can be declared, expanded or fetched;
+* :mod:`.checksum`, :mod:`.crc32c`, :mod:`.crc64` - the checksum algorithms
+  storage servers report, behind one incremental, ``hashlib``-shaped surface;
+* :mod:`.log` - credential redaction for logs and terminals, per-level
+  muting, and a command-line tool's ``-v`` handler;
+* :mod:`.cancel` - cancel scopes and cancel hooks, so every layer looks at
+  the same "has the caller given up?";
+* :mod:`.pipeline` - the bounded, read-ahead byte pump every streamed copy
+  ends in;
+* :mod:`.retry` - the one exponential, capped, full-jitter backoff, and
+  HTTP ``Retry-After``;
+* :mod:`.report` - the versioned ``storage-client-report`` JSON/XML document
+  every tool's ``--json``/``--xml`` writes;
+* :mod:`.errors` - the exception vocabulary xrdcore, xrdsec and xrdhttp raise;
+* :mod:`.fanout` - jobs run side by side on threads, the first failure
+  stopping the rest and raised as itself;
+* :mod:`.sockets` - exactly-N-bytes reads from a framed stream socket;
+* :mod:`.cli` - the exit statuses every command-line tool shares, the
+  wrapper that turns Ctrl-C and a closed pipe into them, and the one-line
+  handler for every other failure.
+
+Importing this package imports none of them: each is loaded when first
+named, so ``import xrdcore`` costs nothing a ``--help`` would notice.
+"""
+
+from __future__ import annotations
+
+import importlib
+from types import ModuleType
+from typing import TYPE_CHECKING
+
+from ._version import __version__
+
+if TYPE_CHECKING:
+    from . import (
+        cancel,
+        checksum,
+        cli,
+        crc32c,
+        crc64,
+        errors,
+        fanout,
+        log,
+        pipeline,
+        report,
+        retry,
+        safexml,
+        sockets,
+    )
+
+__all__ = [
+    "__version__",
+    "cancel",
+    "checksum",
+    "cli",
+    "crc32c",
+    "crc64",
+    "errors",
+    "fanout",
+    "log",
+    "pipeline",
+    "report",
+    "retry",
+    "safexml",
+    "sockets",
+]
+
+_SUBMODULES = frozenset(__all__) - {"__version__"}
+
+
+def __getattr__(name: str) -> ModuleType:
+    """Import a submodule the first time it is named as an attribute."""
+    if name in _SUBMODULES:
+        return importlib.import_module(f"{__name__}.{name}")
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__() -> list[str]:
+    """The public names, including submodules not yet imported."""
+    return sorted(__all__)

xrdcore/_version.py

@@ -0,0 +1,3 @@
+"""The xrdcore distribution version, shared with the build backend."""
+
+__version__ = "0.4.0"

xrdcore/cancel.py

@@ -0,0 +1,242 @@
+"""Cancellation: how a call learns that whoever started it has given up.
+
+There are two ways in, for two kinds of caller, and they meet here so that a
+maintainer has one place to read - and so that every library on the stack
+(the XRootD session, the HTTP connections, the copy pipeline) looks at the
+*same* scope, whichever client started the call.
+
+* **Scopes** - :class:`CancelScope`, below - are how an asyncio front end
+  cancels the worker thread behind a cancelled coroutine. Nothing is torn
+  down: the request is abandoned where it waits, and the connection stays in
+  use.
+* **Hooks** - a ``cancel=`` argument on a long transfer - are how a
+  synchronous caller stops it from another thread (gfal2's ``cancel()``, a
+  GUI button, a signal handler). A hook is asked between pieces of work, and
+  a watcher thread (:func:`watching`) aborts the handles in flight the moment
+  it fires, so a worker blocked on the network wakes at once. That costs the
+  connections, which is the point: the transfer is over, and so is anything
+  it had outstanding.
+
+Scopes
+------
+
+A scope reaches a request already waiting on the wire.
+
+A blocking call cannot be interrupted from outside its thread. What can be
+done is to tell the worker that nobody is waiting for the answer any more,
+and have every place the worker might wait look for that. This module is the
+telling::
+
+    scope = CancelScope()
+    worker = pool.submit(contextvars.copy_context().run, within, scope, call)
+    ...
+    scope.cancel()          # from any thread - the event loop's, typically
+
+A :class:`CancelScope` travels in a :mod:`contextvars` variable, so it reaches
+whatever the call runs - including work started on threads of its own (a
+fan-out, the copy read-ahead in :mod:`xrdcore.pipeline`) that copy the context
+they were started from.
+
+Once a scope is cancelled it stays cancelled (*level*-triggered, as Trio's
+scopes are, not edge-triggered as :mod:`asyncio`'s are): every wait inside it
+raises :class:`Cancelled` from then on, not only the first. That is what makes
+a cancelled worker come back promptly whatever it was doing - a copy that
+caught the first one in a retry, or a fan-out whose other jobs were mid-read,
+would otherwise carry on regardless. The price is that a cancelled call's own
+clean-up round trips (a checkpoint rollback, removing a partial remote target)
+are skipped too; they would be waits on the very server that was too slow.
+
+What looks at the scope is the transport's business: a session before it
+sends a request and while it waits for one, an HTTP connection before it
+sends. Waiting is done in slices of :data:`POLL` seconds while a scope is in
+force (:func:`bounded`), which is how a waiter notices a cancellation without
+anyone having to wake it: the canceller may be an event loop that must never
+block on a session's lock, and a socket read cannot be woken without shutting
+the socket down.
+"""
+
+from __future__ import annotations
+
+import contextvars
+import threading
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from typing import TypeVar
+
+__all__ = [
+    "POLL",
+    "Cancel",
+    "CancelScope",
+    "Cancelled",
+    "as_check",
+    "bounded",
+    "cancelled",
+    "check",
+    "current",
+    "watching",
+    "within",
+]
+
+T = TypeVar("T")
+
+#: The longest a wait inside a scope goes without looking at it, in seconds.
+#: Short enough that a cancelled worker is back well within half a second,
+#: long enough that a parked waiter costs a few wake-ups a second and no more.
+POLL = 0.05
+
+
+class Cancelled(BaseException):
+    """The call was cancelled; the request it was waiting on has been abandoned.
+
+    A :class:`BaseException`, like :class:`asyncio.CancelledError` and for the
+    same reason: it has to travel out of the worker through every
+    ``except Exception`` and every client's own error base on the way - a
+    retry, a fall-back to another replica, a re-open after a lost server -
+    without any of them taking it for a failure worth recovering from. An
+    asyncio caller never sees it: the coroutine that was cancelled has raised
+    its own :class:`asyncio.CancelledError` already, and this ends the
+    worker's call.
+    """
+
+
+class CancelScope:
+    """A flag any thread may set, telling the calls inside the scope to stop.
+
+    ``parent`` is the scope in force where this one was made: a scope nested
+    in a cancelled one is cancelled too, so an inner scope never shields a
+    call from an outer cancellation.
+    """
+
+    __slots__ = ("_event", "_parent")
+
+    def __init__(self, parent: CancelScope | None = None) -> None:
+        """A live scope; ``parent`` is normally :func:`current` where it is made."""
+        self._event = threading.Event()
+        self._parent = parent
+
+    def cancel(self) -> None:
+        """Cancel every call inside this scope. Idempotent, and never blocks."""
+        self._event.set()
+
+    @property
+    def cancelled(self) -> bool:
+        """Whether this scope, or any scope it is nested in, has been cancelled.
+
+        Walks the parent chain on every read instead of copying the parent's
+        state in, so cancelling an outer scope reaches inner scopes made
+        before it was cancelled. Lock-free: ``Event.is_set`` is thread-safe.
+        """
+        if self._event.is_set():
+            return True
+        return self._parent is not None and self._parent.cancelled
+
+    def __repr__(self) -> str:
+        """``CancelScope(live)`` or ``CancelScope(cancelled)``, for debugging."""
+        return f"CancelScope({'cancelled' if self.cancelled else 'live'})"
+
+
+#: The innermost scope in force here, or ``None`` outside every one.
+_scope: contextvars.ContextVar[CancelScope | None] = contextvars.ContextVar(
+    "xrdcore_cancel", default=None
+)
+
+
+def current() -> CancelScope | None:
+    """The scope in force in this thread or task, if any."""
+    return _scope.get()
+
+
+def within(scope: CancelScope, call: Callable[[], T]) -> T:
+    """Run ``call`` with ``scope`` in force, then put the old one back."""
+    token = _scope.set(scope)
+    try:
+        return call()
+    finally:
+        _scope.reset(token)
+
+
+def cancelled() -> bool:
+    """Whether the scope in force has been cancelled."""
+    scope = _scope.get()
+    return scope is not None and scope.cancelled
+
+
+def check(what: str = "the request") -> None:
+    """Raise :class:`Cancelled` if the scope in force has been cancelled."""
+    if cancelled():
+        raise Cancelled(f"{what} was cancelled")
+
+
+def bounded(timeout: float | None) -> float | None:
+    """``timeout``, cut to :data:`POLL` while a scope is in force.
+
+    For a wait that must notice a cancellation: inside a scope it wakes at
+    least that often to look, and outside one it waits as long as it was
+    always going to.
+    """
+    if _scope.get() is None:
+        return timeout
+    return POLL if timeout is None else min(timeout, POLL)
+
+
+# ---------------------------------------------------------------------------
+# Hooks: ``cancel=`` on a long transfer
+# ---------------------------------------------------------------------------
+
+#: What a ``cancel=`` argument may be: a :class:`threading.Event` - set it to
+#: cancel - or a callable answering "should this stop now?", which is what
+#: gfal2's and XrdCl's progress handlers' ``ShouldCancel`` is. A callable is
+#: asked from a worker thread, so it must be thread-safe and quick.
+Cancel = Callable[[], bool] | threading.Event
+
+
+def as_check(cancel: Cancel | None) -> Callable[[], bool] | None:
+    """``cancel`` as a callable, or ``None`` for no hook at all."""
+    if cancel is None:
+        return None
+    if isinstance(cancel, threading.Event):
+        return cancel.is_set
+    return cancel
+
+
+@contextmanager
+def watching(
+    check: Callable[[], bool] | None,
+    on_cancel: Callable[[], None],
+    *,
+    interval: float = POLL,
+    name: str = "xrd-cancel",
+) -> Iterator[None]:
+    """While the block runs, call ``on_cancel`` once if ``check`` says to stop.
+
+    ``check`` is asked every ``interval`` seconds on a thread of its own, which
+    is what reaches a worker blocked between pieces - asking only between
+    them would leave a stalled read to its timeout. ``on_cancel`` runs on that
+    thread and should do no more than wake the workers (abort their handles,
+    set their stop flag); the watcher is gone by the time the block exits, so
+    it never runs after. A hook that raises counts as a cancel: a callback
+    that cannot answer is no reason to keep going a transfer nobody can stop.
+    """
+    if check is None:
+        yield
+        return
+    done = threading.Event()
+
+    def watch() -> None:
+        """Poll ``check`` until the block ends or it says stop, then fire once."""
+        while not done.wait(interval):
+            try:
+                stop = check()
+            except Exception:
+                stop = True
+            if stop:
+                on_cancel()
+                return
+
+    watcher = threading.Thread(target=watch, name=name, daemon=True)
+    watcher.start()
+    try:
+        yield
+    finally:
+        done.set()
+        watcher.join()

xrdcore/checksum.py

@@ -0,0 +1,195 @@
+"""Checksum algorithms, by the names storage servers use.
+
+Every algorithm exposes the incremental ``hashlib``-ish shape so a copy can
+digest as it streams:
+
+    >>> h = new("adler32")
+    >>> h.update(b"hello")
+    >>> h.hexdigest()
+    '062c0215'
+
+Two families sit behind one surface. The CRCs and Adler-32 are "native":
+a step function folds each chunk into an integer register, and the hex form
+is that integer zero-padded to the algorithm's width - the spelling
+``kXR_Qcksum`` answers with and ``xrdadler32`` prints (``adler32`` of an empty
+file is ``00000001``, not ``1``). The cryptographic hashes are
+:mod:`hashlib`'s, wrapped to look the same. Callers never need to know which
+they hold.
+
+Names are XRootD's (``adler32``, ``crc32``, ``crc32c``, ``crc64``, ``md5``...)
+because the same string goes on the wire in ``kXR_query`` checksum requests
+and is compared with what servers answer; HTTP's spellings (RFC 3230 /
+RFC 9530: ``sha-256``, ``adler``) are mapped onto these by
+:mod:`xrdhttp.webdav`. A client with a naming policy of its own - gfal2's
+narrower algorithm set, its ``UNIXcksum`` alias - layers it over :func:`new`
+rather than forking the digests, so every tool computes the same function.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import zlib
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+
+from .crc32c import crc32c
+from .crc64 import crc64, crc64nvme
+
+__all__ = [
+    "Checksum",
+    "ChecksumInfo",
+    "Digest",
+    "algorithms",
+    "checksum_bytes",
+    "checksum_file",
+    "new",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class ChecksumInfo:
+    """A finished checksum as a client reports it: an algorithm name and its hex value.
+
+    One type for every source - a ``root://`` checksum query, a WebDAV
+    ``Digest`` header, a Metalink ``<hash>`` - so checksums from different
+    protocols compare equal when they describe the same bytes. It lives here,
+    below xrdhttp and the clients, because each of them reports checksums and
+    two look-alike classes would never compare equal to each other.
+    """
+
+    algorithm: str
+    value: str
+
+    def __str__(self) -> str:
+        """``algorithm:value``, the form ``xrdcp --cksum`` and gfal2 accept and print."""
+        return f"{self.algorithm}:{self.value}"
+
+
+#: One chunk folded into a running value - the shape every algorithm here has.
+Step = Callable[["bytes | bytearray | memoryview", int], int]
+
+
+class Checksum:
+    """Incremental digest with a stable lowercase hex representation."""
+
+    __slots__ = ("_step", "_value", "_width", "name")
+
+    def __init__(self, name: str, step: Step, seed: int, width: int) -> None:
+        """Start a digest at ``seed`` that ``step`` advances; ``width`` is in hex digits.
+
+        The seed is the algorithm's own initial register (``1`` for
+        Adler-32, ``0`` for the CRCs, whose pre- and post-inversion live
+        inside ``step``). Use :func:`new` rather than building one by hand.
+        """
+        self.name = name
+        self._step = step
+        self._value = seed
+        self._width = width
+
+    def update(self, data: bytes | bytearray | memoryview) -> None:
+        """Fold ``data`` into the running value; chunking never changes the result."""
+        self._value = self._step(data, self._value)
+
+    @property
+    def value(self) -> int:
+        """The running value as an integer, as ``zlib.adler32`` would return it."""
+        return self._value
+
+    def hexdigest(self) -> str:
+        """Lower-case hex, zero-padded to the algorithm's full width."""
+        return f"{self._value:0{self._width}x}"
+
+    def digest(self) -> bytes:
+        """The value as big-endian bytes - the order the hex form is written in."""
+        return self._value.to_bytes(self._width // 2, "big")
+
+    def __repr__(self) -> str:
+        """Name and current hex value."""
+        return f"Checksum({self.name!r}, {self.hexdigest()})"
+
+
+class _HashlibChecksum:
+    """Adapter giving a ``hashlib`` object the same surface as `Checksum`."""
+
+    __slots__ = ("_h", "name")
+
+    def __init__(self, name: str) -> None:
+        """Wrap a fresh :func:`hashlib.new` object for ``name``."""
+        self.name = name
+        self._h = hashlib.new(name)
+
+    def update(self, data: bytes | bytearray | memoryview) -> None:
+        """Feed ``data`` to the hash."""
+        self._h.update(data)
+
+    @property
+    def value(self) -> int:
+        """The digest so far as a big-endian integer, for parity with :class:`Checksum`."""
+        return int.from_bytes(self._h.digest(), "big")
+
+    def hexdigest(self) -> str:
+        """:mod:`hashlib`'s lower-case hex digest; already full width."""
+        return self._h.hexdigest()
+
+    def digest(self) -> bytes:
+        """:mod:`hashlib`'s raw digest bytes."""
+        return self._h.digest()
+
+    def __repr__(self) -> str:
+        """Name and current hex value, spelt as :class:`Checksum` spells it."""
+        return f"Checksum({self.name!r}, {self.hexdigest()})"
+
+
+#: Either kind of digest :func:`new` returns.
+Digest = Checksum | _HashlibChecksum
+
+#: ``name -> (step, seed, width in hex digits)`` for the non-hashlib family.
+_NATIVE: dict[str, tuple[Callable[..., int], int, int]] = {
+    "adler32": (zlib.adler32, 1, 8),
+    "crc32": (zlib.crc32, 0, 8),
+    "crc32c": (crc32c, 0, 8),
+    "crc64": (crc64, 0, 16),
+    "crc64nvme": (crc64nvme, 0, 16),
+}
+#: The one alias a server registers: CRC-64/XZ answers to both names.
+_ALIASES = {"crc64xz": "crc64"}
+#: The :mod:`hashlib` names offered: all in ``hashlib.algorithms_guaranteed``,
+#: so :func:`algorithms` is the same on every platform.
+_HASHLIB = ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")
+
+
+def algorithms() -> tuple[str, ...]:
+    """Every algorithm this build can compute."""
+    return tuple(sorted(_NATIVE)) + _HASHLIB
+
+
+def new(name: str) -> Digest:
+    """A fresh incremental digest for ``name`` (case-insensitive).
+
+    Dashes are ignored, so HTTP's ``SHA-256`` and XRootD's ``sha256`` are
+    one algorithm. Raises :class:`ValueError` naming what is available for
+    any other name - a typo must not silently pick a different algorithm.
+    """
+    key = name.lower().replace("-", "")
+    key = _ALIASES.get(key, key)
+    if key in _NATIVE:
+        step, seed, width = _NATIVE[key]
+        return Checksum(key, step, seed, width)
+    if key in _HASHLIB:
+        return _HashlibChecksum(key)
+    raise ValueError(f"unknown checksum algorithm {name!r}; have {algorithms()}")
+
+
+def checksum_bytes(name: str, data: bytes) -> str:
+    """Hex digest of ``data``."""
+    h = new(name)
+    h.update(data)
+    return h.hexdigest()
+
+
+def checksum_file(name: str, chunks: Iterable[bytes]) -> str:
+    """Hex digest of a stream of chunks."""
+    h = new(name)
+    for chunk in chunks:
+        h.update(chunk)
+    return h.hexdigest()