xrdclient 0.1.0__py3-none-any.whl

xrdclient/__init__.py

@@ -0,0 +1,249 @@
+"""A pure-Python XRootD client.
+
+Speaks the native ``root://`` binary protocol and HTTP/WebDAV, with no
+compiled extension and no XRootD installation required.
+
+    >>> import xrdclient
+    >>> with xrdclient.open("root://eos.example.org//store/data.root") as f:
+    ...     header = f.read(1024)
+
+The five levels of the API, from most to least convenient:
+
+:mod:`xrdclient.easy`
+    One-line verbs on a URL - ``xrdclient.ls``, ``xrdclient.size``, ``xrdclient.read_text`` -
+    for when there is one question to ask and no reason to build anything.
+``xrdclient.open`` / :class:`~xrdclient.path.XRootDPath`
+    File objects and ``pathlib`` semantics.
+:class:`~xrdclient.client.FileSystem` / :class:`~xrdclient.client.File`
+    Explicit per-operation control, sync or async.
+:class:`~xrdclient.session.Session`
+    A single authenticated connection.
+:mod:`xrdclient.proto`
+    The sans-io protocol machinery.
+
+Three packages build on this one and are installed separately:
+`xrdroot <https://github.com/rob-c/xrdroot>`_ reads and writes the ROOT file
+format over any URL here, `xrdml <https://github.com/rob-c/xrdml>`_ hands a
+file of rows to PyTorch a minibatch at a time, and
+`xrddatasets <https://github.com/rob-c/xrddatasets>`_ converts open data into
+ROOT files and publishes the catalogue they are served from.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .client import Checkpoint, File, FileSystem
+from .config import Config, configure, current, find_config_file, override
+from .copy import CopyResult, SyncMode, copy, copy_tree, third_party
+from .easy import (
+    checksum,
+    exists,
+    glob,
+    is_online,
+    ls,
+    mkdir,
+    move,
+    read_bytes,
+    read_text,
+    remove,
+    size,
+    stage,
+    stat,
+    write_bytes,
+    write_text,
+)
+from .errors import (
+    AttrNotFoundError,
+    AuthenticationError,
+    BusyError,
+    ChecksumMismatchError,
+    ConnectionError,
+    CredentialError,
+    InvalidArgumentError,
+    NoMechanismError,
+    NoSpaceError,
+    NotFoundError,
+    PageIntegrityError,
+    ProtocolError,
+    QuotaError,
+    ReadOnlyError,
+    RedirectLimitError,
+    ServerError,
+    ServerTimeoutError,
+    TimeoutError,
+    TLSRequiredError,
+    TokenExpiredError,
+    TooLargeError,
+    TransientError,
+    UnsupportedError,
+    WaitLimitError,
+    XRootDError,
+)
+from .flags import (
+    Access,
+    DirListFlags,
+    LocateFlags,
+    MkDirFlags,
+    OpenFlags,
+    PrepareFlags,
+    QueryCode,
+    StatInfoFlags,
+)
+from .io import open_url as open
+from .path import XRootDPath
+from .path import XRootDPath as Path  # ``xrdclient.Path`` reads the way pathlib does
+from .session.deadline import OperationExpiredError, deadline
+from .types import (
+    CheckpointInfo,
+    ChecksumInfo,
+    CloneRange,
+    DirEntry,
+    LocationInfo,
+    PageResult,
+    PrepareStatus,
+    ProtocolInfo,
+    ReadRange,
+    SpaceInfo,
+    StatInfo,
+    VFSInfo,
+    WriteChunk,
+    human_bytes,
+)
+from .url import XRootDURL, parse
+
+if TYPE_CHECKING:  # bound at runtime by ``__getattr__`` below, named here so
+    # that a type checker and the documentation can both see what they are
+    from .doctor import Check, Report, diagnose
+del TYPE_CHECKING  # imported for the block above, not part of the package
+
+__version__ = "0.1.0"
+
+
+#: Names that live in a submodule nobody should pay to import. Each is bound
+#: on first use by :func:`__getattr__`; the value is the module it comes from.
+_LAZY = {"Check": "doctor", "Report": "doctor", "diagnose": "doctor"}
+
+
+def __getattr__(name: str) -> object:
+    """Expose ``xrdclient.aio`` and the diagnostics lazily.
+
+    Each is one attribute access away and neither costs anything until it is
+    asked for: nothing under ``xrd`` imports :mod:`asyncio` until someone
+    reaches for ``xrdclient.aio``, or the environment checks until someone runs
+    :func:`~xrdclient.doctor.diagnose`. A transfer is the common case and pays for
+    neither.
+    """
+    import importlib
+
+    if name in ("aio", "doctor"):
+        # Not ``from . import aio``: the import system resolves that by asking
+        # this very function for the attribute, and the recursion is infinite.
+        return importlib.import_module(f"{__name__}.{name}")
+    if name in _LAZY:
+        value = getattr(importlib.import_module(f"{__name__}.{_LAZY[name]}"), name)
+        globals()[name] = value  # bound once; later lookups skip this function
+        return value
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+    "__version__",
+    # the facades that are imported on first use
+    "aio",
+    # configuration
+    "Config",
+    "configure",
+    "current",
+    "override",
+    "find_config_file",
+    # a time limit on whatever runs inside a block
+    "deadline",
+    # copying
+    "copy",
+    "copy_tree",
+    "third_party",
+    "CopyResult",
+    "SyncMode",
+    # diagnosing
+    "diagnose",
+    "Check",
+    "Report",
+    # files and paths
+    "open",
+    "XRootDPath",
+    "Path",
+    "FileSystem",
+    "File",
+    "Checkpoint",
+    # one-line verbs, for when a URL is all you have
+    "ls",
+    "glob",
+    "stat",
+    "exists",
+    "size",
+    "checksum",
+    "read_bytes",
+    "read_text",
+    "write_bytes",
+    "write_text",
+    "mkdir",
+    "remove",
+    "move",
+    "stage",
+    "is_online",
+    "human_bytes",
+    # urls
+    "XRootDURL",
+    "parse",
+    # values
+    "CheckpointInfo",
+    "ChecksumInfo",
+    "CloneRange",
+    "DirEntry",
+    "LocationInfo",
+    "PageResult",
+    "ProtocolInfo",
+    "ReadRange",
+    "StatInfo",
+    "SpaceInfo",
+    "PrepareStatus",
+    "VFSInfo",
+    "WriteChunk",
+    # flags
+    "Access",
+    "DirListFlags",
+    "LocateFlags",
+    "MkDirFlags",
+    "OpenFlags",
+    "PrepareFlags",
+    "QueryCode",
+    "StatInfoFlags",
+    # errors
+    "XRootDError",
+    "ProtocolError",
+    "ConnectionError",
+    "TimeoutError",
+    "OperationExpiredError",
+    "TransientError",
+    "AuthenticationError",
+    "NoMechanismError",
+    "CredentialError",
+    "TokenExpiredError",
+    "RedirectLimitError",
+    "WaitLimitError",
+    "ChecksumMismatchError",
+    "PageIntegrityError",
+    "ServerError",
+    "NotFoundError",
+    "NoSpaceError",
+    "UnsupportedError",
+    "ReadOnlyError",
+    "QuotaError",
+    "AttrNotFoundError",
+    "BusyError",
+    "InvalidArgumentError",
+    "ServerTimeoutError",
+    "TLSRequiredError",
+    "TooLargeError",
+]

xrdclient/_compat.py

@@ -0,0 +1,86 @@
+"""What the oldest Python this library runs on has not got.
+
+That is 3.9, because it is what RHEL 9 and AlmaLinux 9 ship, and a physicist
+on a login node at a site that runs one of those cannot choose otherwise. The
+handful of things 3.10 added that this library would use anyway are written
+out here rather than scattered through it, and where the interpreter has the
+real thing, the real thing is what runs.
+
+Nothing else in this package may name a version. If a module needs something
+newer than the floor, it belongs here, with the fallback beside it.
+"""
+
+from __future__ import annotations
+
+import itertools
+import socket
+import sys
+from collections.abc import Iterable, Iterator
+from typing import Any, TypeVar, overload
+
+__all__ = ["SLOTS", "TIMEOUTS", "flag_members", "zip_strict"]
+
+T = TypeVar("T")
+U = TypeVar("U")
+
+#: ``slots=True`` for a dataclass, where the interpreter takes it. Slots make
+#: a record smaller and its attributes quicker to reach; 3.9 has no such
+#: argument, and the only thing missing there is that saving.
+SLOTS: dict[str, bool] = {"slots": True} if sys.version_info >= (3, 10) else {}
+
+#: What a socket raises when it runs out of patience. 3.10 made
+#: :exc:`socket.timeout` another name for :exc:`TimeoutError`; before that they
+#: were two exceptions and only the first was ever raised, so every ``except``
+#: that means "the far end went quiet" names both.
+TIMEOUTS: tuple[type[BaseException], ...] = (socket.timeout, TimeoutError)
+
+#: Whether :func:`zip` can do the length check itself, in C.
+_NATIVE_STRICT_ZIP = sys.version_info >= (3, 10)
+
+
+@overload
+def zip_strict(a: Iterable[T], b: Iterable[U], /) -> Iterator[tuple[T, U]]: ...
+
+
+@overload
+def zip_strict(*iterables: Iterable[Any]) -> Iterator[tuple[Any, ...]]: ...
+
+
+def zip_strict(*iterables: Iterable[Any]) -> Iterator[tuple[Any, ...]]:
+    """:func:`zip`, refusing iterables that turn out to be different lengths.
+
+    ``zip(..., strict=True)`` where there is one, and the same check written
+    out where there is not. The check is the whole point: everything paired
+    this way in this library is two halves of one thing - a flag per path, a
+    byte per byte of the block before it - and a bug that makes them different
+    lengths is invisible if the shorter one simply ends the loop.
+    """
+    if _NATIVE_STRICT_ZIP:
+        return zip(*iterables, strict=True)
+    return _paired(*iterables)
+
+
+def flag_members(flag: Any) -> list[Any]:
+    """The single bits a :class:`~enum.Flag` is made of, in declaration order.
+
+    3.11 taught a flag member to iterate into its own parts. Before that a
+    flag was not iterable at all, so the parts are picked out of the class
+    here: the same members, in the same order, whatever the interpreter. A bit
+    nobody named is dropped, as 3.11 drops it, and a member standing for more
+    than one bit is not a part of anything.
+    """
+    bits = int(flag)
+    return [
+        member
+        for member in type(flag)
+        if member.value and not member.value & (member.value - 1) and member.value & bits
+    ]
+
+
+def _paired(*iterables: Iterable[Any]) -> Iterator[tuple[Any, ...]]:
+    """3.9's :func:`zip`, with the length check 3.10 does for itself."""
+    ended = object()
+    for values in itertools.zip_longest(*iterables, fillvalue=ended):
+        if any(value is ended for value in values):
+            raise ValueError("zip_strict() was given iterables of different lengths")
+        yield values

xrdclient/_log.py

@@ -0,0 +1,96 @@
+"""Logging helpers.
+
+Everything logs under the ``xrdclient.`` hierarchy through a filter that redacts
+credential material, so enabling DEBUG never leaks a token into a log file.
+
+The same filter can mute parts of the hierarchy at some levels and not
+others (:func:`mute`) - what XrdCl's per-level topic masks do, and what the
+compatibility layer's ``SetLogMask`` is built on.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Iterable
+
+__all__ = ["get_logger", "mute", "redact"]
+
+_PATTERNS = (
+    re.compile(r"(authz=)[^&\s'\"]+", re.I),
+    re.compile(r"(Bearer\s+)[A-Za-z0-9._~+/-]+=*", re.I),
+    re.compile(r"(eyJ[A-Za-z0-9_-]{4,})\.[A-Za-z0-9._-]+"),
+    re.compile(r"((?:token|password|secret|keytab|cred)['\"]?\s*[=:]\s*)\S+", re.I),
+)
+
+
+def redact(text: str) -> str:
+    """Replace credential material in ``text`` with ``<redacted>``."""
+    for pat in _PATTERNS:
+        text = pat.sub(r"\1<redacted>", text)
+    return text
+
+
+class _RedactingFilter(logging.Filter):
+    """Interpolate first, then redact the result.
+
+    Redacting the format string and the arguments separately misses anything
+    that only looks like a credential once they are joined - ``"%s=%s"`` with
+    ``("token", secret)`` sails straight through - and it can corrupt the
+    format string itself, since ``"keytab: %s"`` is exactly the shape of a
+    secret assignment. Formatting up front has neither problem, and handlers
+    call :meth:`~logging.LogRecord.getMessage` anyway.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if _muted and _is_muted(record):
+            return False
+        try:
+            message = record.getMessage()
+        except (TypeError, ValueError):
+            return True  # a broken format string is logging's problem, not ours
+        cleaned = redact(message)
+        if cleaned != message or record.args:
+            record.msg = cleaned
+            record.args = ()
+        return True
+
+
+#: ``(lowest level, level above the highest) -> logger prefixes`` whose
+#: records in that band of levels are dropped. Replaced, never mutated, so
+#: the filter reads it without a lock.
+_muted: dict[tuple[int, int], tuple[str, ...]] = {}
+
+
+def mute(low: int, high: int, loggers: Iterable[str]) -> None:
+    """Drop records from ``loggers`` with a level from ``low`` up to ``high``.
+
+    Each name covers its logger and every logger below it, and replaces what
+    was muted in that band before; no names un-mutes it.
+    """
+    global _muted
+    prefixes = tuple(f"{name}." for name in loggers)
+    changed = {band: names for band, names in _muted.items() if band != (low, high)}
+    if prefixes:
+        changed[(low, high)] = prefixes
+    _muted = changed
+
+
+def _is_muted(record: logging.LogRecord) -> bool:
+    name = f"{record.name}."
+    return any(
+        low <= record.levelno < high and name.startswith(prefixes)
+        for (low, high), prefixes in _muted.items()
+    )
+
+
+_filter = _RedactingFilter()
+_root = logging.getLogger("xrdclient")
+_root.addFilter(_filter)
+
+
+def get_logger(name: str) -> logging.Logger:
+    """A logger under ``xrdclient.`` with redaction applied."""
+    log = logging.getLogger(name if name.startswith("xrdclient") else f"xrdclient.{name}")
+    log.addFilter(_filter)
+    return log