xrdcore 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xrdcore/__init__.py +87 -0
- xrdcore/_version.py +3 -0
- xrdcore/cancel.py +242 -0
- xrdcore/checksum.py +195 -0
- xrdcore/cli.py +234 -0
- xrdcore/crc32c.py +116 -0
- xrdcore/crc64.py +95 -0
- xrdcore/errors.py +153 -0
- xrdcore/fanout.py +254 -0
- xrdcore/log.py +299 -0
- xrdcore/pipeline.py +273 -0
- xrdcore/py.typed +0 -0
- xrdcore/report.py +748 -0
- xrdcore/retry.py +178 -0
- xrdcore/safexml.py +81 -0
- xrdcore/sockets.py +47 -0
- xrdcore-0.4.0.dist-info/METADATA +95 -0
- xrdcore-0.4.0.dist-info/RECORD +21 -0
- xrdcore-0.4.0.dist-info/WHEEL +4 -0
- xrdcore-0.4.0.dist-info/licenses/COPYING +674 -0
- xrdcore-0.4.0.dist-info/licenses/LICENSE +165 -0
xrdcore/cli.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""The exit statuses every command-line tool on the stack shares, and how a run ends.
|
|
2
|
+
|
|
3
|
+
xrdarc, xrdwms and the storage clients' tools all end a command the same few
|
|
4
|
+
ways, and used to spell each one separately:
|
|
5
|
+
|
|
6
|
+
* :class:`ExitCode` - the statuses whose meaning is the same in every tool:
|
|
7
|
+
0 success, 1 a failure the tool reported, 2 a usage error (argparse's own),
|
|
8
|
+
70 a bug (``EX_SOFTWARE`` from ``sysexits.h``), and the shell's
|
|
9
|
+
``128 + signal`` for Ctrl-C (130) and a reader that went away (141). A
|
|
10
|
+
tool with codes of its own - xrdarc's 3-7 for credentials, descriptions,
|
|
11
|
+
CEs, local faults and timeouts; xrdwms's for configuration and backends -
|
|
12
|
+
defines its own :class:`enum.IntEnum` that restates these members and adds
|
|
13
|
+
its own, checked by :func:`extends` (an enum with members cannot be
|
|
14
|
+
subclassed, so this is the extension Python allows);
|
|
15
|
+
* :func:`run` - the wrapper around a command's body that turns Ctrl-C into
|
|
16
|
+
130 and ``BrokenPipeError`` into a *quiet* 141, and hands every other
|
|
17
|
+
exception to the tool's own reporting;
|
|
18
|
+
* :func:`silence_stdout` - what makes that 141 quiet;
|
|
19
|
+
* :class:`Fatal` - the usual handler for everything else: one line on
|
|
20
|
+
stderr (``prog: message``, redacted), an ``error`` record in a
|
|
21
|
+
``--json``/``--xml`` report, and the exit status from a table of
|
|
22
|
+
exception classes the tool supplies (:func:`exit_code`), with the
|
|
23
|
+
traceback of a bug logged at DEBUG rather than printed. xrdarc and
|
|
24
|
+
xrdwms end every command through it; xrdclient's ``fail`` is one.
|
|
25
|
+
|
|
26
|
+
Tools that mirror an upstream tool's statuses keep them: gfal2-util's
|
|
27
|
+
``gfal-*`` exit with an errno (``EINTR`` after Ctrl-C), XrdCl's ``xrd*`` with
|
|
28
|
+
theirs, because scripts written against those tools test those numbers.
|
|
29
|
+
They use this module only where their statuses coincide (0, 1) and for
|
|
30
|
+
:func:`silence_stdout` (and xrdclient for :class:`Fatal`'s one line).
|
|
31
|
+
|
|
32
|
+
What a maintainer must not break: the numbers. Scripts test them, and each
|
|
33
|
+
tool's documentation publishes its table.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import dataclasses
|
|
39
|
+
import enum
|
|
40
|
+
import os
|
|
41
|
+
import sys
|
|
42
|
+
from collections.abc import Callable, Iterable, Sequence
|
|
43
|
+
from typing import Any, TypeAlias, TypeVar
|
|
44
|
+
|
|
45
|
+
from .log import get_logger, redact
|
|
46
|
+
|
|
47
|
+
__all__ = ["ExitCode", "ExitRule", "Fatal", "exit_code", "extends", "run", "silence_stdout"]
|
|
48
|
+
|
|
49
|
+
#: A tool's own exit-code enum, as :func:`extends` hands it back.
|
|
50
|
+
E = TypeVar("E", bound=type[enum.IntEnum])
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class ExitCode(enum.IntEnum):
|
|
54
|
+
"""The exit statuses with the same meaning in every tool.
|
|
55
|
+
|
|
56
|
+
Small numbers rather than an errno for the tool's own failures, because
|
|
57
|
+
errno values differ between Linux and macOS and collide with 1 and 2
|
|
58
|
+
(``EPERM``, ``ENOENT``).
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
#: Everything worked.
|
|
62
|
+
OK = 0
|
|
63
|
+
#: The command ran and something it did failed (some jobs, some files).
|
|
64
|
+
FAILED = 1
|
|
65
|
+
#: The command line is wrong: argparse's own status for a usage error.
|
|
66
|
+
USAGE = 2
|
|
67
|
+
#: A bug: an exception nothing expected (``EX_SOFTWARE``).
|
|
68
|
+
INTERNAL = 70
|
|
69
|
+
#: Ctrl-C: 128 + ``SIGINT``, what the shell reports for a process it killed.
|
|
70
|
+
INTERRUPTED = 130
|
|
71
|
+
#: The reader of standard output went away (``... | head``): 128 + ``SIGPIPE``.
|
|
72
|
+
BROKEN_PIPE = 141
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def extends(base: type[enum.IntEnum]) -> Callable[[E], E]:
|
|
76
|
+
"""A class decorator: check that a tool's exit-code enum keeps every code of ``base``.
|
|
77
|
+
|
|
78
|
+
``@extends(ExitCode)`` over ``class ExitCode(enum.IntEnum)`` raises
|
|
79
|
+
:class:`TypeError` at import time when the tool's enum lacks one of
|
|
80
|
+
``base``'s members or gives it another number, so a tool cannot quietly
|
|
81
|
+
drift from the shared table. Restating each member as
|
|
82
|
+
``OK = xrdcore.cli.ExitCode.OK`` keeps the numbers in one place; the
|
|
83
|
+
member then holds the plain integer, as any ``IntEnum`` member does.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
def check(cls: E) -> E:
|
|
87
|
+
"""The decorated enum, unchanged, once it has been found to keep ``base``'s codes."""
|
|
88
|
+
members = cls.__members__
|
|
89
|
+
drifted = [m.name for m in base if m.name not in members or members[m.name] != m]
|
|
90
|
+
if drifted:
|
|
91
|
+
raise TypeError(f"{cls.__name__} does not keep {base.__name__}'s codes: {drifted}")
|
|
92
|
+
return cls
|
|
93
|
+
|
|
94
|
+
return check
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def silence_stdout() -> None:
|
|
98
|
+
"""Point standard output at ``/dev/null`` once the reader of a pipe has gone.
|
|
99
|
+
|
|
100
|
+
Otherwise the interpreter's final flush of ``sys.stdout`` fails again at
|
|
101
|
+
exit and prints ``Exception ignored ... BrokenPipeError`` (and exits
|
|
102
|
+
120). This is what the :mod:`signal` module's documentation recommends
|
|
103
|
+
for ``SIGPIPE``. A stream with no file descriptor (a test's capture) is
|
|
104
|
+
left alone.
|
|
105
|
+
"""
|
|
106
|
+
try:
|
|
107
|
+
fd = sys.stdout.fileno()
|
|
108
|
+
except (AttributeError, ValueError, OSError): # io.UnsupportedOperation is a ValueError
|
|
109
|
+
return
|
|
110
|
+
# The descriptor is replaced rather than ``sys.stdout`` reassigned, so the
|
|
111
|
+
# interpreter's own flush at exit - through whatever object it holds -
|
|
112
|
+
# lands somewhere that accepts it.
|
|
113
|
+
devnull = os.open(os.devnull, os.O_WRONLY)
|
|
114
|
+
os.dup2(devnull, fd)
|
|
115
|
+
os.close(devnull)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def run(
|
|
119
|
+
main: Callable[[], int],
|
|
120
|
+
*,
|
|
121
|
+
interrupted: Callable[[], None] | None = None,
|
|
122
|
+
fatal: Callable[[Exception], int] | None = None,
|
|
123
|
+
) -> int:
|
|
124
|
+
"""Run a command's body and turn the ways it can end into an exit status.
|
|
125
|
+
|
|
126
|
+
``main`` returns the status when it finishes. Ctrl-C is
|
|
127
|
+
:attr:`ExitCode.INTERRUPTED`, after ``interrupted`` (if given) has said
|
|
128
|
+
so; a ``BrokenPipeError`` is :attr:`ExitCode.BROKEN_PIPE`, with nothing
|
|
129
|
+
said - the reader that would have seen it is gone - and stdout silenced
|
|
130
|
+
so the interpreter's exit cannot complain either. Any other exception
|
|
131
|
+
goes to ``fatal``, whose return is the status (the tool reports it and
|
|
132
|
+
maps it to its own table); without ``fatal`` it propagates.
|
|
133
|
+
|
|
134
|
+
The broken pipe is caught before ``fatal`` sees anything, although it
|
|
135
|
+
is an ``OSError`` a tool would otherwise call a local fault: ``| head``
|
|
136
|
+
stopping early is how a pipe is used, not a failure.
|
|
137
|
+
"""
|
|
138
|
+
try:
|
|
139
|
+
return main()
|
|
140
|
+
except KeyboardInterrupt:
|
|
141
|
+
if interrupted is not None:
|
|
142
|
+
interrupted()
|
|
143
|
+
return int(ExitCode.INTERRUPTED)
|
|
144
|
+
except BrokenPipeError:
|
|
145
|
+
silence_stdout()
|
|
146
|
+
return int(ExitCode.BROKEN_PIPE)
|
|
147
|
+
except Exception as exc:
|
|
148
|
+
if fatal is None:
|
|
149
|
+
raise
|
|
150
|
+
return fatal(exc)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
#: One row of a tool's exit-code table: an exception class, and its status -
|
|
154
|
+
#: a number, or a function of the exception for a class whose status depends
|
|
155
|
+
#: on what it carries (xrdarc's ``ARCError`` is the CE's fault when it names a
|
|
156
|
+
#: URL and a local one otherwise).
|
|
157
|
+
ExitRule: TypeAlias = tuple[type[BaseException], int | Callable[[Any], int]]
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def exit_code(exc: BaseException, table: Iterable[ExitRule]) -> int:
|
|
161
|
+
"""The status the first row of ``table`` whose class ``exc`` is an instance of gives.
|
|
162
|
+
|
|
163
|
+
Rows are tried in order, so a subclass goes before its base (a usage
|
|
164
|
+
error that is also the tool's general error must be found as the usage
|
|
165
|
+
error). An exception no row names is a bug: :attr:`ExitCode.INTERNAL`.
|
|
166
|
+
"""
|
|
167
|
+
for kind, code in table:
|
|
168
|
+
if isinstance(exc, kind):
|
|
169
|
+
return code if isinstance(code, int) else code(exc)
|
|
170
|
+
return ExitCode.INTERNAL
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _str(exc: BaseException) -> str:
|
|
174
|
+
"""The exception's own message: :class:`Fatal`'s default ``describe``."""
|
|
175
|
+
return str(exc)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
@dataclasses.dataclass(frozen=True)
|
|
179
|
+
class Fatal:
|
|
180
|
+
"""The handler :func:`run` hands a failure to: say it in one line, return its status.
|
|
181
|
+
|
|
182
|
+
``Fatal("xrdwms", TABLE, logger="xrdwms")`` is passed as ``run``'s
|
|
183
|
+
``fatal`` (and its :meth:`interrupted` as ``interrupted``). For an
|
|
184
|
+
exception it
|
|
185
|
+
|
|
186
|
+
* records it in the active ``--json``/``--xml`` report
|
|
187
|
+
(:func:`xrdcore.report.error`; nothing without one);
|
|
188
|
+
* finds its status in ``codes`` (:func:`exit_code`);
|
|
189
|
+
* says ``prog: message`` on stderr - through the report when there is
|
|
190
|
+
one - where the message is ``describe(exc)`` for a failure the table
|
|
191
|
+
knows and ``internal error: Type: message`` for a bug, whose
|
|
192
|
+
traceback goes to ``logger`` at DEBUG (``-vv``/``-d DEBUG``) instead of
|
|
193
|
+
the terminal. Either way the line is redacted
|
|
194
|
+
(:func:`xrdcore.log.redact`): an error that quotes a URL can quote a
|
|
195
|
+
token with it.
|
|
196
|
+
|
|
197
|
+
It can also be called directly, outside :func:`run`, wherever a tool
|
|
198
|
+
reports a failure and carries on (xrdclient's ``fail``). The line is
|
|
199
|
+
``prog: message`` and not argparse's ``prog: error: message`` because
|
|
200
|
+
that is what each tool has always printed and its tests and users read.
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
#: How the line names the tool: ``xrdwms``, or ``xrdarc stat`` for a subcommand.
|
|
204
|
+
prog: str
|
|
205
|
+
#: The tool's exit-code table, most specific class first.
|
|
206
|
+
codes: Sequence[ExitRule]
|
|
207
|
+
#: The message for a failure the table knows; the exception's own text by default.
|
|
208
|
+
describe: Callable[[BaseException], str] = _str
|
|
209
|
+
#: The logger a bug's traceback goes to at DEBUG; the root by default.
|
|
210
|
+
logger: str = ""
|
|
211
|
+
|
|
212
|
+
def __call__(self, exc: BaseException) -> int:
|
|
213
|
+
"""Report ``exc`` and return its exit status."""
|
|
214
|
+
# Imported here: xrdcore.report imports this module for ExitCode.
|
|
215
|
+
from . import report
|
|
216
|
+
|
|
217
|
+
report.error(exc)
|
|
218
|
+
code = exit_code(exc, self.codes)
|
|
219
|
+
if code == ExitCode.INTERNAL:
|
|
220
|
+
get_logger(self.logger).debug("internal error", exc_info=exc)
|
|
221
|
+
self.say(f"internal error: {type(exc).__name__}: {exc}")
|
|
222
|
+
else:
|
|
223
|
+
self.say(self.describe(exc))
|
|
224
|
+
return int(code)
|
|
225
|
+
|
|
226
|
+
def interrupted(self) -> None:
|
|
227
|
+
"""Say ``prog: interrupted``: what :func:`run` calls on Ctrl-C."""
|
|
228
|
+
self.say("interrupted")
|
|
229
|
+
|
|
230
|
+
def say(self, text: str) -> None:
|
|
231
|
+
"""One redacted ``prog: text`` line on stderr, or a ``message`` record under a report."""
|
|
232
|
+
from . import report
|
|
233
|
+
|
|
234
|
+
report.message(f"{self.prog}: {redact(text)}\n", stderr=True)
|
xrdcore/crc32c.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Castagnoli CRC-32C (poly 0x1EDC6F41, reflected 0x82F63B78).
|
|
2
|
+
|
|
3
|
+
Used by XRootD paged I/O (every ``kXR_pgread``/``kXR_pgwrite`` page carries
|
|
4
|
+
one), by the ``crc32c`` checksum type every storage protocol offers, and by
|
|
5
|
+
S3's ``x-amz-checksum-crc32c``. Not the same polynomial as ``zlib.crc32``
|
|
6
|
+
(IEEE) - never share a table.
|
|
7
|
+
|
|
8
|
+
With the ``fast`` extra (``pip install 'xrdcore[fast]'``, which installs
|
|
9
|
+
``google-crc32c``) the hardware-assisted C implementation is used - gigabytes
|
|
10
|
+
per second. Without it a pure-Python slice-by-eight fallback runs, a few
|
|
11
|
+
MB/s: correct, and fine for checksumming a small file, but it is what bounds
|
|
12
|
+
a bulk verification. :data:`IS_ACCELERATED` says which one is in use.
|
|
13
|
+
|
|
14
|
+
The fallback is public as :func:`crc32c_py`, so a test can compare the two
|
|
15
|
+
implementations and a benchmark can time the slow path on purpose. The
|
|
16
|
+
4 KiB page framing XRootD's paged I/O wraps around it is the XRootD
|
|
17
|
+
client's, not this module's.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import struct
|
|
23
|
+
from collections.abc import Callable
|
|
24
|
+
|
|
25
|
+
__all__ = ["IS_ACCELERATED", "crc32c", "crc32c_py"]
|
|
26
|
+
|
|
27
|
+
#: The reflected Castagnoli polynomial: CRC-32C is computed LSB-first.
|
|
28
|
+
_POLY = 0x82F63B78
|
|
29
|
+
_MASK = 0xFFFFFFFF
|
|
30
|
+
|
|
31
|
+
_Buffer = bytes | bytearray | memoryview
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _build_tables() -> list[list[int]]:
|
|
35
|
+
"""The eight slice-by-eight tables: ``T[0]`` is the classic byte table,
|
|
36
|
+
and ``T[k]`` is ``T[k-1]`` advanced over one more zero byte, so that
|
|
37
|
+
eight bytes can be folded with eight independent lookups."""
|
|
38
|
+
first = []
|
|
39
|
+
for n in range(256):
|
|
40
|
+
crc = n
|
|
41
|
+
for _ in range(8):
|
|
42
|
+
crc = (crc >> 1) ^ _POLY if crc & 1 else crc >> 1
|
|
43
|
+
first.append(crc)
|
|
44
|
+
tables = [first]
|
|
45
|
+
for k in range(1, 8):
|
|
46
|
+
prev = tables[k - 1]
|
|
47
|
+
tables.append([(v >> 8) ^ first[v & 0xFF] for v in prev])
|
|
48
|
+
return tables
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
_T = _build_tables()
|
|
52
|
+
_T0, _T1, _T2, _T3, _T4, _T5, _T6, _T7 = _T
|
|
53
|
+
|
|
54
|
+
#: Eight bytes as one little-endian word and four single bytes: the shape
|
|
55
|
+
#: slice-by-eight consumes, unpacked in C for the whole buffer at once.
|
|
56
|
+
_EIGHT = struct.Struct("<I4B")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def crc32c_py(data: _Buffer, crc: int = 0) -> int:
|
|
60
|
+
"""Slice-by-eight software CRC-32C.
|
|
61
|
+
|
|
62
|
+
``struct.iter_unpack`` does the slicing of every 8-byte group in C, which
|
|
63
|
+
is roughly twice as fast as indexing a memoryview per byte; what remains
|
|
64
|
+
per group is eight table lookups.
|
|
65
|
+
"""
|
|
66
|
+
view = memoryview(data).cast("B")
|
|
67
|
+
whole = len(view) - len(view) % 8
|
|
68
|
+
c = (crc ^ _MASK) & _MASK
|
|
69
|
+
for word, b4, b5, b6, b7 in _EIGHT.iter_unpack(view[:whole]):
|
|
70
|
+
c ^= word
|
|
71
|
+
c = (
|
|
72
|
+
_T7[c & 0xFF]
|
|
73
|
+
^ _T6[(c >> 8) & 0xFF]
|
|
74
|
+
^ _T5[(c >> 16) & 0xFF]
|
|
75
|
+
^ _T4[c >> 24]
|
|
76
|
+
^ _T3[b4]
|
|
77
|
+
^ _T2[b5]
|
|
78
|
+
^ _T1[b6]
|
|
79
|
+
^ _T0[b7]
|
|
80
|
+
)
|
|
81
|
+
for byte in view[whole:]:
|
|
82
|
+
c = _T0[(c ^ byte) & 0xFF] ^ (c >> 8)
|
|
83
|
+
return c ^ _MASK
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _accelerated() -> Callable[[_Buffer, int], int] | None:
|
|
87
|
+
"""``google-crc32c``'s C implementation, or ``None`` if it is not here.
|
|
88
|
+
|
|
89
|
+
The package has its own pure-Python fallback, slower than ours; only its
|
|
90
|
+
C extension (``implementation == "c"``) is worth using.
|
|
91
|
+
"""
|
|
92
|
+
try:
|
|
93
|
+
import google_crc32c
|
|
94
|
+
except ImportError:
|
|
95
|
+
return None
|
|
96
|
+
if getattr(google_crc32c, "implementation", "") != "c":
|
|
97
|
+
return None
|
|
98
|
+
extend = google_crc32c.extend
|
|
99
|
+
|
|
100
|
+
def fast(data: _Buffer, crc: int = 0) -> int:
|
|
101
|
+
"""CRC-32C of ``data``, chained onto ``crc`` (hardware-assisted)."""
|
|
102
|
+
# It takes only ``bytes``; a page is 4 KiB, so the copy is noise.
|
|
103
|
+
return int(extend(crc, data if isinstance(data, bytes) else bytes(data)))
|
|
104
|
+
|
|
105
|
+
return fast
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
_FAST = _accelerated()
|
|
109
|
+
#: Whether :func:`crc32c` is the C implementation (the ``fast`` extra).
|
|
110
|
+
IS_ACCELERATED = _FAST is not None
|
|
111
|
+
_IMPLEMENTATION = _FAST or crc32c_py
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def crc32c(data: _Buffer, crc: int = 0) -> int:
|
|
115
|
+
"""CRC-32C of ``data``, chained onto ``crc``: ``crc32c(b, crc32c(a)) == crc32c(a + b)``."""
|
|
116
|
+
return _IMPLEMENTATION(data, crc)
|
xrdcore/crc64.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""The two 64-bit CRCs XRootD servers report, both reflected and both seeded
|
|
2
|
+
and finished with all ones.
|
|
3
|
+
|
|
4
|
+
``crc64`` is CRC-64/XZ (ECMA-182, normal polynomial ``0x42F0E1EBA9EA3693``) -
|
|
5
|
+
what ``xrdcrc64`` computes and what ``kXR_Qcksum`` and a WebDAV ``Digest``
|
|
6
|
+
header call ``crc64``. ``crc64nvme`` is CRC-64/NVME (Rocksoft, normal
|
|
7
|
+
polynomial ``0xAD93D23594C93659``), the one S3 spells
|
|
8
|
+
``x-amz-checksum-crc64nvme``; a server offers it under its own name, never as
|
|
9
|
+
``crc64``, because the two disagree on every input.
|
|
10
|
+
|
|
11
|
+
Stock XRootD computes neither, so a checksum in one of these names comes from
|
|
12
|
+
a gateway - and this is what verifies it without one.
|
|
13
|
+
|
|
14
|
+
Both are pure Python, slice-by-eight: a few MB/s, which is what a checksum
|
|
15
|
+
verification after a transfer costs. No widely packaged accelerator covers
|
|
16
|
+
them - ``google-crc32c`` is CRC-32C only, and the CRC-64/NVME in ``awscrt``
|
|
17
|
+
is a large native dependency to take for one checksum - so there is no
|
|
18
|
+
optional fast path; a site that verifies large files by ``crc64`` should
|
|
19
|
+
prefer ``adler32`` or ``crc32c`` where the server offers them.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import struct
|
|
25
|
+
|
|
26
|
+
__all__ = ["crc64", "crc64nvme"]
|
|
27
|
+
|
|
28
|
+
_MASK = (1 << 64) - 1
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _tables(reflected_poly: int) -> tuple[list[int], ...]:
|
|
32
|
+
"""The eight slice-by-eight tables: ``T[0]`` is the byte table, ``T[k]``
|
|
33
|
+
is ``T[k-1]`` advanced one more zero byte."""
|
|
34
|
+
first = []
|
|
35
|
+
for n in range(256):
|
|
36
|
+
crc = n
|
|
37
|
+
for _ in range(8):
|
|
38
|
+
crc = (crc >> 1) ^ reflected_poly if crc & 1 else crc >> 1
|
|
39
|
+
first.append(crc)
|
|
40
|
+
tables = [first]
|
|
41
|
+
for _ in range(7):
|
|
42
|
+
tables.append([(value >> 8) ^ first[value & 0xFF] for value in tables[-1]])
|
|
43
|
+
return tuple(tables)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
#: Reflections of 0x42F0E1EBA9EA3693 and 0xAD93D23594C93659.
|
|
47
|
+
_XZ = _tables(0xC96C5795D7870F42)
|
|
48
|
+
_NVME = _tables(0x9A6C9329AC4BC9B5)
|
|
49
|
+
|
|
50
|
+
_WORD = struct.Struct("<Q")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _fold(tables: tuple[list[int], ...], data: bytes | bytearray | memoryview, crc: int) -> int:
|
|
54
|
+
"""Slice-by-eight: each little-endian 64-bit word is folded in with eight lookups."""
|
|
55
|
+
t0, t1, t2, t3, t4, t5, t6, t7 = tables
|
|
56
|
+
view = memoryview(data).cast("B")
|
|
57
|
+
whole = len(view) - len(view) % 8
|
|
58
|
+
c = ~crc & _MASK
|
|
59
|
+
for (word,) in _WORD.iter_unpack(view[:whole]):
|
|
60
|
+
c ^= word
|
|
61
|
+
c = (
|
|
62
|
+
t7[c & 0xFF]
|
|
63
|
+
^ t6[(c >> 8) & 0xFF]
|
|
64
|
+
^ t5[(c >> 16) & 0xFF]
|
|
65
|
+
^ t4[(c >> 24) & 0xFF]
|
|
66
|
+
^ t3[(c >> 32) & 0xFF]
|
|
67
|
+
^ t2[(c >> 40) & 0xFF]
|
|
68
|
+
^ t1[(c >> 48) & 0xFF]
|
|
69
|
+
^ t0[c >> 56]
|
|
70
|
+
)
|
|
71
|
+
return ~_bytewise(t0, view[whole:], c) & _MASK
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _bytewise(table: list[int], view: memoryview, c: int) -> int:
|
|
75
|
+
"""The classic one-lookup-per-byte loop, for the last ``len % 8`` bytes."""
|
|
76
|
+
for byte in view:
|
|
77
|
+
c = table[(c ^ byte) & 0xFF] ^ (c >> 8)
|
|
78
|
+
return c
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def crc64(data: bytes | bytearray | memoryview, crc: int = 0) -> int:
|
|
82
|
+
"""CRC-64/XZ of ``data``, chained onto ``crc``.
|
|
83
|
+
|
|
84
|
+
``crc`` is the running value in the form it is reported in, so chaining
|
|
85
|
+
two chunks gives what one call over both would::
|
|
86
|
+
|
|
87
|
+
>>> crc64(b"9", crc64(b"12345678")) == crc64(b"123456789")
|
|
88
|
+
True
|
|
89
|
+
"""
|
|
90
|
+
return _fold(_XZ, data, crc)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def crc64nvme(data: bytes | bytearray | memoryview, crc: int = 0) -> int:
|
|
94
|
+
"""CRC-64/NVME of ``data``, chained onto ``crc``."""
|
|
95
|
+
return _fold(_NVME, data, crc)
|
xrdcore/errors.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""The error vocabulary every library on the stack raises.
|
|
2
|
+
|
|
3
|
+
xrdcore, xrdsec and xrdhttp raise these and nothing client-specific, so a
|
|
4
|
+
library never has to know which client called it. A client *joins* this
|
|
5
|
+
hierarchy rather than wrapping it:
|
|
6
|
+
|
|
7
|
+
* where a client's error means what one here means, the client's name is
|
|
8
|
+
bound to *this* class. xrdclient does that for all of them
|
|
9
|
+
(``xrdclient.XRootDError is XrdError``, ``xrdclient.errors.CredentialError
|
|
10
|
+
is CredentialError``, ...), so ``except`` written against the client's name
|
|
11
|
+
catches what any library raises, with nothing translated or re-raised. A
|
|
12
|
+
client subclass the other way round would not: ``except`` catches a class
|
|
13
|
+
and what is below it, and the library's class is not below the client's;
|
|
14
|
+
* the client's own errors - XRootD's ``kXR_*`` server errors, gfal2's
|
|
15
|
+
``GError`` - derive from these, by multiple inheritance where they also
|
|
16
|
+
need a builtin or a base of the client's, so code that only knows the
|
|
17
|
+
libraries (``except XrdError``) catches the clients' errors too.
|
|
18
|
+
|
|
19
|
+
The builtins are shadowed on purpose, as the clients shadow them:
|
|
20
|
+
:class:`ConnectionError` and :class:`TimeoutError` each *subclass* their
|
|
21
|
+
builtin, so either spelling of ``except`` catches them.
|
|
22
|
+
|
|
23
|
+
What must not break: every exception here pickles with all its attributes
|
|
24
|
+
(``__reduce__`` + :func:`rebuild`), because errors cross process boundaries
|
|
25
|
+
under fsspec/dask and :mod:`concurrent.futures`. A new exception with
|
|
26
|
+
keyword-only or extra state needs its own ``__reduce__``.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import builtins
|
|
32
|
+
from collections.abc import Callable
|
|
33
|
+
from typing import Any
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"AuthenticationError",
|
|
37
|
+
"ConnectionError",
|
|
38
|
+
"CredentialError",
|
|
39
|
+
"InsecureCredentialError",
|
|
40
|
+
"MetalinkError",
|
|
41
|
+
"ProtocolError",
|
|
42
|
+
"TimeoutError",
|
|
43
|
+
"TokenExpiredError",
|
|
44
|
+
"TransientError",
|
|
45
|
+
"XrdError",
|
|
46
|
+
"rebuild",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class XrdError(Exception):
|
|
51
|
+
"""Base of every error xrdcore, xrdsec and xrdhttp raise."""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
#: What ``__reduce__`` returns here: :func:`rebuild` and the arguments it is called with.
|
|
55
|
+
_Reduced = tuple[Callable[..., BaseException], tuple[object, ...]]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def rebuild(
|
|
59
|
+
cls: Callable[..., BaseException], args: tuple[object, ...], kwargs: dict[str, Any]
|
|
60
|
+
) -> BaseException:
|
|
61
|
+
"""Unpickle an exception that carries more than ``args``: ``cls(*args, **kwargs)``.
|
|
62
|
+
|
|
63
|
+
``BaseException.__reduce__`` replays only ``args``, so an exception with
|
|
64
|
+
keyword-only or extra state returns ``rebuild, (type(self), args,
|
|
65
|
+
kwargs)`` from its own ``__reduce__``. Public because the clients' own
|
|
66
|
+
exceptions (xrdclient's ``ServerError`` family, ...) pickle through it
|
|
67
|
+
too: one function, so a pickle written by any layer loads in any other.
|
|
68
|
+
"""
|
|
69
|
+
return cls(*args, **kwargs)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class ProtocolError(XrdError):
|
|
73
|
+
"""A peer answered with something malformed, unexpected or unsupported."""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class MetalinkError(ProtocolError, ValueError):
|
|
77
|
+
"""A Metalink descriptor is unsafe, malformed, or unusable."""
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class ConnectionError(XrdError, builtins.ConnectionError):
|
|
81
|
+
"""The transport failed."""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class TransientError(ConnectionError):
|
|
85
|
+
"""A retryable failure. Records progress so a caller can resume."""
|
|
86
|
+
|
|
87
|
+
def __init__(self, message: str, *, attempts: int = 0, committed: int = 0) -> None:
|
|
88
|
+
"""Record the failure and how far the operation got.
|
|
89
|
+
|
|
90
|
+
``attempts`` is how many tries were made before giving up; ``committed``
|
|
91
|
+
is the byte offset known to be durable at the far end (for an upload, the
|
|
92
|
+
position the server acknowledged), so a caller can resume from there
|
|
93
|
+
instead of starting over.
|
|
94
|
+
"""
|
|
95
|
+
super().__init__(message)
|
|
96
|
+
self.attempts = attempts
|
|
97
|
+
self.committed = committed
|
|
98
|
+
|
|
99
|
+
def __reduce__(self) -> _Reduced:
|
|
100
|
+
"""Pickle with ``attempts`` and ``committed`` intact.
|
|
101
|
+
|
|
102
|
+
``BaseException.__reduce__`` replays only ``args``, so a failure raised in
|
|
103
|
+
a worker process would arrive in the parent with its progress record
|
|
104
|
+
reset to zero.
|
|
105
|
+
"""
|
|
106
|
+
state = {"attempts": self.attempts, "committed": self.committed}
|
|
107
|
+
return rebuild, (type(self), (self.args[0],), state)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class TimeoutError(TransientError, builtins.TimeoutError):
|
|
111
|
+
"""An operation exceeded its deadline.
|
|
112
|
+
|
|
113
|
+
A timeout is a :class:`TransientError` because that is what it is - the
|
|
114
|
+
request may well succeed on the next attempt - so code that retries on
|
|
115
|
+
``TransientError`` catches it, and code that cares specifically about
|
|
116
|
+
slowness can still catch the builtin :class:`TimeoutError`.
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class AuthenticationError(XrdError):
|
|
121
|
+
"""Logging in failed: no usable credential, or the peer refused it."""
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class CredentialError(AuthenticationError):
|
|
125
|
+
"""A credential was missing, unreadable, or malformed."""
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class TokenExpiredError(CredentialError):
|
|
129
|
+
"""A bearer token is expired; the server would have rejected it."""
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class InsecureCredentialError(CredentialError, builtins.PermissionError):
|
|
133
|
+
"""A credential file that is not private to this user, and so is not used.
|
|
134
|
+
|
|
135
|
+
Raised for a proxy, a bearer-token file or a keytab that is not a regular
|
|
136
|
+
file, belongs to another user, or that group or others may read or write
|
|
137
|
+
- the same refusals ``XrdSecgsi``, ``XrdSecsss`` and the grid's own tools
|
|
138
|
+
make, because a secret anyone else can read is no longer a secret, and
|
|
139
|
+
one anyone else can write is somebody else's credential. The message
|
|
140
|
+
names the file and the command that fixes it; :attr:`path` is the file.
|
|
141
|
+
``except PermissionError`` catches it too.
|
|
142
|
+
"""
|
|
143
|
+
|
|
144
|
+
def __init__(self, message: str, path: str = "") -> None:
|
|
145
|
+
"""``path`` names the offending file, kept apart from the message for programs."""
|
|
146
|
+
# The plain one-argument form: ``OSError`` given an errno or a
|
|
147
|
+
# filename prefixes ``[Errno ...]`` to the sentence it is meant to be.
|
|
148
|
+
super().__init__(message)
|
|
149
|
+
self.path = path
|
|
150
|
+
|
|
151
|
+
def __reduce__(self) -> _Reduced:
|
|
152
|
+
"""Pickle with :attr:`path` intact."""
|
|
153
|
+
return rebuild, (type(self), (self.args[0],), {"path": self.path})
|