transcrypto 1.6.0__py3-none-any.whl → 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- transcrypto/__init__.py +7 -0
- transcrypto/aes.py +150 -44
- transcrypto/base.py +587 -442
- transcrypto/constants.py +20070 -1906
- transcrypto/dsa.py +132 -99
- transcrypto/elgamal.py +116 -84
- transcrypto/modmath.py +88 -78
- transcrypto/profiler.py +225 -175
- transcrypto/rsa.py +126 -90
- transcrypto/sss.py +122 -70
- transcrypto/transcrypto.py +2361 -1419
- {transcrypto-1.6.0.dist-info → transcrypto-1.7.0.dist-info}/METADATA +78 -58
- transcrypto-1.7.0.dist-info/RECORD +17 -0
- {transcrypto-1.6.0.dist-info → transcrypto-1.7.0.dist-info}/WHEEL +1 -2
- transcrypto-1.7.0.dist-info/entry_points.txt +4 -0
- transcrypto/safetrans.py +0 -1228
- transcrypto-1.6.0.dist-info/RECORD +0 -18
- transcrypto-1.6.0.dist-info/top_level.txt +0 -1
- {transcrypto-1.6.0.dist-info → transcrypto-1.7.0.dist-info}/licenses/LICENSE +0 -0
transcrypto/base.py
CHANGED
|
@@ -1,13 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
#
|
|
3
|
-
# Copyright 2025 Daniel Balparda (balparda@github.com) - Apache-2.0 license
|
|
4
|
-
#
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright 2026 Daniel Balparda <balparda@github.com>
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
3
|
"""Balparda's TransCrypto base library."""
|
|
6
4
|
|
|
7
5
|
from __future__ import annotations
|
|
8
6
|
|
|
9
|
-
import abc
|
|
10
|
-
import argparse
|
|
7
|
+
import abc as abstract
|
|
11
8
|
import base64
|
|
12
9
|
import codecs
|
|
13
10
|
import dataclasses
|
|
@@ -18,87 +15,104 @@ import hashlib
|
|
|
18
15
|
import json
|
|
19
16
|
import logging
|
|
20
17
|
import math
|
|
21
|
-
import os
|
|
22
|
-
import
|
|
23
|
-
#
|
|
18
|
+
import os
|
|
19
|
+
import pathlib
|
|
20
|
+
import pickle # noqa: S403
|
|
24
21
|
import secrets
|
|
25
22
|
import sys
|
|
26
|
-
import time
|
|
27
23
|
import threading
|
|
28
|
-
|
|
29
|
-
from
|
|
30
|
-
|
|
24
|
+
import time
|
|
25
|
+
from collections import abc
|
|
26
|
+
from types import TracebackType
|
|
27
|
+
from typing import (
|
|
28
|
+
Any,
|
|
29
|
+
Protocol,
|
|
30
|
+
Self,
|
|
31
|
+
TypeVar,
|
|
32
|
+
cast,
|
|
33
|
+
final,
|
|
34
|
+
runtime_checkable,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
import click
|
|
31
38
|
import numpy as np
|
|
39
|
+
import typer
|
|
40
|
+
import zstandard
|
|
41
|
+
from click import testing as click_testing
|
|
32
42
|
from rich import console as rich_console
|
|
33
43
|
from rich import logging as rich_logging
|
|
34
|
-
from scipy import stats
|
|
35
|
-
import zstandard
|
|
36
|
-
|
|
37
|
-
__author__ = 'balparda@github.com'
|
|
38
|
-
__version__ = '1.6.0' # 2026-01-15, Thu
|
|
39
|
-
__version_tuple__: tuple[int, ...] = tuple(int(v) for v in __version__.split('.'))
|
|
44
|
+
from scipy import stats
|
|
40
45
|
|
|
41
46
|
# Data conversion utils
|
|
42
47
|
|
|
43
|
-
BytesToHex: Callable[[bytes], str] = lambda b: b.hex()
|
|
44
|
-
BytesToInt: Callable[[bytes], int] = lambda b: int.from_bytes(b, 'big', signed=False)
|
|
45
|
-
BytesToEncoded: Callable[[bytes], str] = lambda b: base64.urlsafe_b64encode(b).decode('ascii')
|
|
48
|
+
BytesToHex: abc.Callable[[bytes], str] = lambda b: b.hex()
|
|
49
|
+
BytesToInt: abc.Callable[[bytes], int] = lambda b: int.from_bytes(b, 'big', signed=False)
|
|
50
|
+
BytesToEncoded: abc.Callable[[bytes], str] = lambda b: base64.urlsafe_b64encode(b).decode('ascii')
|
|
46
51
|
|
|
47
|
-
HexToBytes: Callable[[str], bytes] = bytes.fromhex
|
|
48
|
-
IntToFixedBytes: Callable[[int, int], bytes] = lambda i, n: i.to_bytes(n, 'big', signed=False)
|
|
49
|
-
IntToBytes: Callable[[int], bytes] = lambda i: IntToFixedBytes(i, (i.bit_length() + 7) // 8)
|
|
50
|
-
IntToEncoded: Callable[[int], str] = lambda i: BytesToEncoded(IntToBytes(i))
|
|
51
|
-
EncodedToBytes: Callable[[str], bytes] = lambda e: base64.urlsafe_b64decode(e.encode('ascii'))
|
|
52
|
+
HexToBytes: abc.Callable[[str], bytes] = bytes.fromhex
|
|
53
|
+
IntToFixedBytes: abc.Callable[[int, int], bytes] = lambda i, n: i.to_bytes(n, 'big', signed=False)
|
|
54
|
+
IntToBytes: abc.Callable[[int], bytes] = lambda i: IntToFixedBytes(i, (i.bit_length() + 7) // 8)
|
|
55
|
+
IntToEncoded: abc.Callable[[int], str] = lambda i: BytesToEncoded(IntToBytes(i))
|
|
56
|
+
EncodedToBytes: abc.Callable[[str], bytes] = lambda e: base64.urlsafe_b64decode(e.encode('ascii'))
|
|
52
57
|
|
|
53
|
-
PadBytesTo: Callable[[bytes, int], bytes] = lambda b, i: b.rjust((i + 7) // 8, b'\x00')
|
|
58
|
+
PadBytesTo: abc.Callable[[bytes, int], bytes] = lambda b, i: b.rjust((i + 7) // 8, b'\x00')
|
|
54
59
|
|
|
55
60
|
# Time utils
|
|
56
61
|
|
|
57
|
-
MIN_TM = int(
|
|
58
|
-
datetime.datetime(2000, 1, 1, 0, 0, 0).replace(tzinfo=datetime.timezone.utc).timestamp())
|
|
62
|
+
MIN_TM = int(datetime.datetime(2000, 1, 1, 0, 0, 0, tzinfo=datetime.UTC).timestamp())
|
|
59
63
|
TIME_FORMAT = '%Y/%b/%d-%H:%M:%S-UTC'
|
|
60
|
-
TimeStr: Callable[[int | float | None], str] = lambda tm: (
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
+
TimeStr: abc.Callable[[int | float | None], str] = lambda tm: (
|
|
65
|
+
time.strftime(TIME_FORMAT, time.gmtime(tm)) if tm else '-'
|
|
66
|
+
)
|
|
67
|
+
Now: abc.Callable[[], int] = lambda: int(time.time())
|
|
68
|
+
StrNow: abc.Callable[[], str] = lambda: TimeStr(Now())
|
|
64
69
|
|
|
65
70
|
# Logging
|
|
66
71
|
_LOG_FORMAT_NO_PROCESS: str = '%(funcName)s: %(message)s'
|
|
67
72
|
_LOG_FORMAT_WITH_PROCESS: str = '%(processName)s/' + _LOG_FORMAT_NO_PROCESS
|
|
68
73
|
_LOG_FORMAT_DATETIME: str = '[%Y%m%d-%H:%M:%S]' # e.g., [20240131-13:45:30]
|
|
69
|
-
_LOG_LEVELS:
|
|
74
|
+
_LOG_LEVELS: dict[int, int] = {
|
|
75
|
+
0: logging.ERROR,
|
|
76
|
+
1: logging.WARNING,
|
|
77
|
+
2: logging.INFO,
|
|
78
|
+
3: logging.DEBUG,
|
|
79
|
+
}
|
|
70
80
|
_LOG_COMMON_PROVIDERS: set[str] = {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
81
|
+
'werkzeug',
|
|
82
|
+
'gunicorn.error',
|
|
83
|
+
'gunicorn.access',
|
|
84
|
+
'uvicorn',
|
|
85
|
+
'uvicorn.error',
|
|
86
|
+
'uvicorn.access',
|
|
87
|
+
'django.server',
|
|
75
88
|
}
|
|
76
89
|
|
|
77
90
|
# SI prefix table, powers of 1000
|
|
78
91
|
_SI_PREFIXES: dict[int, str] = {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
+
-6: 'a', # atto
|
|
93
|
+
-5: 'f', # femto
|
|
94
|
+
-4: 'p', # pico
|
|
95
|
+
-3: 'n', # nano
|
|
96
|
+
-2: 'µ', # micro (unicode U+00B5) # noqa: RUF001
|
|
97
|
+
-1: 'm', # milli
|
|
98
|
+
0: '', # base
|
|
99
|
+
1: 'k', # kilo
|
|
100
|
+
2: 'M', # mega
|
|
101
|
+
3: 'G', # giga
|
|
102
|
+
4: 'T', # tera
|
|
103
|
+
5: 'P', # peta
|
|
104
|
+
6: 'E', # exa
|
|
92
105
|
}
|
|
93
106
|
|
|
94
107
|
# these control the pickling of data, do NOT ever change, or you will break all databases
|
|
95
108
|
# <https://docs.python.org/3/library/pickle.html#pickle.DEFAULT_PROTOCOL>
|
|
96
109
|
_PICKLE_PROTOCOL = 4 # protocol 4 available since python v3.8 # do NOT ever change!
|
|
97
|
-
PickleGeneric: Callable[[Any], bytes] = lambda o: pickle.dumps(o, protocol=_PICKLE_PROTOCOL)
|
|
98
|
-
UnpickleGeneric: Callable[[bytes], Any] = pickle.loads
|
|
99
|
-
PickleJSON: Callable[[dict[str, Any]], bytes] = lambda d: json.dumps(
|
|
100
|
-
|
|
101
|
-
|
|
110
|
+
PickleGeneric: abc.Callable[[Any], bytes] = lambda o: pickle.dumps(o, protocol=_PICKLE_PROTOCOL)
|
|
111
|
+
UnpickleGeneric: abc.Callable[[bytes], Any] = pickle.loads # noqa: S301
|
|
112
|
+
PickleJSON: abc.Callable[[dict[str, Any]], bytes] = lambda d: json.dumps(
|
|
113
|
+
d, separators=(',', ':')
|
|
114
|
+
).encode('utf-8')
|
|
115
|
+
UnpickleJSON: abc.Callable[[bytes], dict[str, Any]] = lambda b: json.loads(b.decode('utf-8'))
|
|
102
116
|
_PICKLE_AAD = b'transcrypto.base.Serialize.1.0' # do NOT ever change!
|
|
103
117
|
# these help find compressed files, do NOT change unless zstandard changes
|
|
104
118
|
_ZSTD_MAGIC_FRAME = 0xFD2FB528
|
|
@@ -106,11 +120,17 @@ _ZSTD_MAGIC_SKIPPABLE_MIN = 0x184D2A50
|
|
|
106
120
|
_ZSTD_MAGIC_SKIPPABLE_MAX = 0x184D2A5F
|
|
107
121
|
# JSON
|
|
108
122
|
_JSON_DATACLASS_TYPES: set[str] = {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
123
|
+
# native support
|
|
124
|
+
'int',
|
|
125
|
+
'float',
|
|
126
|
+
'str',
|
|
127
|
+
'bool',
|
|
128
|
+
'list[int]',
|
|
129
|
+
'list[float]',
|
|
130
|
+
'list[str]',
|
|
131
|
+
'list[bool]',
|
|
132
|
+
# need conversion/encoding
|
|
133
|
+
'bytes',
|
|
114
134
|
}
|
|
115
135
|
|
|
116
136
|
|
|
@@ -127,10 +147,10 @@ class CryptoError(Error):
|
|
|
127
147
|
|
|
128
148
|
|
|
129
149
|
class ImplementationError(Error, NotImplementedError):
|
|
130
|
-
"""
|
|
150
|
+
"""Feature is not implemented yet (TransCrypto)."""
|
|
131
151
|
|
|
132
152
|
|
|
133
|
-
__console_lock = threading.RLock()
|
|
153
|
+
__console_lock: threading.RLock = threading.RLock()
|
|
134
154
|
__console_singleton: rich_console.Console | None = None
|
|
135
155
|
|
|
136
156
|
|
|
@@ -139,6 +159,7 @@ def Console() -> rich_console.Console:
|
|
|
139
159
|
|
|
140
160
|
Returns:
|
|
141
161
|
rich.console.Console: The global console instance.
|
|
162
|
+
|
|
142
163
|
"""
|
|
143
164
|
with __console_lock:
|
|
144
165
|
if __console_singleton is None:
|
|
@@ -148,46 +169,79 @@ def Console() -> rich_console.Console:
|
|
|
148
169
|
|
|
149
170
|
def ResetConsole() -> None:
|
|
150
171
|
"""Reset the global console instance."""
|
|
151
|
-
global __console_singleton #
|
|
172
|
+
global __console_singleton # noqa: PLW0603
|
|
152
173
|
with __console_lock:
|
|
153
174
|
__console_singleton = None
|
|
154
175
|
|
|
155
176
|
|
|
156
177
|
def InitLogging(
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
178
|
+
verbosity: int,
|
|
179
|
+
/,
|
|
180
|
+
*,
|
|
181
|
+
include_process: bool = False,
|
|
182
|
+
soft_wrap: bool = False,
|
|
183
|
+
color: bool | None = False,
|
|
184
|
+
) -> tuple[rich_console.Console, int, bool]:
|
|
185
|
+
"""Initialize logger (with RichHandler) and get a rich.console.Console singleton.
|
|
186
|
+
|
|
187
|
+
This method will also return the actual decided values for verbosity and color use.
|
|
161
188
|
If you have a CLI app that uses this, its pytests should call `ResetConsole()` in a fixture, like:
|
|
162
189
|
|
|
163
|
-
from transcrypto import
|
|
190
|
+
from transcrypto import logging
|
|
164
191
|
@pytest.fixture(autouse=True)
|
|
165
|
-
def _reset_base_logging():
|
|
166
|
-
|
|
167
|
-
yield
|
|
192
|
+
def _reset_base_logging() -> Generator[None, None, None]: # type: ignore
|
|
193
|
+
logging.ResetConsole()
|
|
194
|
+
yield # stop
|
|
168
195
|
|
|
169
196
|
Args:
|
|
170
|
-
verbosity (int): Logging verbosity level
|
|
197
|
+
verbosity (int): Logging verbosity level: 0==ERROR, 1==WARNING, 2==INFO, 3==DEBUG
|
|
171
198
|
include_process (bool, optional): Whether to include process name in log output.
|
|
172
199
|
soft_wrap (bool, optional): Whether to enable soft wrapping in the console.
|
|
173
|
-
Default is False, and it means rich will hard-wrap long lines (by adding
|
|
200
|
+
Default is False, and it means rich will hard-wrap long lines (by adding line breaks).
|
|
201
|
+
color (bool | None, optional): Whether to enable/disable color output in the console.
|
|
202
|
+
If None, respects NO_COLOR env var.
|
|
174
203
|
|
|
175
204
|
Returns:
|
|
176
|
-
|
|
205
|
+
tuple[rich_console.Console, int, bool]:
|
|
206
|
+
(The initialized console instance, actual log level, actual color use)
|
|
207
|
+
|
|
208
|
+
Raises:
|
|
209
|
+
RuntimeError: if you call this more than once
|
|
210
|
+
|
|
177
211
|
"""
|
|
178
|
-
global __console_singleton #
|
|
212
|
+
global __console_singleton # noqa: PLW0603
|
|
179
213
|
with __console_lock:
|
|
180
214
|
if __console_singleton is not None:
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
215
|
+
raise RuntimeError(
|
|
216
|
+
'calling InitLogging() more than once is forbidden; '
|
|
217
|
+
'use Console() to get a console after first creation'
|
|
218
|
+
)
|
|
219
|
+
# set level
|
|
220
|
+
logging_level: int = _LOG_LEVELS.get(min(verbosity, 3), logging.ERROR)
|
|
221
|
+
# respect NO_COLOR unless the caller has already decided (treat env presence as "disable color")
|
|
222
|
+
no_color: bool = (
|
|
223
|
+
False
|
|
224
|
+
if (os.getenv('NO_COLOR') is None and color is None)
|
|
225
|
+
else ((os.getenv('NO_COLOR') is not None) if color is None else (not color))
|
|
226
|
+
)
|
|
227
|
+
# create console and configure logging
|
|
228
|
+
console = rich_console.Console(soft_wrap=soft_wrap, no_color=no_color)
|
|
184
229
|
logging.basicConfig(
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
230
|
+
level=logging_level,
|
|
231
|
+
format=_LOG_FORMAT_WITH_PROCESS if include_process else _LOG_FORMAT_NO_PROCESS,
|
|
232
|
+
datefmt=_LOG_FORMAT_DATETIME,
|
|
233
|
+
handlers=[
|
|
234
|
+
rich_logging.RichHandler( # we show name/line, but want time & level
|
|
235
|
+
console=console,
|
|
236
|
+
rich_tracebacks=True,
|
|
237
|
+
show_time=True,
|
|
238
|
+
show_level=True,
|
|
239
|
+
show_path=True,
|
|
240
|
+
),
|
|
241
|
+
],
|
|
242
|
+
force=True, # force=True to override any previous logging config
|
|
243
|
+
)
|
|
244
|
+
# configure common loggers
|
|
191
245
|
logging.captureWarnings(True)
|
|
192
246
|
for name in _LOG_COMMON_PROVIDERS:
|
|
193
247
|
log: logging.Logger = logging.getLogger(name)
|
|
@@ -195,11 +249,14 @@ def InitLogging(
|
|
|
195
249
|
log.propagate = True
|
|
196
250
|
log.setLevel(logging_level)
|
|
197
251
|
__console_singleton = console # need a global statement to re-bind this one
|
|
198
|
-
logging.info(
|
|
199
|
-
|
|
252
|
+
logging.info(
|
|
253
|
+
f'Logging initialized at level {logging.getLevelName(logging_level)} / '
|
|
254
|
+
f'{"NO " if no_color else ""}COLOR'
|
|
255
|
+
)
|
|
256
|
+
return (console, logging_level, not no_color)
|
|
200
257
|
|
|
201
258
|
|
|
202
|
-
def HumanizedBytes(inp_sz:
|
|
259
|
+
def HumanizedBytes(inp_sz: float, /) -> str: # noqa: PLR0911
|
|
203
260
|
"""Convert a byte count into a human-readable string using binary prefixes (powers of 1024).
|
|
204
261
|
|
|
205
262
|
Scales the input size by powers of 1024, returning a value with the
|
|
@@ -231,10 +288,11 @@ def HumanizedBytes(inp_sz: int | float, /) -> str: # pylint: disable=too-many-r
|
|
|
231
288
|
'2.00 KiB'
|
|
232
289
|
>>> HumanizedBytes(5 * 1024**3)
|
|
233
290
|
'5.00 GiB'
|
|
291
|
+
|
|
234
292
|
"""
|
|
235
293
|
if inp_sz < 0:
|
|
236
294
|
raise InputError(f'input should be >=0 and got {inp_sz}')
|
|
237
|
-
if inp_sz < 1024:
|
|
295
|
+
if inp_sz < 1024: # noqa: PLR2004
|
|
238
296
|
return f'{inp_sz} B' if isinstance(inp_sz, int) else f'{inp_sz:0.3f} B'
|
|
239
297
|
if inp_sz < 1024 * 1024:
|
|
240
298
|
return f'{(inp_sz / 1024):0.3f} KiB'
|
|
@@ -249,7 +307,7 @@ def HumanizedBytes(inp_sz: int | float, /) -> str: # pylint: disable=too-many-r
|
|
|
249
307
|
return f'{(inp_sz / (1024 * 1024 * 1024 * 1024 * 1024 * 1024)):0.3f} EiB'
|
|
250
308
|
|
|
251
309
|
|
|
252
|
-
def HumanizedDecimal(inp_sz:
|
|
310
|
+
def HumanizedDecimal(inp_sz: float, /, *, unit: str = '') -> str:
|
|
253
311
|
"""Convert a numeric value into a human-readable string using SI metric prefixes.
|
|
254
312
|
|
|
255
313
|
Scales the input value by powers of 1000, returning a value with the
|
|
@@ -286,7 +344,8 @@ def HumanizedDecimal(inp_sz: int | float, /, *, unit: str = '') -> str:
|
|
|
286
344
|
|
|
287
345
|
Raises:
|
|
288
346
|
InputError: If `inp_sz` is not finite.
|
|
289
|
-
|
|
347
|
+
|
|
348
|
+
""" # noqa: RUF002
|
|
290
349
|
if not math.isfinite(inp_sz):
|
|
291
350
|
raise InputError(f'input should finite; got {inp_sz!r}')
|
|
292
351
|
unit = unit.strip()
|
|
@@ -296,8 +355,7 @@ def HumanizedDecimal(inp_sz: int | float, /, *, unit: str = '') -> str:
|
|
|
296
355
|
neg: str = '-' if inp_sz < 0 else ''
|
|
297
356
|
inp_sz = abs(inp_sz)
|
|
298
357
|
# Find exponent of 1000 that keeps value in [1, 1000)
|
|
299
|
-
exp: int
|
|
300
|
-
exp = int(math.floor(math.log10(abs(inp_sz)) / 3))
|
|
358
|
+
exp: int = math.floor(math.log10(abs(inp_sz)) / 3)
|
|
301
359
|
exp = max(min(exp, max(_SI_PREFIXES)), min(_SI_PREFIXES)) # clamp to supported range
|
|
302
360
|
if not exp:
|
|
303
361
|
# No scaling: use int or 4-decimal float
|
|
@@ -305,12 +363,12 @@ def HumanizedDecimal(inp_sz: int | float, /, *, unit: str = '') -> str:
|
|
|
305
363
|
return f'{neg}{int(inp_sz)}{pad_unit}'
|
|
306
364
|
return f'{neg}{inp_sz:0.3f}{pad_unit}'
|
|
307
365
|
# scaled
|
|
308
|
-
scaled: float = inp_sz / (1000
|
|
366
|
+
scaled: float = inp_sz / (1000**exp)
|
|
309
367
|
prefix: str = _SI_PREFIXES[exp]
|
|
310
368
|
return f'{neg}{scaled:0.3f} {prefix}{unit}'
|
|
311
369
|
|
|
312
370
|
|
|
313
|
-
def HumanizedSeconds(inp_secs:
|
|
371
|
+
def HumanizedSeconds(inp_secs: float, /) -> str: # noqa: PLR0911
|
|
314
372
|
"""Convert a duration in seconds into a human-readable time string.
|
|
315
373
|
|
|
316
374
|
Selects the appropriate time unit based on the duration's magnitude:
|
|
@@ -351,17 +409,18 @@ def HumanizedSeconds(inp_secs: int | float, /) -> str: # pylint: disable=too-ma
|
|
|
351
409
|
'42.00 s'
|
|
352
410
|
>>> HumanizedSeconds(3661)
|
|
353
411
|
'1.02 h'
|
|
354
|
-
|
|
412
|
+
|
|
413
|
+
""" # noqa: RUF002
|
|
355
414
|
if not math.isfinite(inp_secs) or inp_secs < 0:
|
|
356
415
|
raise InputError(f'input should be >=0 and got {inp_secs}')
|
|
357
416
|
if inp_secs == 0:
|
|
358
417
|
return '0.000 s'
|
|
359
418
|
inp_secs = float(inp_secs)
|
|
360
|
-
if inp_secs < 0.001:
|
|
361
|
-
return f'{inp_secs * 1000 * 1000:0.3f} µs'
|
|
419
|
+
if inp_secs < 0.001: # noqa: PLR2004
|
|
420
|
+
return f'{inp_secs * 1000 * 1000:0.3f} µs' # noqa: RUF001
|
|
362
421
|
if inp_secs < 1:
|
|
363
422
|
return f'{inp_secs * 1000:0.3f} ms'
|
|
364
|
-
if inp_secs < 60:
|
|
423
|
+
if inp_secs < 60: # noqa: PLR2004
|
|
365
424
|
return f'{inp_secs:0.3f} s'
|
|
366
425
|
if inp_secs < 60 * 60:
|
|
367
426
|
return f'{(inp_secs / 60):0.3f} min'
|
|
@@ -371,8 +430,8 @@ def HumanizedSeconds(inp_secs: int | float, /) -> str: # pylint: disable=too-ma
|
|
|
371
430
|
|
|
372
431
|
|
|
373
432
|
def MeasurementStats(
|
|
374
|
-
|
|
375
|
-
|
|
433
|
+
data: list[int | float], /, *, confidence: float = 0.95
|
|
434
|
+
) -> tuple[int, float, float, float, tuple[float, float], float]:
|
|
376
435
|
"""Compute descriptive statistics for repeated measurements.
|
|
377
436
|
|
|
378
437
|
Given N ≥ 1 measurements, this function computes the sample mean, the
|
|
@@ -401,12 +460,13 @@ def MeasurementStats(
|
|
|
401
460
|
|
|
402
461
|
Raises:
|
|
403
462
|
InputError: if the input list is empty.
|
|
463
|
+
|
|
404
464
|
"""
|
|
405
465
|
# test inputs
|
|
406
466
|
n: int = len(data)
|
|
407
467
|
if not n:
|
|
408
468
|
raise InputError('no data')
|
|
409
|
-
if not 0.5 <= confidence < 1.0:
|
|
469
|
+
if not 0.5 <= confidence < 1.0: # noqa: PLR2004
|
|
410
470
|
raise InputError(f'invalid confidence: {confidence=}')
|
|
411
471
|
# solve trivial case
|
|
412
472
|
if n == 1:
|
|
@@ -414,17 +474,22 @@ def MeasurementStats(
|
|
|
414
474
|
# call scipy for the science data
|
|
415
475
|
np_data = np.array(data)
|
|
416
476
|
mean = np.mean(np_data)
|
|
417
|
-
sem = stats.sem(np_data)
|
|
418
|
-
ci = stats.t.interval(confidence, n - 1, loc=mean, scale=sem)
|
|
419
|
-
t_crit = stats.t.ppf((1.0 + confidence) / 2.0, n - 1)
|
|
420
|
-
error = t_crit * sem # half-width of the CI
|
|
421
|
-
return (n, float(mean), float(sem), float(error), (float(ci[0]), float(ci[1])), confidence)
|
|
477
|
+
sem = stats.sem(np_data)
|
|
478
|
+
ci = stats.t.interval(confidence, n - 1, loc=mean, scale=sem)
|
|
479
|
+
t_crit = stats.t.ppf((1.0 + confidence) / 2.0, n - 1)
|
|
480
|
+
error = t_crit * sem # half-width of the CI
|
|
481
|
+
return (n, float(mean), float(sem), float(error), (float(ci[0]), float(ci[1])), confidence)
|
|
422
482
|
|
|
423
483
|
|
|
424
484
|
def HumanizedMeasurements(
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
485
|
+
data: list[int | float],
|
|
486
|
+
/,
|
|
487
|
+
*,
|
|
488
|
+
unit: str = '',
|
|
489
|
+
parser: abc.Callable[[float], str] | None = None,
|
|
490
|
+
clip_negative: bool = True,
|
|
491
|
+
confidence: float = 0.95,
|
|
492
|
+
) -> str:
|
|
428
493
|
"""Render measurement statistics as a human-readable string.
|
|
429
494
|
|
|
430
495
|
Uses `MeasurementStats()` to compute mean and uncertainty, and formats the
|
|
@@ -449,6 +514,7 @@ def HumanizedMeasurements(
|
|
|
449
514
|
|
|
450
515
|
Returns:
|
|
451
516
|
str: A formatted summary string, e.g.: '9.720 ± 1.831 ms [5.253 … 14.187]95%CI@5'
|
|
517
|
+
|
|
452
518
|
"""
|
|
453
519
|
n: int
|
|
454
520
|
mean: float
|
|
@@ -457,12 +523,14 @@ def HumanizedMeasurements(
|
|
|
457
523
|
conf: float
|
|
458
524
|
unit = unit.strip()
|
|
459
525
|
n, mean, _, error, ci, conf = MeasurementStats(data, confidence=confidence)
|
|
460
|
-
f: Callable[[float], str] = lambda x: (
|
|
461
|
-
|
|
462
|
-
|
|
526
|
+
f: abc.Callable[[float], str] = lambda x: (
|
|
527
|
+
('*0' if clip_negative and x < 0.0 else str(x))
|
|
528
|
+
if parser is None
|
|
529
|
+
else (f'*{parser(0.0)}' if clip_negative and x < 0.0 else parser(x))
|
|
530
|
+
)
|
|
463
531
|
if n == 1:
|
|
464
532
|
return f'{f(mean)}{unit} ±? @1'
|
|
465
|
-
pct =
|
|
533
|
+
pct: int = round(conf * 100)
|
|
466
534
|
return f'{f(mean)}{unit} ± {f(error)}{unit} [{f(ci[0])}{unit} … {f(ci[1])}{unit}]{pct}%CI@{n}'
|
|
467
535
|
|
|
468
536
|
|
|
@@ -470,7 +538,6 @@ class Timer:
|
|
|
470
538
|
"""An execution timing class that can be used as both a context manager and a decorator.
|
|
471
539
|
|
|
472
540
|
Examples:
|
|
473
|
-
|
|
474
541
|
# As a context manager
|
|
475
542
|
with Timer('Block timing'):
|
|
476
543
|
time.sleep(1.2)
|
|
@@ -491,11 +558,12 @@ class Timer:
|
|
|
491
558
|
label (str, optional): Timer label
|
|
492
559
|
emit_log (bool, optional): If True (default) will logging.info() the timer, else will not
|
|
493
560
|
emit_print (bool, optional): If True will print() the timer, else (default) will not
|
|
561
|
+
|
|
494
562
|
"""
|
|
495
563
|
|
|
496
564
|
def __init__(
|
|
497
|
-
|
|
498
|
-
|
|
565
|
+
self, label: str = '', /, *, emit_log: bool = True, emit_print: bool = False
|
|
566
|
+
) -> None:
|
|
499
567
|
"""Initialize the Timer.
|
|
500
568
|
|
|
501
569
|
Args:
|
|
@@ -503,8 +571,6 @@ class Timer:
|
|
|
503
571
|
emit_log (bool, optional): Emit a log message when finished; default is True
|
|
504
572
|
emit_print (bool, optional): Emit a print() message when finished; default is False
|
|
505
573
|
|
|
506
|
-
Raises:
|
|
507
|
-
InputError: empty label
|
|
508
574
|
"""
|
|
509
575
|
self.emit_log: bool = emit_log
|
|
510
576
|
self.emit_print: bool = emit_print
|
|
@@ -514,7 +580,15 @@ class Timer:
|
|
|
514
580
|
|
|
515
581
|
@property
|
|
516
582
|
def elapsed(self) -> float:
|
|
517
|
-
"""Elapsed time. Will be zero until a measurement is available with start/end.
|
|
583
|
+
"""Elapsed time. Will be zero until a measurement is available with start/end.
|
|
584
|
+
|
|
585
|
+
Raises:
|
|
586
|
+
Error: negative elapsed time
|
|
587
|
+
|
|
588
|
+
Returns:
|
|
589
|
+
float: elapsed time, in seconds
|
|
590
|
+
|
|
591
|
+
"""
|
|
518
592
|
if self.start is None or self.end is None:
|
|
519
593
|
return 0.0
|
|
520
594
|
delta: float = self.end - self.start
|
|
@@ -523,27 +597,48 @@ class Timer:
|
|
|
523
597
|
return delta
|
|
524
598
|
|
|
525
599
|
def __str__(self) -> str:
|
|
526
|
-
"""
|
|
600
|
+
"""Get current timer value.
|
|
601
|
+
|
|
602
|
+
Returns:
|
|
603
|
+
str: human-readable representation of current time value
|
|
604
|
+
|
|
605
|
+
"""
|
|
527
606
|
if self.start is None:
|
|
528
607
|
return f'{self.label}: <UNSTARTED>' if self.label else '<UNSTARTED>'
|
|
529
608
|
if self.end is None:
|
|
530
|
-
return (
|
|
531
|
-
|
|
609
|
+
return (
|
|
610
|
+
f'{self.label}: ' if self.label else ''
|
|
611
|
+
) + f'<PARTIAL> {HumanizedSeconds(time.perf_counter() - self.start)}'
|
|
532
612
|
return (f'{self.label}: ' if self.label else '') + f'{HumanizedSeconds(self.elapsed)}'
|
|
533
613
|
|
|
534
614
|
def Start(self) -> None:
|
|
535
|
-
"""Start the timer.
|
|
615
|
+
"""Start the timer.
|
|
616
|
+
|
|
617
|
+
Raises:
|
|
618
|
+
Error: if you try to re-start the timer
|
|
619
|
+
|
|
620
|
+
"""
|
|
536
621
|
if self.start is not None:
|
|
537
622
|
raise Error('Re-starting timer is forbidden')
|
|
538
623
|
self.start = time.perf_counter()
|
|
539
624
|
|
|
540
|
-
def __enter__(self) ->
|
|
541
|
-
"""Start the timer when entering the context.
|
|
625
|
+
def __enter__(self) -> Self:
|
|
626
|
+
"""Start the timer when entering the context.
|
|
627
|
+
|
|
628
|
+
Returns:
|
|
629
|
+
Timer: context object (self)
|
|
630
|
+
|
|
631
|
+
"""
|
|
542
632
|
self.Start()
|
|
543
633
|
return self
|
|
544
634
|
|
|
545
635
|
def Stop(self) -> None:
|
|
546
|
-
"""Stop the timer and emit logging.info with timer message.
|
|
636
|
+
"""Stop the timer and emit logging.info with timer message.
|
|
637
|
+
|
|
638
|
+
Raises:
|
|
639
|
+
Error: trying to re-start timer or stop unstarted timer
|
|
640
|
+
|
|
641
|
+
"""
|
|
547
642
|
if self.start is None:
|
|
548
643
|
raise Error('Stopping an unstarted timer')
|
|
549
644
|
if self.end is not None:
|
|
@@ -556,18 +651,15 @@ class Timer:
|
|
|
556
651
|
Console().print(message)
|
|
557
652
|
|
|
558
653
|
def __exit__(
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
exc_val (BaseException | None): Exception value, if any.
|
|
566
|
-
exc_tb (Any): Traceback object, if any.
|
|
567
|
-
"""
|
|
654
|
+
self,
|
|
655
|
+
unused_exc_type: type[BaseException] | None,
|
|
656
|
+
unused_exc_val: BaseException | None,
|
|
657
|
+
exc_tb: TracebackType | None,
|
|
658
|
+
) -> None:
|
|
659
|
+
"""Stop the timer when exiting the context."""
|
|
568
660
|
self.Stop()
|
|
569
661
|
|
|
570
|
-
_F = TypeVar('_F', bound=Callable[..., Any])
|
|
662
|
+
_F = TypeVar('_F', bound=abc.Callable[..., Any])
|
|
571
663
|
|
|
572
664
|
def __call__(self, func: Timer._F) -> Timer._F:
|
|
573
665
|
"""Allow the Timer to be used as a decorator.
|
|
@@ -577,10 +669,11 @@ class Timer:
|
|
|
577
669
|
|
|
578
670
|
Returns:
|
|
579
671
|
The wrapped function with timing behavior.
|
|
672
|
+
|
|
580
673
|
"""
|
|
581
674
|
|
|
582
675
|
@functools.wraps(func)
|
|
583
|
-
def _Wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
676
|
+
def _Wrapper(*args: Any, **kwargs: Any) -> Any: # noqa: ANN401
|
|
584
677
|
with self.__class__(self.label, emit_log=self.emit_log, emit_print=self.emit_print):
|
|
585
678
|
return func(*args, **kwargs)
|
|
586
679
|
|
|
@@ -602,9 +695,10 @@ def RandBits(n_bits: int, /) -> int:
|
|
|
602
695
|
|
|
603
696
|
Raises:
|
|
604
697
|
InputError: invalid n_bits
|
|
698
|
+
|
|
605
699
|
"""
|
|
606
700
|
# test inputs
|
|
607
|
-
if n_bits < 8:
|
|
701
|
+
if n_bits < 8: # noqa: PLR2004
|
|
608
702
|
raise InputError(f'n_bits must be ≥ 8: {n_bits}')
|
|
609
703
|
# call underlying method
|
|
610
704
|
n: int = 0
|
|
@@ -625,6 +719,7 @@ def RandInt(min_int: int, max_int: int, /) -> int:
|
|
|
625
719
|
|
|
626
720
|
Raises:
|
|
627
721
|
InputError: invalid min/max
|
|
722
|
+
|
|
628
723
|
"""
|
|
629
724
|
# test inputs
|
|
630
725
|
if min_int < 0 or min_int >= max_int:
|
|
@@ -632,11 +727,11 @@ def RandInt(min_int: int, max_int: int, /) -> int:
|
|
|
632
727
|
# uniform over [min_int, max_int]
|
|
633
728
|
span: int = max_int - min_int + 1
|
|
634
729
|
n: int = min_int + secrets.randbelow(span)
|
|
635
|
-
assert min_int <= n <= max_int, 'should never happen: generated number out of range'
|
|
730
|
+
assert min_int <= n <= max_int, 'should never happen: generated number out of range' # noqa: S101
|
|
636
731
|
return n
|
|
637
732
|
|
|
638
733
|
|
|
639
|
-
def RandShuffle[T: Any](seq: MutableSequence[T], /) -> None:
|
|
734
|
+
def RandShuffle[T: Any](seq: abc.MutableSequence[T], /) -> None:
|
|
640
735
|
"""In-place Crypto-random shuffle order for `seq` mutable sequence.
|
|
641
736
|
|
|
642
737
|
Args:
|
|
@@ -644,11 +739,12 @@ def RandShuffle[T: Any](seq: MutableSequence[T], /) -> None:
|
|
|
644
739
|
|
|
645
740
|
Raises:
|
|
646
741
|
InputError: not enough elements
|
|
742
|
+
|
|
647
743
|
"""
|
|
648
744
|
# test inputs
|
|
649
|
-
if (n_seq := len(seq)) < 2:
|
|
745
|
+
if (n_seq := len(seq)) < 2: # noqa: PLR2004
|
|
650
746
|
raise InputError(f'seq must have 2 or more elements: {n_seq}')
|
|
651
|
-
# cryptographically sound Fisher
|
|
747
|
+
# cryptographically sound Fisher-Yates using secrets.randbelow
|
|
652
748
|
for i in range(n_seq - 1, 0, -1):
|
|
653
749
|
j: int = secrets.randbelow(i + 1)
|
|
654
750
|
seq[i], seq[j] = seq[j], seq[i]
|
|
@@ -665,13 +761,14 @@ def RandBytes(n_bytes: int, /) -> bytes:
|
|
|
665
761
|
|
|
666
762
|
Raises:
|
|
667
763
|
InputError: invalid n_bytes
|
|
764
|
+
|
|
668
765
|
"""
|
|
669
766
|
# test inputs
|
|
670
767
|
if n_bytes < 1:
|
|
671
768
|
raise InputError(f'n_bytes must be ≥ 1: {n_bytes}')
|
|
672
769
|
# return from system call
|
|
673
770
|
b: bytes = secrets.token_bytes(n_bytes)
|
|
674
|
-
assert len(b) == n_bytes, 'should never happen: generated bytes incorrect size'
|
|
771
|
+
assert len(b) == n_bytes, 'should never happen: generated bytes incorrect size' # noqa: S101
|
|
675
772
|
return b
|
|
676
773
|
|
|
677
774
|
|
|
@@ -689,6 +786,7 @@ def GCD(a: int, b: int, /) -> int:
|
|
|
689
786
|
|
|
690
787
|
Raises:
|
|
691
788
|
InputError: invalid inputs
|
|
789
|
+
|
|
692
790
|
"""
|
|
693
791
|
# test inputs
|
|
694
792
|
if a < 0 or b < 0 or (not a and not b):
|
|
@@ -718,6 +816,7 @@ def ExtendedGCD(a: int, b: int, /) -> tuple[int, int, int]:
|
|
|
718
816
|
|
|
719
817
|
Raises:
|
|
720
818
|
InputError: invalid inputs
|
|
819
|
+
|
|
721
820
|
"""
|
|
722
821
|
# test inputs
|
|
723
822
|
if a < 0 or b < 0 or (not a and not b):
|
|
@@ -749,6 +848,7 @@ def Hash256(data: bytes, /) -> bytes:
|
|
|
749
848
|
32 bytes (256 bits) of SHA-256 hash;
|
|
750
849
|
if converted to hexadecimal (with BytesToHex() or hex()) will be 64 chars of string;
|
|
751
850
|
if converted to int (big-endian, unsigned, with BytesToInt()) will be 0 ≤ i < 2**256
|
|
851
|
+
|
|
752
852
|
"""
|
|
753
853
|
return hashlib.sha256(data).digest()
|
|
754
854
|
|
|
@@ -763,6 +863,7 @@ def Hash512(data: bytes, /) -> bytes:
|
|
|
763
863
|
64 bytes (512 bits) of SHA-512 hash;
|
|
764
864
|
if converted to hexadecimal (with BytesToHex() or hex()) will be 128 chars of string;
|
|
765
865
|
if converted to int (big-endian, unsigned, with BytesToInt()) will be 0 ≤ i < 2**512
|
|
866
|
+
|
|
766
867
|
"""
|
|
767
868
|
return hashlib.sha512(data).digest()
|
|
768
869
|
|
|
@@ -781,17 +882,18 @@ def FileHash(full_path: str, /, *, digest: str = 'sha256') -> bytes:
|
|
|
781
882
|
|
|
782
883
|
Raises:
|
|
783
884
|
InputError: file could not be found
|
|
885
|
+
|
|
784
886
|
"""
|
|
785
887
|
# test inputs
|
|
786
888
|
digest = digest.lower().strip().replace('-', '') # normalize so we can accept e.g. "SHA-256"
|
|
787
|
-
if digest not in
|
|
889
|
+
if digest not in {'sha256', 'sha512'}:
|
|
788
890
|
raise InputError(f'unrecognized digest: {digest!r}')
|
|
789
891
|
full_path = full_path.strip()
|
|
790
|
-
if not full_path or not
|
|
892
|
+
if not full_path or not pathlib.Path(full_path).exists():
|
|
791
893
|
raise InputError(f'file {full_path!r} not found for hashing')
|
|
792
894
|
# compute hash
|
|
793
895
|
logging.info(f'Hashing file {full_path!r}')
|
|
794
|
-
with open(
|
|
896
|
+
with pathlib.Path(full_path).open('rb') as file_obj:
|
|
795
897
|
return hashlib.file_digest(file_obj, digest).digest()
|
|
796
898
|
|
|
797
899
|
|
|
@@ -805,31 +907,36 @@ def ObfuscateSecret(data: str | bytes | int, /) -> str:
|
|
|
805
907
|
Args:
|
|
806
908
|
data (str | bytes | int): Data to obfuscate
|
|
807
909
|
|
|
910
|
+
Raises:
|
|
911
|
+
InputError: _description_
|
|
912
|
+
|
|
808
913
|
Returns:
|
|
809
|
-
|
|
914
|
+
str: obfuscated string, e.g. "aabbccdd…"
|
|
915
|
+
|
|
810
916
|
"""
|
|
811
917
|
if isinstance(data, str):
|
|
812
918
|
data = data.encode('utf-8')
|
|
813
919
|
elif isinstance(data, int):
|
|
814
920
|
data = IntToBytes(data)
|
|
815
|
-
if not isinstance(data, bytes):
|
|
921
|
+
if not isinstance(data, bytes): # pyright: ignore[reportUnnecessaryIsInstance]
|
|
816
922
|
raise InputError(f'invalid type for data: {type(data)}')
|
|
817
923
|
return BytesToHex(Hash512(data))[:8] + '…'
|
|
818
924
|
|
|
819
925
|
|
|
820
926
|
class CryptoInputType(enum.StrEnum):
|
|
821
927
|
"""Types of inputs that can represent arbitrary bytes."""
|
|
928
|
+
|
|
822
929
|
# prefixes; format prefixes are all 4 bytes
|
|
823
|
-
PATH = '@'
|
|
824
|
-
STDIN = '@-'
|
|
825
|
-
HEX = 'hex:'
|
|
930
|
+
PATH = '@' # @path on disk → read bytes from a file
|
|
931
|
+
STDIN = '@-' # stdin
|
|
932
|
+
HEX = 'hex:' # hex:deadbeef → decode hex
|
|
826
933
|
BASE64 = 'b64:' # b64:... → decode base64
|
|
827
|
-
STR = 'str:'
|
|
828
|
-
RAW = 'raw:'
|
|
934
|
+
STR = 'str:' # str:hello → UTF-8 encode the literal
|
|
935
|
+
RAW = 'raw:' # raw:... → byte literals via \\xNN escapes (rare but handy)
|
|
829
936
|
|
|
830
937
|
|
|
831
938
|
def BytesToRaw(b: bytes, /) -> str:
|
|
832
|
-
"""Convert bytes to double-quoted string with \\xNN escapes where needed.
|
|
939
|
+
r"""Convert bytes to double-quoted string with \\xNN escapes where needed.
|
|
833
940
|
|
|
834
941
|
1. map bytes 0..255 to same code points (latin1)
|
|
835
942
|
2. escape non-printables/backslash/quotes via unicode_escape
|
|
@@ -839,21 +946,23 @@ def BytesToRaw(b: bytes, /) -> str:
|
|
|
839
946
|
|
|
840
947
|
Returns:
|
|
841
948
|
str: double-quoted string with \\xNN escapes where needed
|
|
949
|
+
|
|
842
950
|
"""
|
|
843
951
|
inner: str = b.decode('latin1').encode('unicode_escape').decode('ascii')
|
|
844
|
-
return f'"{inner.replace('"', r
|
|
952
|
+
return f'"{inner.replace('"', r"\"")}"'
|
|
845
953
|
|
|
846
954
|
|
|
847
955
|
def RawToBytes(s: str, /) -> bytes:
|
|
848
|
-
"""Convert double-quoted string with \\xNN escapes where needed to bytes.
|
|
956
|
+
r"""Convert double-quoted string with \\xNN escapes where needed to bytes.
|
|
849
957
|
|
|
850
958
|
Args:
|
|
851
959
|
s (str): input (expects a double-quoted string; parses \\xNN, \n, \\ etc)
|
|
852
960
|
|
|
853
961
|
Returns:
|
|
854
962
|
bytes: data
|
|
963
|
+
|
|
855
964
|
"""
|
|
856
|
-
if len(s) >= 2 and s[0] == s[-1] == '"':
|
|
965
|
+
if len(s) >= 2 and s[0] == s[-1] == '"': # noqa: PLR2004
|
|
857
966
|
s = s[1:-1]
|
|
858
967
|
# decode backslash escapes to code points, then map 0..255 -> bytes
|
|
859
968
|
return codecs.decode(s, 'unicode_escape').encode('latin1')
|
|
@@ -868,21 +977,23 @@ def DetectInputType(data_str: str, /) -> CryptoInputType | None:
|
|
|
868
977
|
Returns:
|
|
869
978
|
CryptoInputType | None: type if has a known prefix, None otherwise
|
|
870
979
|
|
|
871
|
-
Raises:
|
|
872
|
-
InputError: unexpected type or conversion error
|
|
873
980
|
"""
|
|
874
981
|
data_str = data_str.strip()
|
|
875
982
|
if data_str == CryptoInputType.STDIN:
|
|
876
983
|
return CryptoInputType.STDIN
|
|
877
984
|
for t in (
|
|
878
|
-
|
|
879
|
-
|
|
985
|
+
CryptoInputType.PATH,
|
|
986
|
+
CryptoInputType.STR,
|
|
987
|
+
CryptoInputType.HEX,
|
|
988
|
+
CryptoInputType.BASE64,
|
|
989
|
+
CryptoInputType.RAW,
|
|
990
|
+
):
|
|
880
991
|
if data_str.startswith(t):
|
|
881
992
|
return t
|
|
882
993
|
return None
|
|
883
994
|
|
|
884
995
|
|
|
885
|
-
def BytesFromInput(data_str: str, /, *, expect: CryptoInputType | None = None) -> bytes: #
|
|
996
|
+
def BytesFromInput(data_str: str, /, *, expect: CryptoInputType | None = None) -> bytes: # noqa: C901, PLR0911, PLR0912
|
|
886
997
|
"""Parse input `data_str` into `bytes`. May auto-detect or enforce a type of input.
|
|
887
998
|
|
|
888
999
|
Can load from disk ('@'). Can load from stdin ('@-').
|
|
@@ -899,6 +1010,7 @@ def BytesFromInput(data_str: str, /, *, expect: CryptoInputType | None = None) -
|
|
|
899
1010
|
|
|
900
1011
|
Raises:
|
|
901
1012
|
InputError: unexpected type or conversion error
|
|
1013
|
+
|
|
902
1014
|
"""
|
|
903
1015
|
data_str = data_str.strip()
|
|
904
1016
|
# auto-detect
|
|
@@ -908,8 +1020,8 @@ def BytesFromInput(data_str: str, /, *, expect: CryptoInputType | None = None) -
|
|
|
908
1020
|
raise InputError(f'Expected type {expect=} is different from detected type {detected_type=}')
|
|
909
1021
|
# now we know they don't conflict, so unify them; remove prefix if we have it
|
|
910
1022
|
expect = detected_type if expect is None else expect
|
|
911
|
-
assert expect is not None, 'should never happen: type should be known here'
|
|
912
|
-
data_str = data_str
|
|
1023
|
+
assert expect is not None, 'should never happen: type should be known here' # noqa: S101
|
|
1024
|
+
data_str = data_str.removeprefix(expect)
|
|
913
1025
|
# for every type something different will happen now
|
|
914
1026
|
try:
|
|
915
1027
|
match expect:
|
|
@@ -919,18 +1031,17 @@ def BytesFromInput(data_str: str, /, *, expect: CryptoInputType | None = None) -
|
|
|
919
1031
|
stream = getattr(sys.stdin, 'buffer', None)
|
|
920
1032
|
if stream is None:
|
|
921
1033
|
text: str = sys.stdin.read()
|
|
922
|
-
if not isinstance(text, str): #
|
|
923
|
-
raise InputError('sys.stdin.read() produced non-text data')
|
|
1034
|
+
if not isinstance(text, str): # pyright: ignore[reportUnnecessaryIsInstance]
|
|
1035
|
+
raise InputError('sys.stdin.read() produced non-text data') # noqa: TRY301
|
|
924
1036
|
return text.encode('utf-8')
|
|
925
1037
|
data: bytes = stream.read()
|
|
926
|
-
if not isinstance(data, bytes): #
|
|
927
|
-
raise InputError('sys.stdin.buffer.read() produced non-binary data')
|
|
1038
|
+
if not isinstance(data, bytes): # pyright: ignore[reportUnnecessaryIsInstance]
|
|
1039
|
+
raise InputError('sys.stdin.buffer.read() produced non-binary data') # noqa: TRY301
|
|
928
1040
|
return data
|
|
929
1041
|
case CryptoInputType.PATH:
|
|
930
|
-
if not
|
|
931
|
-
raise InputError(f'cannot find file {data_str!r}')
|
|
932
|
-
|
|
933
|
-
return file_obj.read()
|
|
1042
|
+
if not pathlib.Path(data_str).exists():
|
|
1043
|
+
raise InputError(f'cannot find file {data_str!r}') # noqa: TRY301
|
|
1044
|
+
return pathlib.Path(data_str).read_bytes()
|
|
934
1045
|
case CryptoInputType.STR:
|
|
935
1046
|
return data_str.encode('utf-8')
|
|
936
1047
|
case CryptoInputType.HEX:
|
|
@@ -940,24 +1051,27 @@ def BytesFromInput(data_str: str, /, *, expect: CryptoInputType | None = None) -
|
|
|
940
1051
|
case CryptoInputType.RAW:
|
|
941
1052
|
return RawToBytes(data_str)
|
|
942
1053
|
case _:
|
|
943
|
-
raise InputError(f'invalid type {expect!r}')
|
|
1054
|
+
raise InputError(f'invalid type {expect!r}') # noqa: TRY301
|
|
944
1055
|
except Exception as err:
|
|
945
1056
|
raise InputError(f'invalid input: {err}') from err
|
|
946
1057
|
|
|
947
1058
|
|
|
948
1059
|
@dataclasses.dataclass(kw_only=True, slots=True, frozen=True, repr=False)
|
|
949
|
-
class CryptoKey(
|
|
1060
|
+
class CryptoKey(abstract.ABC):
|
|
950
1061
|
"""A cryptographic key."""
|
|
951
1062
|
|
|
1063
|
+
@abstract.abstractmethod
|
|
952
1064
|
def __post_init__(self) -> None:
|
|
953
1065
|
"""Check data."""
|
|
1066
|
+
# every sub-class of CryptoKey has to implement its own version of __post_init__()
|
|
954
1067
|
|
|
955
|
-
@
|
|
1068
|
+
@abstract.abstractmethod
|
|
956
1069
|
def __str__(self) -> str:
|
|
957
1070
|
"""Safe (no secrets) string representation of the key.
|
|
958
1071
|
|
|
959
1072
|
Returns:
|
|
960
1073
|
string representation of the key without leaking secrets
|
|
1074
|
+
|
|
961
1075
|
"""
|
|
962
1076
|
# every sub-class of CryptoKey has to implement its own version of __str__()
|
|
963
1077
|
|
|
@@ -967,6 +1081,7 @@ class CryptoKey(abc.ABC):
|
|
|
967
1081
|
|
|
968
1082
|
Returns:
|
|
969
1083
|
string representation of the key without leaking secrets
|
|
1084
|
+
|
|
970
1085
|
"""
|
|
971
1086
|
# concrete __repr__() delegates to the (abstract) __str__():
|
|
972
1087
|
# this avoids marking __repr__() abstract while still unifying behavior
|
|
@@ -982,12 +1097,13 @@ class CryptoKey(abc.ABC):
|
|
|
982
1097
|
|
|
983
1098
|
Returns:
|
|
984
1099
|
string with all the object's fields explicit values
|
|
1100
|
+
|
|
985
1101
|
"""
|
|
986
1102
|
cls: str = type(self).__name__
|
|
987
1103
|
parts: list[str] = []
|
|
988
1104
|
for field in dataclasses.fields(self):
|
|
989
1105
|
val: Any = getattr(self, field.name) # getattr is fine with frozen/slots
|
|
990
|
-
parts.append(f'{field.name}={
|
|
1106
|
+
parts.append(f'{field.name}={val!r}')
|
|
991
1107
|
return f'{cls}({", ".join(parts)})'
|
|
992
1108
|
|
|
993
1109
|
@final
|
|
@@ -1000,13 +1116,15 @@ class CryptoKey(abc.ABC):
|
|
|
1000
1116
|
|
|
1001
1117
|
Raises:
|
|
1002
1118
|
ImplementationError: object has types that are not supported in JSON
|
|
1119
|
+
|
|
1003
1120
|
"""
|
|
1004
1121
|
self_dict: dict[str, Any] = dataclasses.asdict(self)
|
|
1005
1122
|
for field in dataclasses.fields(self):
|
|
1006
1123
|
# check the type is OK
|
|
1007
1124
|
if field.type not in _JSON_DATACLASS_TYPES:
|
|
1008
1125
|
raise ImplementationError(
|
|
1009
|
-
|
|
1126
|
+
f'Unsupported JSON field {field.name!r}/{field.type} not in {_JSON_DATACLASS_TYPES}'
|
|
1127
|
+
)
|
|
1010
1128
|
# convert types that we accept but JSON does not
|
|
1011
1129
|
if field.type == 'bytes':
|
|
1012
1130
|
self_dict[field.name] = BytesToEncoded(self_dict[field.name])
|
|
@@ -1020,8 +1138,6 @@ class CryptoKey(abc.ABC):
|
|
|
1020
1138
|
Returns:
|
|
1021
1139
|
str: JSON representation of the object, tightly packed
|
|
1022
1140
|
|
|
1023
|
-
Raises:
|
|
1024
|
-
ImplementationError: object has types that are not supported in JSON
|
|
1025
1141
|
"""
|
|
1026
1142
|
return json.dumps(self._json_dict, separators=(',', ':'))
|
|
1027
1143
|
|
|
@@ -1033,8 +1149,6 @@ class CryptoKey(abc.ABC):
|
|
|
1033
1149
|
Returns:
|
|
1034
1150
|
str: JSON representation of the object formatted for humans
|
|
1035
1151
|
|
|
1036
|
-
Raises:
|
|
1037
|
-
ImplementationError: object has types that are not supported in JSON
|
|
1038
1152
|
"""
|
|
1039
1153
|
return json.dumps(self._json_dict, indent=4, sort_keys=True)
|
|
1040
1154
|
|
|
@@ -1051,9 +1165,11 @@ class CryptoKey(abc.ABC):
|
|
|
1051
1165
|
|
|
1052
1166
|
Raises:
|
|
1053
1167
|
InputError: unexpected type/fields
|
|
1168
|
+
ImplementationError: unsupported JSON field
|
|
1169
|
+
|
|
1054
1170
|
"""
|
|
1055
1171
|
# check we got exactly the fields we needed
|
|
1056
|
-
cls_fields: set[str] =
|
|
1172
|
+
cls_fields: set[str] = {f.name for f in dataclasses.fields(cls)}
|
|
1057
1173
|
json_fields: set[str] = set(json_dict)
|
|
1058
1174
|
if cls_fields != json_fields:
|
|
1059
1175
|
raise InputError(f'JSON data decoded to unexpected fields: {cls_fields=} / {json_fields=}')
|
|
@@ -1061,7 +1177,8 @@ class CryptoKey(abc.ABC):
|
|
|
1061
1177
|
for field in dataclasses.fields(cls):
|
|
1062
1178
|
if field.type not in _JSON_DATACLASS_TYPES:
|
|
1063
1179
|
raise ImplementationError(
|
|
1064
|
-
|
|
1180
|
+
f'Unsupported JSON field {field.name!r}/{field.type} not in {_JSON_DATACLASS_TYPES}'
|
|
1181
|
+
)
|
|
1065
1182
|
if field.type == 'bytes':
|
|
1066
1183
|
json_dict[field.name] = EncodedToBytes(json_dict[field.name])
|
|
1067
1184
|
# build the object
|
|
@@ -1080,10 +1197,11 @@ class CryptoKey(abc.ABC):
|
|
|
1080
1197
|
|
|
1081
1198
|
Raises:
|
|
1082
1199
|
InputError: unexpected type/fields
|
|
1200
|
+
|
|
1083
1201
|
"""
|
|
1084
1202
|
# get the dict back
|
|
1085
1203
|
json_dict: dict[str, Any] = json.loads(json_data)
|
|
1086
|
-
if not isinstance(json_dict, dict): #
|
|
1204
|
+
if not isinstance(json_dict, dict): # pyright: ignore[reportUnnecessaryIsInstance]
|
|
1087
1205
|
raise InputError(f'JSON data decoded to unexpected type: {type(json_dict)}')
|
|
1088
1206
|
return cls._FromJSONDict(json_dict)
|
|
1089
1207
|
|
|
@@ -1094,12 +1212,13 @@ class CryptoKey(abc.ABC):
|
|
|
1094
1212
|
|
|
1095
1213
|
Returns:
|
|
1096
1214
|
bytes, pickled, representation of the object
|
|
1215
|
+
|
|
1097
1216
|
"""
|
|
1098
1217
|
return self.Blob()
|
|
1099
1218
|
|
|
1100
1219
|
@final
|
|
1101
1220
|
def Blob(self, /, *, key: Encryptor | None = None, silent: bool = True) -> bytes:
|
|
1102
|
-
"""
|
|
1221
|
+
"""Get serial (bytes) representation of the object with more options, including encryption.
|
|
1103
1222
|
|
|
1104
1223
|
Args:
|
|
1105
1224
|
key (Encryptor, optional): if given will key.Encrypt() data before saving
|
|
@@ -1107,6 +1226,7 @@ class CryptoKey(abc.ABC):
|
|
|
1107
1226
|
|
|
1108
1227
|
Returns:
|
|
1109
1228
|
bytes, pickled, representation of the object
|
|
1229
|
+
|
|
1110
1230
|
"""
|
|
1111
1231
|
return Serialize(self._json_dict, compress=-2, key=key, silent=silent, pickler=PickleJSON)
|
|
1112
1232
|
|
|
@@ -1117,6 +1237,7 @@ class CryptoKey(abc.ABC):
|
|
|
1117
1237
|
|
|
1118
1238
|
Returns:
|
|
1119
1239
|
str, pickled, base64, representation of the object
|
|
1240
|
+
|
|
1120
1241
|
"""
|
|
1121
1242
|
return self.Encoded()
|
|
1122
1243
|
|
|
@@ -1130,6 +1251,7 @@ class CryptoKey(abc.ABC):
|
|
|
1130
1251
|
|
|
1131
1252
|
Returns:
|
|
1132
1253
|
str, pickled, base64, representation of the object
|
|
1254
|
+
|
|
1133
1255
|
"""
|
|
1134
1256
|
return CryptoInputType.BASE64 + BytesToEncoded(self.Blob(key=key, silent=silent))
|
|
1135
1257
|
|
|
@@ -1140,6 +1262,7 @@ class CryptoKey(abc.ABC):
|
|
|
1140
1262
|
|
|
1141
1263
|
Returns:
|
|
1142
1264
|
str, pickled, hexadecimal, representation of the object
|
|
1265
|
+
|
|
1143
1266
|
"""
|
|
1144
1267
|
return self.Hex()
|
|
1145
1268
|
|
|
@@ -1153,6 +1276,7 @@ class CryptoKey(abc.ABC):
|
|
|
1153
1276
|
|
|
1154
1277
|
Returns:
|
|
1155
1278
|
str, pickled, hexadecimal, representation of the object
|
|
1279
|
+
|
|
1156
1280
|
"""
|
|
1157
1281
|
return CryptoInputType.HEX + BytesToHex(self.Blob(key=key, silent=silent))
|
|
1158
1282
|
|
|
@@ -1163,6 +1287,7 @@ class CryptoKey(abc.ABC):
|
|
|
1163
1287
|
|
|
1164
1288
|
Returns:
|
|
1165
1289
|
str, pickled, raw escaped binary, representation of the object
|
|
1290
|
+
|
|
1166
1291
|
"""
|
|
1167
1292
|
return self.Raw()
|
|
1168
1293
|
|
|
@@ -1176,13 +1301,13 @@ class CryptoKey(abc.ABC):
|
|
|
1176
1301
|
|
|
1177
1302
|
Returns:
|
|
1178
1303
|
str, pickled, raw escaped binary, representation of the object
|
|
1304
|
+
|
|
1179
1305
|
"""
|
|
1180
1306
|
return CryptoInputType.RAW + BytesToRaw(self.Blob(key=key, silent=silent))
|
|
1181
1307
|
|
|
1182
1308
|
@final
|
|
1183
1309
|
@classmethod
|
|
1184
|
-
def Load(
|
|
1185
|
-
cls, data: str | bytes, /, *, key: Decryptor | None = None, silent: bool = True) -> Self:
|
|
1310
|
+
def Load(cls, data: str | bytes, /, *, key: Decryptor | None = None, silent: bool = True) -> Self:
|
|
1186
1311
|
"""Load (create) object from serialized bytes or string.
|
|
1187
1312
|
|
|
1188
1313
|
Args:
|
|
@@ -1193,6 +1318,10 @@ class CryptoKey(abc.ABC):
|
|
|
1193
1318
|
|
|
1194
1319
|
Returns:
|
|
1195
1320
|
a CryptoKey object ready for use
|
|
1321
|
+
|
|
1322
|
+
Raises:
|
|
1323
|
+
InputError: decode error
|
|
1324
|
+
|
|
1196
1325
|
"""
|
|
1197
1326
|
# if this is a string, then we suppose it is base64
|
|
1198
1327
|
if isinstance(data, str):
|
|
@@ -1200,15 +1329,16 @@ class CryptoKey(abc.ABC):
|
|
|
1200
1329
|
# we now have bytes and we suppose it came from CryptoKey.blob()/CryptoKey.CryptoBlob()
|
|
1201
1330
|
try:
|
|
1202
1331
|
json_dict: dict[str, Any] = DeSerialize(
|
|
1203
|
-
|
|
1332
|
+
data=data, key=key, silent=silent, unpickler=UnpickleJSON
|
|
1333
|
+
)
|
|
1204
1334
|
return cls._FromJSONDict(json_dict)
|
|
1205
1335
|
except Exception as err:
|
|
1206
1336
|
raise InputError(f'input decode error: {err}') from err
|
|
1207
1337
|
|
|
1208
1338
|
|
|
1209
1339
|
@runtime_checkable
|
|
1210
|
-
class Encryptor(Protocol):
|
|
1211
|
-
"""Abstract interface for a class that has encryption
|
|
1340
|
+
class Encryptor(Protocol):
|
|
1341
|
+
"""Abstract interface for a class that has encryption.
|
|
1212
1342
|
|
|
1213
1343
|
Contract:
|
|
1214
1344
|
- If algorithm accepts a `nonce` or `tag` these have to be handled internally by the
|
|
@@ -1221,9 +1351,10 @@ class Encryptor(Protocol): # pylint: disable=too-few-public-methods
|
|
|
1221
1351
|
Metadata like nonce/tag may be:
|
|
1222
1352
|
- returned alongside `ciphertext`/`signature`, or
|
|
1223
1353
|
- bundled/serialized into `ciphertext`/`signature` by the implementation.
|
|
1354
|
+
|
|
1224
1355
|
"""
|
|
1225
1356
|
|
|
1226
|
-
@
|
|
1357
|
+
@abstract.abstractmethod
|
|
1227
1358
|
def Encrypt(self, plaintext: bytes, /, *, associated_data: bytes | None = None) -> bytes:
|
|
1228
1359
|
"""Encrypt `plaintext` and return `ciphertext`.
|
|
1229
1360
|
|
|
@@ -1239,14 +1370,15 @@ class Encryptor(Protocol): # pylint: disable=too-few-public-methods
|
|
|
1239
1370
|
Raises:
|
|
1240
1371
|
InputError: invalid inputs
|
|
1241
1372
|
CryptoError: internal crypto failures
|
|
1373
|
+
|
|
1242
1374
|
"""
|
|
1243
1375
|
|
|
1244
1376
|
|
|
1245
1377
|
@runtime_checkable
|
|
1246
|
-
class Decryptor(Protocol):
|
|
1378
|
+
class Decryptor(Protocol):
|
|
1247
1379
|
"""Abstract interface for a class that has decryption (see contract/notes in Encryptor)."""
|
|
1248
1380
|
|
|
1249
|
-
@
|
|
1381
|
+
@abstract.abstractmethod
|
|
1250
1382
|
def Decrypt(self, ciphertext: bytes, /, *, associated_data: bytes | None = None) -> bytes:
|
|
1251
1383
|
"""Decrypt `ciphertext` and return the original `plaintext`.
|
|
1252
1384
|
|
|
@@ -1260,16 +1392,18 @@ class Decryptor(Protocol): # pylint: disable=too-few-public-methods
|
|
|
1260
1392
|
Raises:
|
|
1261
1393
|
InputError: invalid inputs
|
|
1262
1394
|
CryptoError: internal crypto failures, authentication failure, key mismatch, etc
|
|
1395
|
+
|
|
1263
1396
|
"""
|
|
1264
1397
|
|
|
1265
1398
|
|
|
1266
1399
|
@runtime_checkable
|
|
1267
|
-
class Verifier(Protocol):
|
|
1400
|
+
class Verifier(Protocol):
|
|
1268
1401
|
"""Abstract interface for asymmetric signature verify. (see contract/notes in Encryptor)."""
|
|
1269
1402
|
|
|
1270
|
-
@
|
|
1403
|
+
@abstract.abstractmethod
|
|
1271
1404
|
def Verify(
|
|
1272
|
-
|
|
1405
|
+
self, message: bytes, signature: bytes, /, *, associated_data: bytes | None = None
|
|
1406
|
+
) -> bool:
|
|
1273
1407
|
"""Verify a `signature` for `message`. True if OK; False if failed verification.
|
|
1274
1408
|
|
|
1275
1409
|
Args:
|
|
@@ -1283,14 +1417,15 @@ class Verifier(Protocol): # pylint: disable=too-few-public-methods
|
|
|
1283
1417
|
Raises:
|
|
1284
1418
|
InputError: invalid inputs
|
|
1285
1419
|
CryptoError: internal crypto failures, authentication failure, key mismatch, etc
|
|
1420
|
+
|
|
1286
1421
|
"""
|
|
1287
1422
|
|
|
1288
1423
|
|
|
1289
1424
|
@runtime_checkable
|
|
1290
|
-
class Signer(Protocol):
|
|
1425
|
+
class Signer(Protocol):
|
|
1291
1426
|
"""Abstract interface for asymmetric signing. (see contract/notes in Encryptor)."""
|
|
1292
1427
|
|
|
1293
|
-
@
|
|
1428
|
+
@abstract.abstractmethod
|
|
1294
1429
|
def Sign(self, message: bytes, /, *, associated_data: bytes | None = None) -> bytes:
|
|
1295
1430
|
"""Sign `message` and return the `signature`.
|
|
1296
1431
|
|
|
@@ -1306,13 +1441,20 @@ class Signer(Protocol): # pylint: disable=too-few-public-methods
|
|
|
1306
1441
|
Raises:
|
|
1307
1442
|
InputError: invalid inputs
|
|
1308
1443
|
CryptoError: internal crypto failures
|
|
1444
|
+
|
|
1309
1445
|
"""
|
|
1310
1446
|
|
|
1311
1447
|
|
|
1312
|
-
def Serialize(
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1448
|
+
def Serialize(
|
|
1449
|
+
python_obj: Any, # noqa: ANN401
|
|
1450
|
+
/,
|
|
1451
|
+
*,
|
|
1452
|
+
file_path: str | None = None,
|
|
1453
|
+
compress: int | None = 3,
|
|
1454
|
+
key: Encryptor | None = None,
|
|
1455
|
+
silent: bool = False,
|
|
1456
|
+
pickler: abc.Callable[[Any], bytes] = PickleGeneric,
|
|
1457
|
+
) -> bytes:
|
|
1316
1458
|
"""Serialize a Python object into a BLOB, optionally compress / encrypt / save to disk.
|
|
1317
1459
|
|
|
1318
1460
|
Data path is:
|
|
@@ -1324,14 +1466,14 @@ def Serialize( # pylint:disable=too-many-arguments
|
|
|
1324
1466
|
|
|
1325
1467
|
Compression levels / speed can be controlled by `compress`. Use this as reference:
|
|
1326
1468
|
|
|
1327
|
-
| Level | Speed | Compression ratio
|
|
1328
|
-
| -------- | ------------|
|
|
1329
|
-
| -5 to -1 | Fastest | Poor (better than
|
|
1330
|
-
| 0…3 | Very fast | Good ratio
|
|
1331
|
-
| 4…6 | Moderate | Better ratio
|
|
1332
|
-
| 7…10 | Slower | Marginally better ratio
|
|
1333
|
-
| 11…15 | Much slower | Slight gains
|
|
1334
|
-
| 16…22 | Very slow | Tiny gains
|
|
1469
|
+
| Level | Speed | Compression ratio | Typical use case |
|
|
1470
|
+
| -------- | ------------| ------------------------| --------------------------------------- |
|
|
1471
|
+
| -5 to -1 | Fastest | Poor (better than none) | Real-time / very latency-sensitive |
|
|
1472
|
+
| 0…3 | Very fast | Good ratio | Default CLI choice, safe baseline |
|
|
1473
|
+
| 4…6 | Moderate | Better ratio | Good compromise for general persistence |
|
|
1474
|
+
| 7…10 | Slower | Marginally better ratio | Only if storage space is precious |
|
|
1475
|
+
| 11…15 | Much slower | Slight gains | Large archives, not for runtime use |
|
|
1476
|
+
| 16…22 | Very slow | Tiny gains | Archival-only, multi-GB datasets |
|
|
1335
1477
|
|
|
1336
1478
|
Args:
|
|
1337
1479
|
python_obj (Any): serializable Python object
|
|
@@ -1346,6 +1488,7 @@ def Serialize( # pylint:disable=too-many-arguments
|
|
|
1346
1488
|
|
|
1347
1489
|
Returns:
|
|
1348
1490
|
bytes: serialized binary data corresponding to obj + (compression) + (encryption)
|
|
1491
|
+
|
|
1349
1492
|
"""
|
|
1350
1493
|
messages: list[str] = []
|
|
1351
1494
|
with Timer('Serialization complete', emit_log=False) as tm_all:
|
|
@@ -1356,8 +1499,8 @@ def Serialize( # pylint:disable=too-many-arguments
|
|
|
1356
1499
|
messages.append(f' {tm_pickle}, {HumanizedBytes(len(obj))}')
|
|
1357
1500
|
# compress, if needed
|
|
1358
1501
|
if compress is not None:
|
|
1359
|
-
compress =
|
|
1360
|
-
compress =
|
|
1502
|
+
compress = max(compress, -22)
|
|
1503
|
+
compress = min(compress, 22)
|
|
1361
1504
|
with Timer(f'COMPRESS@{compress}', emit_log=False) as tm_compress:
|
|
1362
1505
|
obj = zstandard.ZstdCompressor(level=compress).compress(obj)
|
|
1363
1506
|
if not silent:
|
|
@@ -1371,21 +1514,24 @@ def Serialize( # pylint:disable=too-many-arguments
|
|
|
1371
1514
|
# optionally save to disk
|
|
1372
1515
|
if file_path is not None:
|
|
1373
1516
|
with Timer('SAVE', emit_log=False) as tm_save:
|
|
1374
|
-
|
|
1375
|
-
file_obj.write(obj)
|
|
1517
|
+
pathlib.Path(file_path).write_bytes(obj)
|
|
1376
1518
|
if not silent:
|
|
1377
1519
|
messages.append(f' {tm_save}, to {file_path!r}')
|
|
1378
1520
|
# log and return
|
|
1379
1521
|
if not silent:
|
|
1380
|
-
logging.info(f'{tm_all}; parts:\n
|
|
1522
|
+
logging.info(f'{tm_all}; parts:\n{"\n".join(messages)}')
|
|
1381
1523
|
return obj
|
|
1382
1524
|
|
|
1383
1525
|
|
|
1384
|
-
def DeSerialize(
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1526
|
+
def DeSerialize( # noqa: C901
|
|
1527
|
+
*,
|
|
1528
|
+
data: bytes | None = None,
|
|
1529
|
+
file_path: str | None = None,
|
|
1530
|
+
key: Decryptor | None = None,
|
|
1531
|
+
silent: bool = False,
|
|
1532
|
+
unpickler: abc.Callable[[bytes], Any] = UnpickleGeneric,
|
|
1533
|
+
) -> Any: # noqa: ANN401
|
|
1534
|
+
"""Load (de-serializes) a BLOB back to a Python object, optionally decrypting / decompressing.
|
|
1389
1535
|
|
|
1390
1536
|
Data path is:
|
|
1391
1537
|
|
|
@@ -1396,15 +1542,17 @@ def DeSerialize(
|
|
|
1396
1542
|
Compression versus no compression will be automatically detected.
|
|
1397
1543
|
|
|
1398
1544
|
Args:
|
|
1399
|
-
data (bytes, optional): if given, use this as binary data string (input);
|
|
1400
|
-
|
|
1401
|
-
file_path (str, optional): if given, use this as file path to load binary data
|
|
1402
|
-
|
|
1403
|
-
key (Decryptor, optional): if given will key.Decrypt() data before decompressing/loading
|
|
1404
|
-
|
|
1405
|
-
|
|
1545
|
+
data (bytes | None, optional): if given, use this as binary data string (input);
|
|
1546
|
+
if you use this option, `file_path` will be ignored
|
|
1547
|
+
file_path (str | None, optional): if given, use this as file path to load binary data
|
|
1548
|
+
string (input); if you use this option, `data` will be ignored. Defaults to None.
|
|
1549
|
+
key (Decryptor | None, optional): if given will key.Decrypt() data before decompressing/loading.
|
|
1550
|
+
Defaults to None.
|
|
1551
|
+
silent (bool, optional): if True will not log; default is False (will log). Defaults to False.
|
|
1552
|
+
unpickler (Callable[[bytes], Any], optional): if not given, will just be the `pickle` module;
|
|
1406
1553
|
if given will be a method to convert a `bytes` representation back to a Python object;
|
|
1407
|
-
UnpickleGeneric is the default, but another useful value is UnpickleJSON
|
|
1554
|
+
UnpickleGeneric is the default, but another useful value is UnpickleJSON.
|
|
1555
|
+
Defaults to UnpickleGeneric.
|
|
1408
1556
|
|
|
1409
1557
|
Returns:
|
|
1410
1558
|
De-Serialized Python object corresponding to data
|
|
@@ -1412,24 +1560,24 @@ def DeSerialize(
|
|
|
1412
1560
|
Raises:
|
|
1413
1561
|
InputError: invalid inputs
|
|
1414
1562
|
CryptoError: internal crypto failures, authentication failure, key mismatch, etc
|
|
1415
|
-
|
|
1563
|
+
|
|
1564
|
+
""" # noqa: DOC502
|
|
1416
1565
|
# test inputs
|
|
1417
1566
|
if (data is None and file_path is None) or (data is not None and file_path is not None):
|
|
1418
1567
|
raise InputError('you must provide only one of either `data` or `file_path`')
|
|
1419
|
-
if file_path and not
|
|
1568
|
+
if file_path and not pathlib.Path(file_path).exists():
|
|
1420
1569
|
raise InputError(f'invalid file_path: {file_path!r}')
|
|
1421
|
-
if data and len(data) < 4:
|
|
1570
|
+
if data and len(data) < 4: # noqa: PLR2004
|
|
1422
1571
|
raise InputError('invalid data: too small')
|
|
1423
1572
|
# start the pipeline
|
|
1424
|
-
obj: bytes = data
|
|
1573
|
+
obj: bytes = data or b''
|
|
1425
1574
|
messages: list[str] = [f'DATA: {HumanizedBytes(len(obj))}'] if data and not silent else []
|
|
1426
1575
|
with Timer('De-Serialization complete', emit_log=False) as tm_all:
|
|
1427
1576
|
# optionally load from disk
|
|
1428
1577
|
if file_path:
|
|
1429
|
-
assert not obj, 'should never happen: if we have a file obj should be empty'
|
|
1578
|
+
assert not obj, 'should never happen: if we have a file obj should be empty' # noqa: S101
|
|
1430
1579
|
with Timer('LOAD', emit_log=False) as tm_load:
|
|
1431
|
-
|
|
1432
|
-
obj = file_obj.read()
|
|
1580
|
+
obj = pathlib.Path(file_path).read_bytes()
|
|
1433
1581
|
if not silent:
|
|
1434
1582
|
messages.append(f' {tm_load}, {HumanizedBytes(len(obj))}, from {file_path!r}')
|
|
1435
1583
|
# decrypt, if needed
|
|
@@ -1439,16 +1587,19 @@ def DeSerialize(
|
|
|
1439
1587
|
if not silent:
|
|
1440
1588
|
messages.append(f' {tm_crypto}, {HumanizedBytes(len(obj))}')
|
|
1441
1589
|
# decompress: we try to detect compression to determine if we must call zstandard
|
|
1442
|
-
if (
|
|
1443
|
-
|
|
1444
|
-
|
|
1590
|
+
if (
|
|
1591
|
+
len(obj) >= 4 # noqa: PLR2004
|
|
1592
|
+
and (
|
|
1593
|
+
((magic := int.from_bytes(obj[:4], 'little')) == _ZSTD_MAGIC_FRAME)
|
|
1594
|
+
or (_ZSTD_MAGIC_SKIPPABLE_MIN <= magic <= _ZSTD_MAGIC_SKIPPABLE_MAX)
|
|
1595
|
+
)
|
|
1596
|
+
):
|
|
1445
1597
|
with Timer('DECOMPRESS', emit_log=False) as tm_decompress:
|
|
1446
1598
|
obj = zstandard.ZstdDecompressor().decompress(obj)
|
|
1447
1599
|
if not silent:
|
|
1448
1600
|
messages.append(f' {tm_decompress}, {HumanizedBytes(len(obj))}')
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
messages.append(' (no compression detected)')
|
|
1601
|
+
elif not silent:
|
|
1602
|
+
messages.append(' (no compression detected)')
|
|
1452
1603
|
# create the actual object = unpickle
|
|
1453
1604
|
with Timer('UNPICKLE', emit_log=False) as tm_unpickle:
|
|
1454
1605
|
python_obj: Any = unpickler(obj)
|
|
@@ -1456,7 +1607,7 @@ def DeSerialize(
|
|
|
1456
1607
|
messages.append(f' {tm_unpickle}')
|
|
1457
1608
|
# log and return
|
|
1458
1609
|
if not silent:
|
|
1459
|
-
logging.info(f'{tm_all}; parts:\n
|
|
1610
|
+
logging.info(f'{tm_all}; parts:\n{"\n".join(messages)}')
|
|
1460
1611
|
return python_obj
|
|
1461
1612
|
|
|
1462
1613
|
|
|
@@ -1474,6 +1625,7 @@ class PublicBid512(CryptoKey):
|
|
|
1474
1625
|
Attributes:
|
|
1475
1626
|
public_key (bytes): 512-bits random value
|
|
1476
1627
|
public_hash (bytes): SHA-512 hash of (public_key || private_key || secret_bid)
|
|
1628
|
+
|
|
1477
1629
|
"""
|
|
1478
1630
|
|
|
1479
1631
|
public_key: bytes
|
|
@@ -1484,9 +1636,9 @@ class PublicBid512(CryptoKey):
|
|
|
1484
1636
|
|
|
1485
1637
|
Raises:
|
|
1486
1638
|
InputError: invalid inputs
|
|
1639
|
+
|
|
1487
1640
|
"""
|
|
1488
|
-
|
|
1489
|
-
if len(self.public_key) != 64 or len(self.public_hash) != 64:
|
|
1641
|
+
if len(self.public_key) != 64 or len(self.public_hash) != 64: # noqa: PLR2004
|
|
1490
1642
|
raise InputError(f'invalid public_key or public_hash: {self}')
|
|
1491
1643
|
|
|
1492
1644
|
def __str__(self) -> str:
|
|
@@ -1494,10 +1646,13 @@ class PublicBid512(CryptoKey):
|
|
|
1494
1646
|
|
|
1495
1647
|
Returns:
|
|
1496
1648
|
string representation of PublicBid
|
|
1649
|
+
|
|
1497
1650
|
"""
|
|
1498
|
-
return (
|
|
1499
|
-
|
|
1500
|
-
|
|
1651
|
+
return (
|
|
1652
|
+
'PublicBid512('
|
|
1653
|
+
f'public_key={BytesToEncoded(self.public_key)}, '
|
|
1654
|
+
f'public_hash={BytesToHex(self.public_hash)})'
|
|
1655
|
+
)
|
|
1501
1656
|
|
|
1502
1657
|
def VerifyBid(self, private_key: bytes, secret: bytes, /) -> bool:
|
|
1503
1658
|
"""Verify a bid. True if OK; False if failed verification.
|
|
@@ -1509,21 +1664,30 @@ class PublicBid512(CryptoKey):
|
|
|
1509
1664
|
Returns:
|
|
1510
1665
|
True if bid is valid, False otherwise
|
|
1511
1666
|
|
|
1512
|
-
Raises:
|
|
1513
|
-
InputError: invalid inputs
|
|
1514
1667
|
"""
|
|
1515
1668
|
try:
|
|
1516
1669
|
# creating the PrivateBid object will validate everything; InputError we allow to propagate
|
|
1517
1670
|
PrivateBid512(
|
|
1518
|
-
|
|
1519
|
-
|
|
1671
|
+
public_key=self.public_key,
|
|
1672
|
+
public_hash=self.public_hash,
|
|
1673
|
+
private_key=private_key,
|
|
1674
|
+
secret_bid=secret,
|
|
1675
|
+
)
|
|
1520
1676
|
return True # if we got here, all is good
|
|
1521
1677
|
except CryptoError:
|
|
1522
1678
|
return False # bid does not match the public commitment
|
|
1523
1679
|
|
|
1524
1680
|
@classmethod
|
|
1525
1681
|
def Copy(cls, other: PublicBid512, /) -> Self:
|
|
1526
|
-
"""Initialize a public bid by taking the public parts of a public/private bid.
|
|
1682
|
+
"""Initialize a public bid by taking the public parts of a public/private bid.
|
|
1683
|
+
|
|
1684
|
+
Args:
|
|
1685
|
+
other (PublicBid512): the bid to copy from
|
|
1686
|
+
|
|
1687
|
+
Returns:
|
|
1688
|
+
Self: an initialized PublicBid512
|
|
1689
|
+
|
|
1690
|
+
"""
|
|
1527
1691
|
return cls(public_key=other.public_key, public_hash=other.public_hash)
|
|
1528
1692
|
|
|
1529
1693
|
|
|
@@ -1534,6 +1698,7 @@ class PrivateBid512(PublicBid512):
|
|
|
1534
1698
|
Attributes:
|
|
1535
1699
|
private_key (bytes): 512-bits random value
|
|
1536
1700
|
secret_bid (bytes): Any number of bytes (≥1) to bid on (e.g., UTF-8 encoded string)
|
|
1701
|
+
|
|
1537
1702
|
"""
|
|
1538
1703
|
|
|
1539
1704
|
private_key: bytes
|
|
@@ -1545,9 +1710,10 @@ class PrivateBid512(PublicBid512):
|
|
|
1545
1710
|
Raises:
|
|
1546
1711
|
InputError: invalid inputs
|
|
1547
1712
|
CryptoError: bid does not match the public commitment
|
|
1713
|
+
|
|
1548
1714
|
"""
|
|
1549
|
-
super(PrivateBid512, self).__post_init__()
|
|
1550
|
-
if len(self.private_key) != 64 or len(self.secret_bid) < 1:
|
|
1715
|
+
super(PrivateBid512, self).__post_init__()
|
|
1716
|
+
if len(self.private_key) != 64 or len(self.secret_bid) < 1: # noqa: PLR2004
|
|
1551
1717
|
raise InputError(f'invalid private_key or secret_bid: {self}')
|
|
1552
1718
|
if self.public_hash != Hash512(self.public_key + self.private_key + self.secret_bid):
|
|
1553
1719
|
raise CryptoError(f'inconsistent bid: {self}')
|
|
@@ -1557,11 +1723,14 @@ class PrivateBid512(PublicBid512):
|
|
|
1557
1723
|
|
|
1558
1724
|
Returns:
|
|
1559
1725
|
string representation of PrivateBid without leaking secrets
|
|
1726
|
+
|
|
1560
1727
|
"""
|
|
1561
|
-
return (
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1728
|
+
return (
|
|
1729
|
+
'PrivateBid512('
|
|
1730
|
+
f'{super(PrivateBid512, self).__str__()}, '
|
|
1731
|
+
f'private_key={ObfuscateSecret(self.private_key)}, '
|
|
1732
|
+
f'secret_bid={ObfuscateSecret(self.secret_bid)})'
|
|
1733
|
+
)
|
|
1565
1734
|
|
|
1566
1735
|
@classmethod
|
|
1567
1736
|
def New(cls, secret: bytes, /) -> Self:
|
|
@@ -1575,199 +1744,175 @@ class PrivateBid512(PublicBid512):
|
|
|
1575
1744
|
|
|
1576
1745
|
Raises:
|
|
1577
1746
|
InputError: invalid inputs
|
|
1747
|
+
|
|
1578
1748
|
"""
|
|
1579
1749
|
# test inputs
|
|
1580
1750
|
if len(secret) < 1:
|
|
1581
1751
|
raise InputError(f'invalid secret length: {len(secret)}')
|
|
1582
1752
|
# generate random values
|
|
1583
|
-
public_key: bytes = RandBytes(64)
|
|
1753
|
+
public_key: bytes = RandBytes(64) # 512 bits
|
|
1584
1754
|
private_key: bytes = RandBytes(64) # 512 bits
|
|
1585
1755
|
# build object
|
|
1586
1756
|
return cls(
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
def
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
return
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
def
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
if
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
Will treat epilog strings as examples, splitting on '$$' to get multiple examples.
|
|
1757
|
+
public_key=public_key,
|
|
1758
|
+
public_hash=Hash512(public_key + private_key + secret),
|
|
1759
|
+
private_key=private_key,
|
|
1760
|
+
secret_bid=secret,
|
|
1761
|
+
)
|
|
1762
|
+
|
|
1763
|
+
|
|
1764
|
+
@dataclasses.dataclass(kw_only=True, slots=True, frozen=True)
|
|
1765
|
+
class CLIConfig:
|
|
1766
|
+
"""CLI global context, storing the configuration."""
|
|
1767
|
+
|
|
1768
|
+
console: rich_console.Console
|
|
1769
|
+
verbose: int
|
|
1770
|
+
color: bool | None
|
|
1771
|
+
|
|
1772
|
+
|
|
1773
|
+
def CLIErrorGuard[**P](fn: abc.Callable[P, None], /) -> abc.Callable[P, None]:
|
|
1774
|
+
"""Guard CLI command functions.
|
|
1775
|
+
|
|
1776
|
+
Returns:
|
|
1777
|
+
A wrapped function that catches expected user-facing errors and prints them consistently.
|
|
1778
|
+
|
|
1779
|
+
"""
|
|
1780
|
+
|
|
1781
|
+
@functools.wraps(fn)
|
|
1782
|
+
def _Wrapper(*args: P.args, **kwargs: P.kwargs) -> None:
|
|
1783
|
+
try:
|
|
1784
|
+
# call the actual function
|
|
1785
|
+
fn(*args, **kwargs)
|
|
1786
|
+
except (Error, ValueError) as err:
|
|
1787
|
+
# get context
|
|
1788
|
+
ctx: object | None = dict(kwargs).get('ctx')
|
|
1789
|
+
if not isinstance(ctx, typer.Context):
|
|
1790
|
+
ctx = next((a for a in args if isinstance(a, typer.Context)), None)
|
|
1791
|
+
# print error nicely
|
|
1792
|
+
if isinstance(ctx, typer.Context):
|
|
1793
|
+
# we have context
|
|
1794
|
+
obj: CLIConfig = cast('CLIConfig', ctx.obj)
|
|
1795
|
+
if obj.verbose >= 2: # verbose >= 2 means INFO level or more verbose # noqa: PLR2004
|
|
1796
|
+
obj.console.print_exception() # print full traceback
|
|
1797
|
+
else:
|
|
1798
|
+
obj.console.print(str(err)) # print only error message
|
|
1799
|
+
# no context
|
|
1800
|
+
elif logging.getLogger().getEffectiveLevel() < logging.INFO:
|
|
1801
|
+
Console().print(str(err)) # print only error message (DEBUG level is verbose already)
|
|
1802
|
+
else:
|
|
1803
|
+
Console().print_exception() # print full traceback (less verbose mode needs it)
|
|
1804
|
+
|
|
1805
|
+
return _Wrapper
|
|
1806
|
+
|
|
1807
|
+
|
|
1808
|
+
def _ClickWalk(
|
|
1809
|
+
command: click.Command,
|
|
1810
|
+
ctx: typer.Context,
|
|
1811
|
+
path: list[str],
|
|
1812
|
+
/,
|
|
1813
|
+
) -> abc.Iterator[tuple[list[str], click.Command, typer.Context]]:
|
|
1814
|
+
"""Recursively walk Click commands/groups.
|
|
1815
|
+
|
|
1816
|
+
Yields:
|
|
1817
|
+
tuple[list[str], click.Command, typer.Context]: path, command, ctx
|
|
1818
|
+
|
|
1819
|
+
"""
|
|
1820
|
+
yield (path, command, ctx) # yield self
|
|
1821
|
+
# now walk subcommands, if any
|
|
1822
|
+
sub_cmd: click.Command | None
|
|
1823
|
+
sub_ctx: typer.Context
|
|
1824
|
+
# prefer the explicit `.commands` mapping when present; otherwise fall back to
|
|
1825
|
+
# click's `list_commands()`/`get_command()` for dynamic groups
|
|
1826
|
+
if not isinstance(command, click.Group):
|
|
1827
|
+
return
|
|
1828
|
+
# explicit commands mapping
|
|
1829
|
+
if command.commands:
|
|
1830
|
+
for name, sub_cmd in sorted(command.commands.items()):
|
|
1831
|
+
sub_ctx = typer.Context(sub_cmd, info_name=name, parent=ctx)
|
|
1832
|
+
yield from _ClickWalk(sub_cmd, sub_ctx, [*path, name])
|
|
1833
|
+
return
|
|
1834
|
+
# dynamic commands
|
|
1835
|
+
for name in sorted(command.list_commands(ctx)):
|
|
1836
|
+
sub_cmd = command.get_command(ctx, name)
|
|
1837
|
+
if sub_cmd is None:
|
|
1838
|
+
continue # skip invalid subcommands
|
|
1839
|
+
sub_ctx = typer.Context(sub_cmd, info_name=name, parent=ctx)
|
|
1840
|
+
yield from _ClickWalk(sub_cmd, sub_ctx, [*path, name])
|
|
1841
|
+
|
|
1842
|
+
|
|
1843
|
+
def GenerateTyperHelpMarkdown(
|
|
1844
|
+
typer_app: typer.Typer,
|
|
1845
|
+
/,
|
|
1846
|
+
*,
|
|
1847
|
+
prog_name: str,
|
|
1848
|
+
heading_level: int = 1,
|
|
1849
|
+
code_fence_language: str = 'text',
|
|
1850
|
+
) -> str:
|
|
1851
|
+
"""Capture `--help` for a Typer CLI and all subcommands as Markdown.
|
|
1852
|
+
|
|
1853
|
+
This function converts a Typer app to its underlying Click command tree and then:
|
|
1854
|
+
- invokes `--help` for the root ("Main") command
|
|
1855
|
+
- walks commands/subcommands recursively
|
|
1856
|
+
- invokes `--help` for each command path
|
|
1857
|
+
|
|
1858
|
+
It emits a Markdown document with a heading per command and a fenced block
|
|
1859
|
+
containing the exact `--help` output.
|
|
1860
|
+
|
|
1861
|
+
Notes:
|
|
1862
|
+
- This uses Click's `CliRunner().invoke(...)` for faithful output.
|
|
1863
|
+
- The walk is generic over Click `MultiCommand`/`Group` structures.
|
|
1864
|
+
- If a command cannot be loaded, it is skipped.
|
|
1697
1865
|
|
|
1698
1866
|
Args:
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1867
|
+
typer_app: The Typer app (e.g. `app`).
|
|
1868
|
+
prog_name: Program name used in usage strings (e.g. "profiler").
|
|
1869
|
+
heading_level: Markdown heading level for each command section.
|
|
1870
|
+
code_fence_language: Language tag for fenced blocks (default: "text").
|
|
1702
1871
|
|
|
1703
1872
|
Returns:
|
|
1704
|
-
|
|
1873
|
+
Markdown string.
|
|
1705
1874
|
|
|
1706
|
-
Raises:
|
|
1707
|
-
InputError: invalid app name
|
|
1708
1875
|
"""
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
help_text = _HelpText(sub_parser, parent_sub_action)
|
|
1753
|
-
if help_text:
|
|
1754
|
-
lines.append(f'\n{help_text}')
|
|
1755
|
-
usage: str = sub_parser.format_usage().replace('usage: ', '').strip()
|
|
1756
|
-
lines.append('\n```bash')
|
|
1757
|
-
lines.append(str(usage))
|
|
1758
|
-
lines.append('```\n')
|
|
1759
|
-
# Options/args table
|
|
1760
|
-
rows: list[tuple[str, str]] = _RowsForActions(sub_parser._actions) # type: ignore[attr-defined] # pylint: disable=protected-access
|
|
1761
|
-
if rows:
|
|
1762
|
-
lines.append(_MarkdownTable(rows))
|
|
1763
|
-
lines.append('')
|
|
1764
|
-
# Examples (if any) - stored in epilog argument
|
|
1765
|
-
epilog: str = sub_parser.epilog.strip() if sub_parser.epilog else ''
|
|
1766
|
-
if epilog:
|
|
1767
|
-
lines.append('**Example:**\n')
|
|
1768
|
-
lines.append('```bash')
|
|
1769
|
-
for epilog_line in epilog.split('$$'):
|
|
1770
|
-
lines.append(f'$ {parser.prog} {epilog_line.strip()}')
|
|
1771
|
-
lines.append('```\n')
|
|
1772
|
-
# join all lines as the markdown string
|
|
1773
|
-
return ('\n'.join(lines)).strip()
|
|
1876
|
+
# prepare Click root command and context
|
|
1877
|
+
click_root: click.Command = typer.main.get_command(typer_app)
|
|
1878
|
+
root_ctx: typer.Context = typer.Context(click_root, info_name=prog_name)
|
|
1879
|
+
runner = click_testing.CliRunner()
|
|
1880
|
+
parts: list[str] = []
|
|
1881
|
+
for path, _, _ in _ClickWalk(click_root, root_ctx, []):
|
|
1882
|
+
# build command path
|
|
1883
|
+
command_path: str = ' '.join([prog_name, *path]).strip()
|
|
1884
|
+
heading_prefix: str = '#' * max(1, heading_level + len(path))
|
|
1885
|
+
ResetConsole() # ensure clean state for each command (also it raises on duplicate loggers)
|
|
1886
|
+
# invoke --help for this command path
|
|
1887
|
+
result: click_testing.Result = runner.invoke(
|
|
1888
|
+
click_root,
|
|
1889
|
+
[*path, '--help'],
|
|
1890
|
+
prog_name=prog_name,
|
|
1891
|
+
color=False,
|
|
1892
|
+
)
|
|
1893
|
+
if result.exit_code != 0 and not result.output:
|
|
1894
|
+
continue # skip invalid commands
|
|
1895
|
+
# build markdown section
|
|
1896
|
+
global_prefix: str = ( # only for the top-level command
|
|
1897
|
+
(
|
|
1898
|
+
'<!-- cspell:disable -->\n'
|
|
1899
|
+
'<!-- auto-generated; DO NOT EDIT! see base.GenerateTyperHelpMarkdown() -->\n\n'
|
|
1900
|
+
)
|
|
1901
|
+
if not path
|
|
1902
|
+
else ''
|
|
1903
|
+
)
|
|
1904
|
+
extras: str = ( # type of command, by level
|
|
1905
|
+
('Command-Line Interface' if not path else 'Command') if len(path) <= 1 else 'Sub-Command'
|
|
1906
|
+
)
|
|
1907
|
+
parts.extend(
|
|
1908
|
+
(
|
|
1909
|
+
f'{global_prefix}{heading_prefix} `{command_path}` {extras}',
|
|
1910
|
+
'',
|
|
1911
|
+
f'```{code_fence_language}',
|
|
1912
|
+
result.output.strip(),
|
|
1913
|
+
'```',
|
|
1914
|
+
'',
|
|
1915
|
+
)
|
|
1916
|
+
)
|
|
1917
|
+
# join all parts and return
|
|
1918
|
+
return '\n'.join(parts).rstrip()
|