skewsplit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- skewsplit/__init__.py +87 -0
- skewsplit/__main__.py +5 -0
- skewsplit/cli.py +345 -0
- skewsplit/core/__init__.py +9 -0
- skewsplit/core/codecs.py +287 -0
- skewsplit/core/errors.py +34 -0
- skewsplit/core/inputs.py +148 -0
- skewsplit/core/pgarray.py +95 -0
- skewsplit/core/pgstats.py +236 -0
- skewsplit/core/plan.py +289 -0
- skewsplit/core/planner.py +256 -0
- skewsplit/postgres/__init__.py +53 -0
- skewsplit/postgres/catalog.py +265 -0
- skewsplit/postgres/connection.py +27 -0
- skewsplit/postgres/statistics.py +258 -0
- skewsplit/postgres/verify.py +201 -0
- skewsplit/py.typed +0 -0
- skewsplit-0.1.0.dist-info/METADATA +97 -0
- skewsplit-0.1.0.dist-info/RECORD +22 -0
- skewsplit-0.1.0.dist-info/WHEEL +4 -0
- skewsplit-0.1.0.dist-info/entry_points.txt +3 -0
- skewsplit-0.1.0.dist-info/licenses/LICENSE +201 -0
skewsplit/__init__.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Balanced JDBC partition predicates for Spark, planned from column statistics.
|
|
2
|
+
|
|
3
|
+
:func:`plan` plans from a distribution you already have. To plan from a
|
|
4
|
+
PostgreSQL table's statistics, see :mod:`skewsplit.postgres`.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from importlib.metadata import version
|
|
10
|
+
from typing import TYPE_CHECKING
|
|
11
|
+
|
|
12
|
+
from skewsplit.core import planner
|
|
13
|
+
from skewsplit.core.errors import SpecError
|
|
14
|
+
from skewsplit.core.plan import Bound, Plan, PlanWarning, Predicate
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from collections.abc import Mapping
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"Bound",
|
|
21
|
+
"Plan",
|
|
22
|
+
"PlanWarning",
|
|
23
|
+
"Predicate",
|
|
24
|
+
"SpecError",
|
|
25
|
+
"__version__",
|
|
26
|
+
"plan",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
__version__: str = version("skewsplit")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def plan(distribution: Mapping[str, object], partitions: int) -> Plan:
|
|
33
|
+
"""Plan ``partitions`` predicates from a column's distribution.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
distribution: The input of spec section 1 without ``partitions``: the
|
|
37
|
+
column's name and type, its NULL rows, its most common values and
|
|
38
|
+
a histogram of the other rows, with values in the type's text form.
|
|
39
|
+
partitions: The most predicates to plan besides the NULL predicate,
|
|
40
|
+
1 to 10000.
|
|
41
|
+
|
|
42
|
+
Raises:
|
|
43
|
+
SpecError: If the input is invalid, with the code of spec section 9.
|
|
44
|
+
|
|
45
|
+
>>> import skewsplit
|
|
46
|
+
>>> result = skewsplit.plan(
|
|
47
|
+
... {
|
|
48
|
+
... "column": "customer_id",
|
|
49
|
+
... "type": "int4",
|
|
50
|
+
... "null_rows": 100,
|
|
51
|
+
... "mcv": [{"value": "7", "rows": 60}, {"value": "42", "rows": 500}],
|
|
52
|
+
... "histogram": {"bounds": ["1", "250", "500", "750", "1000"], "rows": 400},
|
|
53
|
+
... },
|
|
54
|
+
... 4,
|
|
55
|
+
... )
|
|
56
|
+
|
|
57
|
+
``predicates`` is the list Spark's JDBC reader takes, one partition each:
|
|
58
|
+
|
|
59
|
+
>>> result.predicates[:3]
|
|
60
|
+
['"customer_id" IS NULL', '"customer_id" < 42', '"customer_id" = 42']
|
|
61
|
+
|
|
62
|
+
``report`` shows the estimated rows of each and the warnings:
|
|
63
|
+
|
|
64
|
+
>>> print(result.report) # doctest: +ELLIPSIS
|
|
65
|
+
customer_id (int4): 1060 estimated rows, 4 of 4 partitions plus one for NULL, skew score 1.887
|
|
66
|
+
rows predicate
|
|
67
|
+
100 "customer_id" IS NULL
|
|
68
|
+
76 "customer_id" < 42
|
|
69
|
+
500 "customer_id" = 42
|
|
70
|
+
192 "customer_id" > 42 AND "customer_id" < 520
|
|
71
|
+
192 "customer_id" >= 520
|
|
72
|
+
warning dominant_value: 42 holds 500 rows, ...
|
|
73
|
+
|
|
74
|
+
``to_dict`` and ``to_json`` give the output of spec section 10:
|
|
75
|
+
|
|
76
|
+
>>> result.to_dict()["skew_score"]
|
|
77
|
+
1.887
|
|
78
|
+
>>> import json
|
|
79
|
+
>>> json.loads(result.to_json()) == result.to_dict()
|
|
80
|
+
True
|
|
81
|
+
"""
|
|
82
|
+
if "partitions" in distribution:
|
|
83
|
+
error = SpecError(
|
|
84
|
+
"invalid_input", "give partitions as the argument of plan(), not in the distribution"
|
|
85
|
+
)
|
|
86
|
+
raise error
|
|
87
|
+
return planner.plan({**distribution, "partitions": partitions})
|
skewsplit/__main__.py
ADDED
skewsplit/cli.py
ADDED
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
"""The ``skewsplit`` command line.
|
|
2
|
+
|
|
3
|
+
``plan`` prints a column's predicates and skew report, ``verify`` checks that
|
|
4
|
+
every row of the table matches exactly one predicate, and ``stats`` prints the
|
|
5
|
+
column's distribution as JSON, for other tools to plan from.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import TYPE_CHECKING, Any, NoReturn
|
|
16
|
+
|
|
17
|
+
from skewsplit import __version__
|
|
18
|
+
from skewsplit.core.errors import SpecError
|
|
19
|
+
from skewsplit.core.inputs import parse_input
|
|
20
|
+
from skewsplit.core.plan import SPEC_VERSION, Plan
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from collections.abc import Callable, Sequence
|
|
24
|
+
|
|
25
|
+
import psycopg
|
|
26
|
+
|
|
27
|
+
from skewsplit.postgres.verify import Verification
|
|
28
|
+
|
|
29
|
+
# What verify needs of a plan: the column, each predicate's SQL and
|
|
30
|
+
# estimated rows, the requested partitions and the estimated skew score.
|
|
31
|
+
PlanFile = tuple[str, list[tuple[str, int]], int, float | None]
|
|
32
|
+
|
|
33
|
+
OK = 0
|
|
34
|
+
VERIFY_FAILED = 1
|
|
35
|
+
USAGE = 2
|
|
36
|
+
INVALID = 3
|
|
37
|
+
DATABASE = 4
|
|
38
|
+
|
|
39
|
+
EPILOG = f"""\
|
|
40
|
+
The output contains real values from the column: treat it like the data.
|
|
41
|
+
|
|
42
|
+
Connect with --dsn, a libpq connection string or URI, or leave it out to use
|
|
43
|
+
the PG* environment variables. A password is better kept in PGPASSWORD or
|
|
44
|
+
~/.pgpass than on the command line, where other users can see it.
|
|
45
|
+
|
|
46
|
+
exit codes:
|
|
47
|
+
{OK} success
|
|
48
|
+
{VERIFY_FAILED} verify found rows that match no predicate, or more than one
|
|
49
|
+
{USAGE} the command line is wrong, or skewsplit[postgres] is not installed
|
|
50
|
+
{INVALID} the column cannot be planned, or the plan file is not valid
|
|
51
|
+
{DATABASE} the database could not be reached, or a query failed or timed out
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
_PLAN_FORMATS: dict[str, Callable[[Plan], str]] = {
|
|
56
|
+
"text": lambda plan: plan.report,
|
|
57
|
+
"json": lambda plan: plan.to_json(indent=2),
|
|
58
|
+
"pyspark": Plan.to_pyspark,
|
|
59
|
+
"scala": Plan.to_scala,
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class CommandError(Exception):
|
|
64
|
+
"""A command failed: print ``message`` and exit with ``code``."""
|
|
65
|
+
|
|
66
|
+
def __init__(self, code: int, message: str) -> None:
|
|
67
|
+
"""Create a failure with an exit code."""
|
|
68
|
+
super().__init__(message)
|
|
69
|
+
self.code = code
|
|
70
|
+
self.message = message
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
74
|
+
"""Return the argument parser for the ``skewsplit`` command."""
|
|
75
|
+
parser = argparse.ArgumentParser(
|
|
76
|
+
prog="skewsplit",
|
|
77
|
+
description="Plan balanced JDBC partition predicates for Spark.",
|
|
78
|
+
epilog=EPILOG,
|
|
79
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
80
|
+
)
|
|
81
|
+
parser.add_argument("--version", action="version", version=f"skewsplit {__version__}")
|
|
82
|
+
commands = parser.add_subparsers(dest="command", metavar="COMMAND", required=True)
|
|
83
|
+
|
|
84
|
+
def command(name: str, summary: str) -> argparse.ArgumentParser:
|
|
85
|
+
sub = commands.add_parser(
|
|
86
|
+
name,
|
|
87
|
+
help=summary,
|
|
88
|
+
description=f"{summary}.",
|
|
89
|
+
epilog=EPILOG,
|
|
90
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
91
|
+
)
|
|
92
|
+
sub.add_argument("--dsn", default="", help="libpq connection string or URI")
|
|
93
|
+
sub.add_argument("--table", required=True, help="the table, as SQL names it")
|
|
94
|
+
return sub
|
|
95
|
+
|
|
96
|
+
plan = command("plan", "Plan predicates from the column's statistics in pg_stats")
|
|
97
|
+
plan.add_argument("--column", required=True, help="the column to split, not quoted")
|
|
98
|
+
plan.add_argument("--partitions", required=True, type=int, metavar="N")
|
|
99
|
+
plan.add_argument(
|
|
100
|
+
"--format",
|
|
101
|
+
choices=tuple(_PLAN_FORMATS),
|
|
102
|
+
default="text",
|
|
103
|
+
help="the skew report, the plan as JSON, or PySpark or Scala code that reads the table",
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
verify = command("verify", "Check that every row of the table matches exactly one predicate")
|
|
107
|
+
verify.add_argument(
|
|
108
|
+
"--plan",
|
|
109
|
+
metavar="FILE",
|
|
110
|
+
help="a plan from `plan --format json`, or - for standard input;"
|
|
111
|
+
" without it, plan from the statistics with --column and --partitions",
|
|
112
|
+
)
|
|
113
|
+
verify.add_argument("--column", help="the column to split, not quoted")
|
|
114
|
+
verify.add_argument("--partitions", type=int, metavar="N")
|
|
115
|
+
verify.add_argument(
|
|
116
|
+
"--counts",
|
|
117
|
+
action="store_true",
|
|
118
|
+
help="also count each predicate's rows, one query each, for the actual skew",
|
|
119
|
+
)
|
|
120
|
+
verify.add_argument(
|
|
121
|
+
"--statement-timeout-ms",
|
|
122
|
+
type=_positive,
|
|
123
|
+
metavar="MS",
|
|
124
|
+
help="cancel a query that runs longer than this",
|
|
125
|
+
)
|
|
126
|
+
verify.add_argument("--format", choices=("text", "json"), default="text")
|
|
127
|
+
|
|
128
|
+
stats = command(
|
|
129
|
+
"stats", "Print the column's distribution from pg_stats as JSON (spec section 1)"
|
|
130
|
+
)
|
|
131
|
+
stats.add_argument("--column", required=True, help="the column, not quoted")
|
|
132
|
+
stats.add_argument(
|
|
133
|
+
"--partitions",
|
|
134
|
+
type=int,
|
|
135
|
+
metavar="N",
|
|
136
|
+
help="add partitions, which makes the output a complete planner input",
|
|
137
|
+
)
|
|
138
|
+
return parser
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
142
|
+
"""Run the command line and return its exit code."""
|
|
143
|
+
parser = build_parser()
|
|
144
|
+
args = parser.parse_args(argv)
|
|
145
|
+
if args.command == "verify":
|
|
146
|
+
if args.plan is not None and (args.column is not None or args.partitions is not None):
|
|
147
|
+
parser.error("verify takes --plan, or --column and --partitions, not both")
|
|
148
|
+
if args.plan is None and (args.column is None or args.partitions is None):
|
|
149
|
+
parser.error("verify needs --plan, or --column and --partitions")
|
|
150
|
+
try:
|
|
151
|
+
output, code = _run(args)
|
|
152
|
+
except CommandError as error:
|
|
153
|
+
sys.stderr.write(f"skewsplit: {error.message}\n")
|
|
154
|
+
return error.code
|
|
155
|
+
sys.stdout.write(output + "\n")
|
|
156
|
+
return code
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _run(args: argparse.Namespace) -> tuple[str, int]:
|
|
160
|
+
plan_file: PlanFile | None = None
|
|
161
|
+
if args.command == "verify" and args.plan is not None:
|
|
162
|
+
plan_file = _read_plan(args.plan)
|
|
163
|
+
try:
|
|
164
|
+
import psycopg # noqa: PLC0415 -- the postgres extra is optional
|
|
165
|
+
|
|
166
|
+
from skewsplit import postgres # noqa: PLC0415
|
|
167
|
+
from skewsplit.postgres.verify import verification_report # noqa: PLC0415
|
|
168
|
+
except ModuleNotFoundError as error:
|
|
169
|
+
if error.name != "psycopg":
|
|
170
|
+
raise
|
|
171
|
+
message = "this command needs psycopg: pip install 'skewsplit[postgres]'"
|
|
172
|
+
raise CommandError(USAGE, message) from None
|
|
173
|
+
secrets = passwords(args.dsn)
|
|
174
|
+
try:
|
|
175
|
+
with _connect(args.dsn, secrets) as conn:
|
|
176
|
+
if args.command == "plan":
|
|
177
|
+
result = postgres.plan(conn, args.table, args.column, args.partitions)
|
|
178
|
+
return _PLAN_FORMATS[args.format](result), OK
|
|
179
|
+
if args.command == "stats":
|
|
180
|
+
return _stats(postgres.distribution(conn, args.table, args.column), args), OK
|
|
181
|
+
if plan_file is None:
|
|
182
|
+
planned = postgres.plan(conn, args.table, args.column, args.partitions)
|
|
183
|
+
plan_file = (
|
|
184
|
+
planned.column,
|
|
185
|
+
[(part.sql, part.estimated_rows) for part in planned.parts],
|
|
186
|
+
planned.requested_partitions,
|
|
187
|
+
planned.skew_score,
|
|
188
|
+
)
|
|
189
|
+
column, estimates, partitions, skew = plan_file
|
|
190
|
+
found = postgres.verify(
|
|
191
|
+
conn,
|
|
192
|
+
args.table,
|
|
193
|
+
column,
|
|
194
|
+
[predicate for predicate, _ in estimates],
|
|
195
|
+
counts=args.counts,
|
|
196
|
+
statement_timeout_ms=args.statement_timeout_ms,
|
|
197
|
+
)
|
|
198
|
+
except (SpecError, postgres.CatalogError) as error:
|
|
199
|
+
raise CommandError(INVALID, f"{error.code}: {error.message}") from None
|
|
200
|
+
except psycopg.Error as error:
|
|
201
|
+
raise CommandError(DATABASE, redact(str(error).strip(), secrets)) from None
|
|
202
|
+
code = OK if found.ok else VERIFY_FAILED
|
|
203
|
+
if args.format == "json":
|
|
204
|
+
report = verification_report(found, estimates, partitions, skew)
|
|
205
|
+
return json.dumps(report, indent=2, ensure_ascii=False), code
|
|
206
|
+
return _verify_text(found, estimates, partitions, skew), code
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _read_plan(name: str) -> PlanFile:
|
|
210
|
+
"""Read the parts of a JSON plan that verify needs."""
|
|
211
|
+
where = "the plan on standard input" if name == "-" else f"the plan {name}"
|
|
212
|
+
data = _load(name, where)
|
|
213
|
+
if not isinstance(data, dict):
|
|
214
|
+
_invalid(f"{where} is not a JSON object")
|
|
215
|
+
if data.get("spec_version") != SPEC_VERSION:
|
|
216
|
+
_invalid(f"{where} needs spec_version {SPEC_VERSION!r}, as `skewsplit plan` writes")
|
|
217
|
+
column = data.get("column")
|
|
218
|
+
partitions = _integer(data.get("requested_partitions"))
|
|
219
|
+
predicates = data.get("predicates")
|
|
220
|
+
if not isinstance(column, str) or partitions is None:
|
|
221
|
+
_invalid(f"{where} needs a column and requested_partitions")
|
|
222
|
+
if not isinstance(predicates, list) or not predicates:
|
|
223
|
+
_invalid(f"{where} needs a list of predicates")
|
|
224
|
+
estimates = []
|
|
225
|
+
for predicate in predicates:
|
|
226
|
+
fields = predicate if isinstance(predicate, dict) else {}
|
|
227
|
+
sql, rows = fields.get("sql"), _integer(fields.get("estimated_rows"))
|
|
228
|
+
if not isinstance(sql, str) or rows is None:
|
|
229
|
+
_invalid(f"{where} needs the sql and estimated_rows of every predicate")
|
|
230
|
+
estimates.append((sql, rows))
|
|
231
|
+
skew = data.get("skew_score")
|
|
232
|
+
if skew is None:
|
|
233
|
+
return column, estimates, partitions, None
|
|
234
|
+
if isinstance(skew, bool) or not isinstance(skew, (int, float)):
|
|
235
|
+
_invalid(f"{where} needs a skew_score that is a number or null")
|
|
236
|
+
return column, estimates, partitions, float(skew)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _load(name: str, where: str) -> object:
|
|
240
|
+
try:
|
|
241
|
+
text = sys.stdin.read() if name == "-" else Path(name).read_text(encoding="utf-8")
|
|
242
|
+
return json.loads(text)
|
|
243
|
+
except OSError as error:
|
|
244
|
+
_invalid(f"cannot read {where}: {error.strerror}")
|
|
245
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
|
246
|
+
_invalid(f"{where} is not JSON: {error}")
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _stats(data: dict[str, object], args: argparse.Namespace) -> str:
|
|
250
|
+
if args.partitions is not None:
|
|
251
|
+
# In the order of spec section 1, and checked as a planner input.
|
|
252
|
+
data = {
|
|
253
|
+
**{key: data[key] for key in ("column", "type", "scale") if key in data},
|
|
254
|
+
"partitions": args.partitions,
|
|
255
|
+
**data,
|
|
256
|
+
}
|
|
257
|
+
parse_input(data)
|
|
258
|
+
return json.dumps(data, indent=2, ensure_ascii=False)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _verify_text(
|
|
262
|
+
found: Verification,
|
|
263
|
+
estimates: list[tuple[str, int]],
|
|
264
|
+
partitions: int,
|
|
265
|
+
skew: float | None,
|
|
266
|
+
) -> str:
|
|
267
|
+
summary = (
|
|
268
|
+
f"OK: each of the {found.total_rows} rows of {found.table} matches exactly one predicate"
|
|
269
|
+
)
|
|
270
|
+
if not found.ok:
|
|
271
|
+
summary = (
|
|
272
|
+
f"FAILED: of the {found.total_rows} rows of {found.table}, no predicate matches"
|
|
273
|
+
f" {found.unmatched_rows} and more than one matches {found.overlapping_rows}"
|
|
274
|
+
)
|
|
275
|
+
lines = [summary]
|
|
276
|
+
if found.predicate_rows is not None:
|
|
277
|
+
counted = zip(estimates, found.predicate_rows)
|
|
278
|
+
rows = [(str(estimated), str(actual)) for (_, estimated), actual in counted]
|
|
279
|
+
left = max(len("estimated"), *(len(estimated) for estimated, _ in rows))
|
|
280
|
+
right = max(len("actual"), *(len(actual) for _, actual in rows))
|
|
281
|
+
lines.append(f"{'estimated':>{left}} {'actual':>{right}} predicate")
|
|
282
|
+
lines.extend(
|
|
283
|
+
f"{estimated:>{left}} {actual:>{right}} {sql}"
|
|
284
|
+
for (estimated, actual), (sql, _) in zip(rows, estimates)
|
|
285
|
+
)
|
|
286
|
+
actual_skew = found.skew_milli(partitions)
|
|
287
|
+
lines.append(
|
|
288
|
+
f"skew score: estimated {_score(skew)},"
|
|
289
|
+
f" actual {_score(None if actual_skew is None else actual_skew / 1000)}"
|
|
290
|
+
)
|
|
291
|
+
return "\n".join(lines)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _score(skew: float | None) -> str:
|
|
295
|
+
return "none" if skew is None else f"{skew:.3f}"
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def passwords(dsn: str) -> set[str]:
|
|
299
|
+
"""Return the passwords that error messages must not show."""
|
|
300
|
+
import psycopg # noqa: PLC0415
|
|
301
|
+
from psycopg.conninfo import conninfo_to_dict # noqa: PLC0415
|
|
302
|
+
|
|
303
|
+
try:
|
|
304
|
+
params = conninfo_to_dict(dsn)
|
|
305
|
+
except psycopg.Error:
|
|
306
|
+
# Not libpq's message: it can quote any part of the string, the password too.
|
|
307
|
+
raise CommandError(USAGE, "--dsn is not a valid libpq connection string or URI") from None
|
|
308
|
+
found = {str(params.get("password") or ""), os.environ.get("PGPASSWORD", "")}
|
|
309
|
+
return found - {""}
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _connect(dsn: str, secrets: set[str]) -> psycopg.Connection[Any]:
|
|
313
|
+
import psycopg # noqa: PLC0415
|
|
314
|
+
|
|
315
|
+
try:
|
|
316
|
+
return psycopg.connect(dsn, autocommit=True)
|
|
317
|
+
except psycopg.Error as error:
|
|
318
|
+
message = redact(str(error).strip(), secrets)
|
|
319
|
+
raise CommandError(DATABASE, f"cannot connect: {message}") from None
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def redact(message: str, secrets: set[str]) -> str:
|
|
323
|
+
"""Return ``message`` with each of ``secrets`` replaced, the longest first."""
|
|
324
|
+
for secret in sorted(secrets, key=len, reverse=True):
|
|
325
|
+
message = message.replace(secret, "********")
|
|
326
|
+
return message
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _positive(text: str) -> int:
|
|
330
|
+
try:
|
|
331
|
+
value = int(text)
|
|
332
|
+
except ValueError:
|
|
333
|
+
value = 0
|
|
334
|
+
if value <= 0:
|
|
335
|
+
msg = f"must be a positive whole number, not {text!r}"
|
|
336
|
+
raise argparse.ArgumentTypeError(msg)
|
|
337
|
+
return value
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _integer(value: object) -> int | None:
|
|
341
|
+
return value if isinstance(value, int) and not isinstance(value, bool) else None
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _invalid(message: str) -> NoReturn:
|
|
345
|
+
raise CommandError(INVALID, message)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""The pure planning core.
|
|
2
|
+
|
|
3
|
+
Everything in this package uses the standard library only and does no I/O.
|
|
4
|
+
Adapters such as the PostgreSQL reader turn database statistics into the core's
|
|
5
|
+
input shape; the core turns that shape into predicates. See
|
|
6
|
+
docs/adr/0001-monorepo-and-pure-core.md.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
__all__: list[str] = []
|