skewsplit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
skewsplit/__init__.py ADDED
@@ -0,0 +1,87 @@
1
+ """Balanced JDBC partition predicates for Spark, planned from column statistics.
2
+
3
+ :func:`plan` plans from a distribution you already have. To plan from a
4
+ PostgreSQL table's statistics, see :mod:`skewsplit.postgres`.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from importlib.metadata import version
10
+ from typing import TYPE_CHECKING
11
+
12
+ from skewsplit.core import planner
13
+ from skewsplit.core.errors import SpecError
14
+ from skewsplit.core.plan import Bound, Plan, PlanWarning, Predicate
15
+
16
+ if TYPE_CHECKING:
17
+ from collections.abc import Mapping
18
+
19
+ __all__ = [
20
+ "Bound",
21
+ "Plan",
22
+ "PlanWarning",
23
+ "Predicate",
24
+ "SpecError",
25
+ "__version__",
26
+ "plan",
27
+ ]
28
+
29
+ __version__: str = version("skewsplit")
30
+
31
+
32
+ def plan(distribution: Mapping[str, object], partitions: int) -> Plan:
33
+ """Plan ``partitions`` predicates from a column's distribution.
34
+
35
+ Args:
36
+ distribution: The input of spec section 1 without ``partitions``: the
37
+ column's name and type, its NULL rows, its most common values and
38
+ a histogram of the other rows, with values in the type's text form.
39
+ partitions: The most predicates to plan besides the NULL predicate,
40
+ 1 to 10000.
41
+
42
+ Raises:
43
+ SpecError: If the input is invalid, with the code of spec section 9.
44
+
45
+ >>> import skewsplit
46
+ >>> result = skewsplit.plan(
47
+ ... {
48
+ ... "column": "customer_id",
49
+ ... "type": "int4",
50
+ ... "null_rows": 100,
51
+ ... "mcv": [{"value": "7", "rows": 60}, {"value": "42", "rows": 500}],
52
+ ... "histogram": {"bounds": ["1", "250", "500", "750", "1000"], "rows": 400},
53
+ ... },
54
+ ... 4,
55
+ ... )
56
+
57
+ ``predicates`` is the list Spark's JDBC reader takes, one partition each:
58
+
59
+ >>> result.predicates[:3]
60
+ ['"customer_id" IS NULL', '"customer_id" < 42', '"customer_id" = 42']
61
+
62
+ ``report`` shows the estimated rows of each and the warnings:
63
+
64
+ >>> print(result.report) # doctest: +ELLIPSIS
65
+ customer_id (int4): 1060 estimated rows, 4 of 4 partitions plus one for NULL, skew score 1.887
66
+ rows predicate
67
+ 100 "customer_id" IS NULL
68
+ 76 "customer_id" < 42
69
+ 500 "customer_id" = 42
70
+ 192 "customer_id" > 42 AND "customer_id" < 520
71
+ 192 "customer_id" >= 520
72
+ warning dominant_value: 42 holds 500 rows, ...
73
+
74
+ ``to_dict`` and ``to_json`` give the output of spec section 10:
75
+
76
+ >>> result.to_dict()["skew_score"]
77
+ 1.887
78
+ >>> import json
79
+ >>> json.loads(result.to_json()) == result.to_dict()
80
+ True
81
+ """
82
+ if "partitions" in distribution:
83
+ error = SpecError(
84
+ "invalid_input", "give partitions as the argument of plan(), not in the distribution"
85
+ )
86
+ raise error
87
+ return planner.plan({**distribution, "partitions": partitions})
skewsplit/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ """Allow ``python -m skewsplit``."""
2
+
3
+ from skewsplit.cli import main
4
+
5
+ raise SystemExit(main())
skewsplit/cli.py ADDED
@@ -0,0 +1,345 @@
1
+ """The ``skewsplit`` command line.
2
+
3
+ ``plan`` prints a column's predicates and skew report, ``verify`` checks that
4
+ every row of the table matches exactly one predicate, and ``stats`` prints the
5
+ column's distribution as JSON, for other tools to plan from.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import json
12
+ import os
13
+ import sys
14
+ from pathlib import Path
15
+ from typing import TYPE_CHECKING, Any, NoReturn
16
+
17
+ from skewsplit import __version__
18
+ from skewsplit.core.errors import SpecError
19
+ from skewsplit.core.inputs import parse_input
20
+ from skewsplit.core.plan import SPEC_VERSION, Plan
21
+
22
+ if TYPE_CHECKING:
23
+ from collections.abc import Callable, Sequence
24
+
25
+ import psycopg
26
+
27
+ from skewsplit.postgres.verify import Verification
28
+
29
+ # What verify needs of a plan: the column, each predicate's SQL and
30
+ # estimated rows, the requested partitions and the estimated skew score.
31
+ PlanFile = tuple[str, list[tuple[str, int]], int, float | None]
32
+
33
+ OK = 0
34
+ VERIFY_FAILED = 1
35
+ USAGE = 2
36
+ INVALID = 3
37
+ DATABASE = 4
38
+
39
+ EPILOG = f"""\
40
+ The output contains real values from the column: treat it like the data.
41
+
42
+ Connect with --dsn, a libpq connection string or URI, or leave it out to use
43
+ the PG* environment variables. A password is better kept in PGPASSWORD or
44
+ ~/.pgpass than on the command line, where other users can see it.
45
+
46
+ exit codes:
47
+ {OK} success
48
+ {VERIFY_FAILED} verify found rows that match no predicate, or more than one
49
+ {USAGE} the command line is wrong, or skewsplit[postgres] is not installed
50
+ {INVALID} the column cannot be planned, or the plan file is not valid
51
+ {DATABASE} the database could not be reached, or a query failed or timed out
52
+ """
53
+
54
+
55
+ _PLAN_FORMATS: dict[str, Callable[[Plan], str]] = {
56
+ "text": lambda plan: plan.report,
57
+ "json": lambda plan: plan.to_json(indent=2),
58
+ "pyspark": Plan.to_pyspark,
59
+ "scala": Plan.to_scala,
60
+ }
61
+
62
+
63
+ class CommandError(Exception):
64
+ """A command failed: print ``message`` and exit with ``code``."""
65
+
66
+ def __init__(self, code: int, message: str) -> None:
67
+ """Create a failure with an exit code."""
68
+ super().__init__(message)
69
+ self.code = code
70
+ self.message = message
71
+
72
+
73
+ def build_parser() -> argparse.ArgumentParser:
74
+ """Return the argument parser for the ``skewsplit`` command."""
75
+ parser = argparse.ArgumentParser(
76
+ prog="skewsplit",
77
+ description="Plan balanced JDBC partition predicates for Spark.",
78
+ epilog=EPILOG,
79
+ formatter_class=argparse.RawDescriptionHelpFormatter,
80
+ )
81
+ parser.add_argument("--version", action="version", version=f"skewsplit {__version__}")
82
+ commands = parser.add_subparsers(dest="command", metavar="COMMAND", required=True)
83
+
84
+ def command(name: str, summary: str) -> argparse.ArgumentParser:
85
+ sub = commands.add_parser(
86
+ name,
87
+ help=summary,
88
+ description=f"{summary}.",
89
+ epilog=EPILOG,
90
+ formatter_class=argparse.RawDescriptionHelpFormatter,
91
+ )
92
+ sub.add_argument("--dsn", default="", help="libpq connection string or URI")
93
+ sub.add_argument("--table", required=True, help="the table, as SQL names it")
94
+ return sub
95
+
96
+ plan = command("plan", "Plan predicates from the column's statistics in pg_stats")
97
+ plan.add_argument("--column", required=True, help="the column to split, not quoted")
98
+ plan.add_argument("--partitions", required=True, type=int, metavar="N")
99
+ plan.add_argument(
100
+ "--format",
101
+ choices=tuple(_PLAN_FORMATS),
102
+ default="text",
103
+ help="the skew report, the plan as JSON, or PySpark or Scala code that reads the table",
104
+ )
105
+
106
+ verify = command("verify", "Check that every row of the table matches exactly one predicate")
107
+ verify.add_argument(
108
+ "--plan",
109
+ metavar="FILE",
110
+ help="a plan from `plan --format json`, or - for standard input;"
111
+ " without it, plan from the statistics with --column and --partitions",
112
+ )
113
+ verify.add_argument("--column", help="the column to split, not quoted")
114
+ verify.add_argument("--partitions", type=int, metavar="N")
115
+ verify.add_argument(
116
+ "--counts",
117
+ action="store_true",
118
+ help="also count each predicate's rows, one query each, for the actual skew",
119
+ )
120
+ verify.add_argument(
121
+ "--statement-timeout-ms",
122
+ type=_positive,
123
+ metavar="MS",
124
+ help="cancel a query that runs longer than this",
125
+ )
126
+ verify.add_argument("--format", choices=("text", "json"), default="text")
127
+
128
+ stats = command(
129
+ "stats", "Print the column's distribution from pg_stats as JSON (spec section 1)"
130
+ )
131
+ stats.add_argument("--column", required=True, help="the column, not quoted")
132
+ stats.add_argument(
133
+ "--partitions",
134
+ type=int,
135
+ metavar="N",
136
+ help="add partitions, which makes the output a complete planner input",
137
+ )
138
+ return parser
139
+
140
+
141
+ def main(argv: Sequence[str] | None = None) -> int:
142
+ """Run the command line and return its exit code."""
143
+ parser = build_parser()
144
+ args = parser.parse_args(argv)
145
+ if args.command == "verify":
146
+ if args.plan is not None and (args.column is not None or args.partitions is not None):
147
+ parser.error("verify takes --plan, or --column and --partitions, not both")
148
+ if args.plan is None and (args.column is None or args.partitions is None):
149
+ parser.error("verify needs --plan, or --column and --partitions")
150
+ try:
151
+ output, code = _run(args)
152
+ except CommandError as error:
153
+ sys.stderr.write(f"skewsplit: {error.message}\n")
154
+ return error.code
155
+ sys.stdout.write(output + "\n")
156
+ return code
157
+
158
+
159
+ def _run(args: argparse.Namespace) -> tuple[str, int]:
160
+ plan_file: PlanFile | None = None
161
+ if args.command == "verify" and args.plan is not None:
162
+ plan_file = _read_plan(args.plan)
163
+ try:
164
+ import psycopg # noqa: PLC0415 -- the postgres extra is optional
165
+
166
+ from skewsplit import postgres # noqa: PLC0415
167
+ from skewsplit.postgres.verify import verification_report # noqa: PLC0415
168
+ except ModuleNotFoundError as error:
169
+ if error.name != "psycopg":
170
+ raise
171
+ message = "this command needs psycopg: pip install 'skewsplit[postgres]'"
172
+ raise CommandError(USAGE, message) from None
173
+ secrets = passwords(args.dsn)
174
+ try:
175
+ with _connect(args.dsn, secrets) as conn:
176
+ if args.command == "plan":
177
+ result = postgres.plan(conn, args.table, args.column, args.partitions)
178
+ return _PLAN_FORMATS[args.format](result), OK
179
+ if args.command == "stats":
180
+ return _stats(postgres.distribution(conn, args.table, args.column), args), OK
181
+ if plan_file is None:
182
+ planned = postgres.plan(conn, args.table, args.column, args.partitions)
183
+ plan_file = (
184
+ planned.column,
185
+ [(part.sql, part.estimated_rows) for part in planned.parts],
186
+ planned.requested_partitions,
187
+ planned.skew_score,
188
+ )
189
+ column, estimates, partitions, skew = plan_file
190
+ found = postgres.verify(
191
+ conn,
192
+ args.table,
193
+ column,
194
+ [predicate for predicate, _ in estimates],
195
+ counts=args.counts,
196
+ statement_timeout_ms=args.statement_timeout_ms,
197
+ )
198
+ except (SpecError, postgres.CatalogError) as error:
199
+ raise CommandError(INVALID, f"{error.code}: {error.message}") from None
200
+ except psycopg.Error as error:
201
+ raise CommandError(DATABASE, redact(str(error).strip(), secrets)) from None
202
+ code = OK if found.ok else VERIFY_FAILED
203
+ if args.format == "json":
204
+ report = verification_report(found, estimates, partitions, skew)
205
+ return json.dumps(report, indent=2, ensure_ascii=False), code
206
+ return _verify_text(found, estimates, partitions, skew), code
207
+
208
+
209
+ def _read_plan(name: str) -> PlanFile:
210
+ """Read the parts of a JSON plan that verify needs."""
211
+ where = "the plan on standard input" if name == "-" else f"the plan {name}"
212
+ data = _load(name, where)
213
+ if not isinstance(data, dict):
214
+ _invalid(f"{where} is not a JSON object")
215
+ if data.get("spec_version") != SPEC_VERSION:
216
+ _invalid(f"{where} needs spec_version {SPEC_VERSION!r}, as `skewsplit plan` writes")
217
+ column = data.get("column")
218
+ partitions = _integer(data.get("requested_partitions"))
219
+ predicates = data.get("predicates")
220
+ if not isinstance(column, str) or partitions is None:
221
+ _invalid(f"{where} needs a column and requested_partitions")
222
+ if not isinstance(predicates, list) or not predicates:
223
+ _invalid(f"{where} needs a list of predicates")
224
+ estimates = []
225
+ for predicate in predicates:
226
+ fields = predicate if isinstance(predicate, dict) else {}
227
+ sql, rows = fields.get("sql"), _integer(fields.get("estimated_rows"))
228
+ if not isinstance(sql, str) or rows is None:
229
+ _invalid(f"{where} needs the sql and estimated_rows of every predicate")
230
+ estimates.append((sql, rows))
231
+ skew = data.get("skew_score")
232
+ if skew is None:
233
+ return column, estimates, partitions, None
234
+ if isinstance(skew, bool) or not isinstance(skew, (int, float)):
235
+ _invalid(f"{where} needs a skew_score that is a number or null")
236
+ return column, estimates, partitions, float(skew)
237
+
238
+
239
+ def _load(name: str, where: str) -> object:
240
+ try:
241
+ text = sys.stdin.read() if name == "-" else Path(name).read_text(encoding="utf-8")
242
+ return json.loads(text)
243
+ except OSError as error:
244
+ _invalid(f"cannot read {where}: {error.strerror}")
245
+ except (UnicodeDecodeError, json.JSONDecodeError) as error:
246
+ _invalid(f"{where} is not JSON: {error}")
247
+
248
+
249
+ def _stats(data: dict[str, object], args: argparse.Namespace) -> str:
250
+ if args.partitions is not None:
251
+ # In the order of spec section 1, and checked as a planner input.
252
+ data = {
253
+ **{key: data[key] for key in ("column", "type", "scale") if key in data},
254
+ "partitions": args.partitions,
255
+ **data,
256
+ }
257
+ parse_input(data)
258
+ return json.dumps(data, indent=2, ensure_ascii=False)
259
+
260
+
261
+ def _verify_text(
262
+ found: Verification,
263
+ estimates: list[tuple[str, int]],
264
+ partitions: int,
265
+ skew: float | None,
266
+ ) -> str:
267
+ summary = (
268
+ f"OK: each of the {found.total_rows} rows of {found.table} matches exactly one predicate"
269
+ )
270
+ if not found.ok:
271
+ summary = (
272
+ f"FAILED: of the {found.total_rows} rows of {found.table}, no predicate matches"
273
+ f" {found.unmatched_rows} and more than one matches {found.overlapping_rows}"
274
+ )
275
+ lines = [summary]
276
+ if found.predicate_rows is not None:
277
+ counted = zip(estimates, found.predicate_rows)
278
+ rows = [(str(estimated), str(actual)) for (_, estimated), actual in counted]
279
+ left = max(len("estimated"), *(len(estimated) for estimated, _ in rows))
280
+ right = max(len("actual"), *(len(actual) for _, actual in rows))
281
+ lines.append(f"{'estimated':>{left}} {'actual':>{right}} predicate")
282
+ lines.extend(
283
+ f"{estimated:>{left}} {actual:>{right}} {sql}"
284
+ for (estimated, actual), (sql, _) in zip(rows, estimates)
285
+ )
286
+ actual_skew = found.skew_milli(partitions)
287
+ lines.append(
288
+ f"skew score: estimated {_score(skew)},"
289
+ f" actual {_score(None if actual_skew is None else actual_skew / 1000)}"
290
+ )
291
+ return "\n".join(lines)
292
+
293
+
294
+ def _score(skew: float | None) -> str:
295
+ return "none" if skew is None else f"{skew:.3f}"
296
+
297
+
298
+ def passwords(dsn: str) -> set[str]:
299
+ """Return the passwords that error messages must not show."""
300
+ import psycopg # noqa: PLC0415
301
+ from psycopg.conninfo import conninfo_to_dict # noqa: PLC0415
302
+
303
+ try:
304
+ params = conninfo_to_dict(dsn)
305
+ except psycopg.Error:
306
+ # Not libpq's message: it can quote any part of the string, the password too.
307
+ raise CommandError(USAGE, "--dsn is not a valid libpq connection string or URI") from None
308
+ found = {str(params.get("password") or ""), os.environ.get("PGPASSWORD", "")}
309
+ return found - {""}
310
+
311
+
312
+ def _connect(dsn: str, secrets: set[str]) -> psycopg.Connection[Any]:
313
+ import psycopg # noqa: PLC0415
314
+
315
+ try:
316
+ return psycopg.connect(dsn, autocommit=True)
317
+ except psycopg.Error as error:
318
+ message = redact(str(error).strip(), secrets)
319
+ raise CommandError(DATABASE, f"cannot connect: {message}") from None
320
+
321
+
322
+ def redact(message: str, secrets: set[str]) -> str:
323
+ """Return ``message`` with each of ``secrets`` replaced, the longest first."""
324
+ for secret in sorted(secrets, key=len, reverse=True):
325
+ message = message.replace(secret, "********")
326
+ return message
327
+
328
+
329
+ def _positive(text: str) -> int:
330
+ try:
331
+ value = int(text)
332
+ except ValueError:
333
+ value = 0
334
+ if value <= 0:
335
+ msg = f"must be a positive whole number, not {text!r}"
336
+ raise argparse.ArgumentTypeError(msg)
337
+ return value
338
+
339
+
340
+ def _integer(value: object) -> int | None:
341
+ return value if isinstance(value, int) and not isinstance(value, bool) else None
342
+
343
+
344
+ def _invalid(message: str) -> NoReturn:
345
+ raise CommandError(INVALID, message)
@@ -0,0 +1,9 @@
1
+ """The pure planning core.
2
+
3
+ Everything in this package uses the standard library only and does no I/O.
4
+ Adapters such as the PostgreSQL reader turn database statistics into the core's
5
+ input shape; the core turns that shape into predicates. See
6
+ docs/adr/0001-monorepo-and-pure-core.md.
7
+ """
8
+
9
+ __all__: list[str] = []