rowvioli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rowvioli/__init__.py ADDED
File without changes
rowvioli/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ main()
rowvioli/apply.py ADDED
@@ -0,0 +1,45 @@
1
+ """Write the definition into the target. Needs no source and no driver for one."""
2
+ import base64
3
+ import json
4
+ import os
5
+ from importlib.resources import files
6
+
7
+ import adbc_driver_postgresql.dbapi as adbc
8
+
9
+ from . import definition, seal, target
10
+
11
+
12
+ def apply(directory, key_file, init=False):
13
+ sources, tables, digest = definition.read(directory, os.environ)
14
+ try:
15
+ public = seal.public_key(key_file)
16
+ except OSError as e:
17
+ raise SystemExit(f"public key {key_file}: {e.strerror}")
18
+ except Exception:
19
+ raise SystemExit(f"{key_file} holds no public key")
20
+ for s in sources:
21
+ s["connection"] = base64.b64encode(seal.seal(public, s["name"], s["connection"])).decode()
22
+ s["key_id"] = seal.key_id(public)
23
+ try:
24
+ conn = target.connect()
25
+ except SystemExit as e:
26
+ if not init or e.code != target.NO_SCHEMA:
27
+ raise
28
+ conn = target.connect(init=True)
29
+ print("the schema _rowvioli is ready in the target")
30
+ # a changed target name moves nothing: say so, so that no stale mirror is left behind unknowingly
31
+ old = {(src, tbl): f"{schema}.{name}" for src, tbl, schema, name in target.sql(
32
+ conn, "SELECT source, source_table, target_schema, target_table FROM _rowvioli.tables")}
33
+ changed = []
34
+ for t in tables:
35
+ was, now = old.get((t["source"], t["source_table"])), f"{t['target_schema']}.{t['target_table']}"
36
+ if was not in (None, now):
37
+ changed.append(f"{t['source']}: {t['source_table']}: target name changes from {was} to {now}; the old "
38
+ f"tables ({was}, its __log and __history) stay where they are")
39
+ payload = json.dumps({"sources": sources, "tables": tables, "definition": digest}, default=str)
40
+ try:
41
+ target.transaction(conn, files(__package__).joinpath("sql/apply.sql").read_text(), [payload])
42
+ except adbc.Error as e: # Postgres names what it refused, for example a time zone it does not know
43
+ raise SystemExit(f"the target refused the definition, nothing was changed: {e}")
44
+ print(*changed, sep="\n", end="\n" if changed else "")
45
+ print(f"applied {len(sources)} sources with {len(tables)} tables")
rowvioli/cli.py ADDED
@@ -0,0 +1,119 @@
1
+ """The commands: arguments, the answer on stdout, the exit code."""
2
+ import argparse
3
+ import importlib.metadata
4
+ import json
5
+ import os
6
+ import sys
7
+ import time
8
+ from pathlib import Path
9
+
10
+ import adbc_driver_postgresql.dbapi as adbc
11
+
12
+ from . import apply, contract, definition, load, seal, target, work
13
+
14
+
15
+ def main():
16
+ p = argparse.ArgumentParser(prog="rowvioli")
17
+ p.add_argument("--version", action="version", version=importlib.metadata.version("rowvioli"))
18
+ sub = p.add_subparsers(dest="command", required=True)
19
+ sub.add_parser("init", help="create the schema _rowvioli in the target")
20
+ sub.add_parser("key", help="create a key pair").add_argument("action", choices=["new"])
21
+ lp = sub.add_parser("list", help="print the tables of a source")
22
+ lp.add_argument("source")
23
+ lp.add_argument("--definition", default="sources", help="directory of the definition")
24
+ ap = sub.add_parser("apply", help="write the definition into the target")
25
+ ap.add_argument("directory")
26
+ ap.add_argument("--check", action="store_true", help="only read and check the definition")
27
+ ap.add_argument("--init", action="store_true", help="create the schema _rowvioli first if it is missing")
28
+ ap.add_argument("--key", help="public key file (default: worker.pub next to the directory)")
29
+ rp = sub.add_parser("run", help="load what is due, once")
30
+ rp.add_argument("source", nargs="?")
31
+ rp.add_argument("table", nargs="?")
32
+ rp.add_argument("--slots", type=int, default=4)
33
+ sp = sub.add_parser("status", help="print the state of the enabled tables")
34
+ sp.add_argument("source", nargs="?")
35
+ sp.add_argument("table", nargs="?")
36
+ sub.add_parser("work", help="load what is due, permanently").add_argument("--slots", type=int, default=4)
37
+ a = p.parse_args()
38
+ if getattr(a, "slots", 1) < 1: # with no slot `run` would wait forever for what is due
39
+ p.error("--slots must be at least 1")
40
+ try:
41
+ command(a)
42
+ except adbc.Error as e: # the target refused or is unreachable: its message, not a traceback
43
+ raise SystemExit(load.scrub(load.target_secrets(), str(e)))
44
+
45
+
46
+ def command(a):
47
+ needs = {"init": ["ROWVIOLI_TARGET_URL"], "status": ["ROWVIOLI_TARGET_URL"],
48
+ "run": ["ROWVIOLI_TARGET_URL", "ROWVIOLI_KEY_FILE"], "work": ["ROWVIOLI_TARGET_URL", "ROWVIOLI_KEY_FILE"]}
49
+ missing = [v for v in needs.get(a.command, []) if not os.environ.get(v)]
50
+ if missing:
51
+ raise SystemExit(f"{' and '.join(missing)} {'is' if len(missing) == 1 else 'are'} not set")
52
+ if a.command == "init":
53
+ target.connect(init=True)
54
+ print("the schema _rowvioli is ready in the target")
55
+ elif a.command == "key":
56
+ seal.new_key()
57
+ print("created worker.key (keep it secret) and worker.pub")
58
+ elif a.command == "apply" and a.check:
59
+ definition.read(a.directory)
60
+ print("the definition is valid")
61
+ elif a.command == "apply":
62
+ apply.apply(a.directory, a.key or str(Path(a.directory).resolve().parent / "worker.pub"), a.init)
63
+ elif a.command == "list":
64
+ print_tables(a.definition, a.source)
65
+ elif a.command == "status":
66
+ print_status(a.source or "", a.table or "")
67
+ elif a.command in ("run", "work"):
68
+ if a.command == "work":
69
+ work.loop(a.slots)
70
+ return
71
+ started = time.monotonic()
72
+ answers, crashed = work.run(a.source or "", a.table or "", a.slots)
73
+ if not answers and not a.source:
74
+ work.say_waiting()
75
+ print(json.dumps({"seconds": round(time.monotonic() - started, 1), "tables": answers}))
76
+ sys.exit(1 if crashed or any(answer["status"] != "ok" for answer in answers.values()) else 0)
77
+
78
+
79
+ STATUS = f"""SELECT source, source_table, target_table, extract(epoch FROM max_age)::bigint,
80
+ extract(epoch FROM age)::bigint,
81
+ CASE WHEN due_at <= now() THEN NULL ELSE ceil(extract(epoch FROM due_at - now()))::bigint END,
82
+ status, reason, rows_written, rows_last_run
83
+ FROM _rowvioli.status
84
+ WHERE {work.SCOPE}
85
+ ORDER BY source, source_table"""
86
+
87
+
88
+ def print_status(source, table):
89
+ """The status view as an aligned table. It formats; status, due time and progress are the view's."""
90
+ conn = target.connect(versioned=False)
91
+ lines = [["source", "table", "target", "max_age", "age", "due", "status", "reason"]]
92
+ for src, name, target_name, max_age, age, wait, status, reason, written, last in target.sql(
93
+ conn, STATUS, [source, table]):
94
+ if status == "loading": # progress in place of a reason, none yet: no reason
95
+ reason = "" if written is None else f"{written:,} of about {last:,} rows" if last else f"{written:,} rows"
96
+ lines.append([src, name, target_name, definition.duration(max_age),
97
+ "" if age is None else definition.duration(age),
98
+ "now" if wait is None else "in " + definition.duration(wait), status, reason or ""])
99
+ widths = [max(len(line[i]) for line in lines) for i in range(7)]
100
+ for line in lines:
101
+ print((" ".join(c.ljust(w) for c, w in zip(line, widths)) + " " + line[7]).rstrip())
102
+
103
+
104
+ def print_tables(directory, name):
105
+ """The tables of a source from its connector's `tables`, as lines for the source's tables block."""
106
+ sources, _, _ = definition.read(directory, os.environ, only=name)
107
+ s = next((s for s in sources if s["name"] == name), None)
108
+ if s is None:
109
+ raise SystemExit(f"{directory} has no source {name}")
110
+ connector = contract.connector(s["type"])
111
+ if not hasattr(connector, "tables"):
112
+ raise SystemExit(f"connector {s['type']} of source {name} cannot list tables")
113
+ try:
114
+ tables = connector.tables(s["connection"])
115
+ except Exception as e: # a driver may quote the connection in its message
116
+ raise SystemExit(load.scrub(load.connection_values(s["connection"]), str(e) or type(e).__name__))
117
+ for table, key in tables.items():
118
+ # a JSON string and a JSON list of strings are TOML as well
119
+ print(f"{json.dumps(table)} = {{ key = {json.dumps(key)} }}" if key else f"{json.dumps(table)} = {{}}")
File without changes
@@ -0,0 +1,54 @@
1
+ """The file connector: expects `path`, a directory; a table is a file name in it. Reads CSV, Excel and Parquet
2
+ through DuckDB by extension. Parquet keeps its own types; CSV and Excel arrive as text, except the columns the option
3
+ `columns` names with a DuckDB type. Every other option is a named parameter of DuckDB's reader, such as `sep` or
4
+ `sheet`; a name that belongs to rowvioli, such as `parallel`, reaches no connector."""
5
+ import re
6
+ from pathlib import Path
7
+
8
+ import duckdb
9
+ import pyarrow as pa
10
+
11
+ from ..target import quote
12
+
13
+ READERS = {".csv": "read_csv", ".xlsx": "read_xlsx", ".parquet": "read_parquet"}
14
+ # a type name with an optional precision, since it is written into the query: `integer`, `decimal(10,2)`
15
+ TYPE = re.compile(r"[A-Za-z][A-Za-z0-9_ ]*(\(\d+(,\s*\d+)?\))?")
16
+
17
+
18
+ def read(connection, table, options):
19
+ file = (Path(connection["path"]) / table).resolve()
20
+ if not file.is_relative_to(Path(connection["path"]).resolve()):
21
+ raise Exception(f"{table}: a table must be a file inside {connection['path']}")
22
+ reader = READERS.get(file.suffix.lower())
23
+ if reader is None:
24
+ raise Exception(f"{table}: the file connector reads {', '.join(READERS)} files")
25
+ if not file.is_file():
26
+ raise Exception(f"{table}: no such file in {connection['path']}")
27
+ options = dict(options)
28
+ columns = options.pop("columns", {})
29
+ if reader != "read_parquet":
30
+ options["all_varchar"] = True
31
+ for name in options:
32
+ if not name.isidentifier():
33
+ raise Exception(f"{table}: {name} is not an option of DuckDB's {reader}")
34
+ for type in columns.values():
35
+ if not TYPE.fullmatch(type):
36
+ raise Exception(f"{table}: {type} is not a column type")
37
+ casts = ", ".join(f'CAST({quote(c)} AS {t}) AS {quote(c)}' for c, t in columns.items())
38
+ db = duckdb.connect()
39
+ result = db.sql(f"SELECT *{f' REPLACE ({casts})' if casts else ''} FROM {reader}(?"
40
+ + "".join(f", {name} = ?" for name in options) + ")", params=[str(file), *options.values()])
41
+ batches = result.to_arrow_reader()
42
+
43
+ def keep_open(): # the reader needs the connection until the last batch
44
+ try:
45
+ yield from batches
46
+ finally:
47
+ db.close()
48
+ return pa.RecordBatchReader.from_batches(batches.schema, keep_open())
49
+
50
+
51
+ def tables(connection):
52
+ """The readable files of the directory, without keys."""
53
+ return {p.name: None for p in sorted(Path(connection["path"]).iterdir()) if p.suffix.lower() in READERS}
54
+
@@ -0,0 +1,43 @@
1
+ """The odbc connector: expects `string`, an ODBC connection string; reads the whole table through arrow-odbc as a
2
+ stream of batches whose schema follows the driver's column descriptions. The driver is installed by the operator."""
3
+ import re
4
+
5
+ import arrow_odbc
6
+
7
+ # not quoted, since quoting differs between sources, and a plain name needs none on any of them
8
+ PLAIN = re.compile(r"\w+(\.\w+)*")
9
+
10
+
11
+ def read(connection, table, options):
12
+ if not PLAIN.fullmatch(table):
13
+ raise Exception(f"{table}: an odbc table name is dotted parts of letters, digits and underscores")
14
+ # shortcut: 1 MiB per text value, a longer one fails the run; when sources hold more, add an option
15
+ return arrow_odbc.read_arrow_batches_from_odbc(
16
+ "SELECT * FROM " + table, connection["string"],
17
+ max_bytes_per_batch=16 * 2**20, max_text_size=2**20).into_pyarrow_record_batch_reader()
18
+
19
+
20
+ def tables(connection):
21
+ """Every base table as `schema.table`, with its primary key in key order or None, from INFORMATION_SCHEMA."""
22
+ conn = arrow_odbc.connect(connection["string"])
23
+ try:
24
+ reader = conn.read_arrow_batches("""
25
+ SELECT t.TABLE_SCHEMA, t.TABLE_NAME, k.COLUMN_NAME
26
+ FROM INFORMATION_SCHEMA.TABLES t
27
+ LEFT JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS c
28
+ ON c.TABLE_SCHEMA = t.TABLE_SCHEMA AND c.TABLE_NAME = t.TABLE_NAME AND c.CONSTRAINT_TYPE = 'PRIMARY KEY'
29
+ LEFT JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE k ON k.TABLE_SCHEMA = c.TABLE_SCHEMA
30
+ AND k.TABLE_NAME = c.TABLE_NAME AND k.CONSTRAINT_NAME = c.CONSTRAINT_NAME
31
+ WHERE t.TABLE_TYPE = 'BASE TABLE'
32
+ AND UPPER(t.TABLE_SCHEMA) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG', 'SYS')
33
+ ORDER BY k.ORDINAL_POSITION""")
34
+ except arrow_odbc.Error as e:
35
+ raise Exception(f"the tables could not be read from INFORMATION_SCHEMA, write the tables block by hand;"
36
+ f" loading is not affected ({e})")
37
+ found = {}
38
+ for batch in reader:
39
+ for schema, table, column in zip(*batch.to_pydict().values()):
40
+ key = found.setdefault(f"{schema}.{table}", [])
41
+ if column is not None:
42
+ key.append(column)
43
+ return {table: key or None for table, key in sorted(found.items())}
@@ -0,0 +1,69 @@
1
+ """The postgres connector: expects `url`, reads the whole table through ADBC as a stream of batches."""
2
+ import adbc_driver_postgresql.dbapi as adbc
3
+ import pyarrow as pa
4
+
5
+ from ..target import quote
6
+
7
+
8
+ def read(connection, table, options):
9
+ conn = adbc.connect(connection["url"])
10
+ cur = conn.cursor()
11
+ name = ".".join(quote(p) for p in table.split("."))
12
+ # the schema without running the query; each column is read as the type that arrives faithfully
13
+ columns = ", ".join(_read_as(f) for f in cur.adbc_execute_schema("SELECT * FROM " + name))
14
+ cur.execute(f"SELECT {columns} FROM {name}")
15
+ reader = cur.fetch_record_batch()
16
+ schema = pa.schema([pa.field(f.name, _decimal(f)) if _is_numeric(f) else f for f in reader.schema])
17
+
18
+ def batches():
19
+ try:
20
+ for b in reader:
21
+ yield pa.record_batch([c.storage.cast(f.type) if _is_numeric(r) else c
22
+ for c, f, r in zip(b.columns, schema, reader.schema)], schema=schema)
23
+ finally:
24
+ cur.close()
25
+ conn.close()
26
+ return pa.RecordBatchReader.from_batches(schema, batches())
27
+
28
+
29
+ def tables(connection):
30
+ """Every table outside the system schemas as `schema.table`, with its primary key in key order or None."""
31
+ with adbc.connect(connection["url"]) as conn, conn.cursor() as cur:
32
+ cur.execute(r"""
33
+ SELECT n.nspname || '.' || c.relname,
34
+ (SELECT array_agg(a.attname::text ORDER BY k.ord)
35
+ FROM pg_index i, unnest(i.indkey) WITH ORDINALITY k(attnum, ord), pg_attribute a
36
+ WHERE i.indrelid = c.oid AND i.indisprimary AND a.attrelid = c.oid AND a.attnum = k.attnum)
37
+ FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
38
+ WHERE c.relkind IN ('r', 'p') AND NOT c.relispartition
39
+ AND n.nspname <> 'information_schema' AND n.nspname NOT LIKE 'pg\_%'
40
+ ORDER BY 1""")
41
+ return dict(cur.fetchall())
42
+
43
+
44
+ def _read_as(field):
45
+ """The column in the select list. `money` arrives as cents in an integer, so it is read as `numeric`. A type
46
+ the driver cannot name arrives as its wire bytes (Arrow's opaque over binary), so it is read as text; an
47
+ array of such a type arrives as one text value, not as an array. Every other type is read as it is (measured in tests/test_run.py)."""
48
+ column, type = quote(field.name), (field.metadata or {}).get(b"POSTGRESQL:type")
49
+ if type in (b"money", b"_money"):
50
+ return f"{column}::numeric{'[]' if type == b'_money' else ''} AS {column}"
51
+ item = field.type.value_type if pa.types.is_list(field.type) else field.type
52
+ if isinstance(item, pa.OpaqueType) and pa.types.is_binary(item.storage_type):
53
+ return f"{column}::text AS {column}"
54
+ return column
55
+
56
+
57
+ def _is_numeric(field):
58
+ return (field.metadata or {}).get(b"POSTGRESQL:type") == b"numeric"
59
+
60
+
61
+ def _decimal(field):
62
+ """The driver reads `numeric` as text and cannot write text into `numeric` (README "Types"), so it becomes a
63
+ decimal with the column's precision and scale, and the target gets `numeric` again."""
64
+ typmod = int((field.metadata or {}).get(b"POSTGRESQL:typmod", b"-1"))
65
+ if typmod < 0:
66
+ # unconstrained: 38 digits on either side of the point; more fails the run (README "Types")
67
+ return pa.decimal256(76, 38)
68
+ p, s = (typmod - 4) >> 16, (typmod - 4) & 0xFFFF
69
+ return pa.decimal128(p, s) if p <= 38 else pa.decimal256(p, s)
@@ -0,0 +1,85 @@
1
+ """The rest connector: expects `url` and, optionally, `token` (sent as a Bearer header). A table is an endpoint: the
2
+ option `path` is appended to `url`, `columns` names every column with a type (text, int8, float8, bool, date, timestamp,
3
+ timestamptz) and `page` names a query parameter that counts from 1 until a page comes back empty; a page equal to the one
4
+ before fails the run. Each response is a JSON array of flat objects and becomes one batch."""
5
+ from datetime import date, datetime, timezone
6
+ from urllib.parse import urlsplit
7
+
8
+ import httpx
9
+ import pyarrow as pa
10
+
11
+ TYPES = {"text": pa.string(), "int8": pa.int64(), "float8": pa.float64(), "bool": pa.bool_(), "date": pa.date32(),
12
+ "timestamp": pa.timestamp("us"), "timestamptz": pa.timestamp("us", tz="UTC")}
13
+
14
+
15
+ def _timestamptz(text):
16
+ t = datetime.fromisoformat(text)
17
+ return t if t.tzinfo else t.replace(tzinfo=timezone.utc) # shortcut: no zone means UTC, when sources differ add an option
18
+
19
+
20
+ def _timestamp(text):
21
+ t = datetime.fromisoformat(text)
22
+ if t.tzinfo:
23
+ raise ValueError(f"{text} carries a zone, use timestamptz")
24
+ return t
25
+
26
+
27
+ PARSE = {"date": date.fromisoformat, "timestamp": _timestamp, "timestamptz": _timestamptz}
28
+
29
+
30
+ def read(connection, table, options):
31
+ for option in ("path", "columns"):
32
+ if option not in options:
33
+ raise Exception(f"{table}: the option {option} is missing")
34
+ columns = options["columns"]
35
+ for name, type in columns.items():
36
+ if type not in TYPES:
37
+ raise Exception(f"{table}: column {name}: {type} is not a type, use {', '.join(TYPES)}")
38
+ schema = pa.schema([(name, TYPES[type]) for name, type in columns.items()])
39
+ if not connection.get("token", "").isascii(): # httpx's own error would show the character
40
+ raise Exception(f"{table}: the token holds a character outside ASCII")
41
+ headers = {"Authorization": f"Bearer {connection['token']}"} if connection.get("token") else {}
42
+ path, paged, page, previous = options["path"], "page" in options, 1, None
43
+ address = connection["url"] + path
44
+ here, there = urlsplit(connection["url"]), urlsplit(address)
45
+ # the token goes to the connection's host only: `.evil.com/x` or `@host/x` would move it
46
+ if not path.startswith("/") or (here.scheme, here.hostname, here.port) != (there.scheme, there.hostname, there.port):
47
+ raise Exception(f"{table}: the option path must start with / and stay on the connection's host")
48
+ while True:
49
+ response = httpx.get(address, params={options["page"]: page} if paged else None,
50
+ headers=headers, timeout=None) # max_seconds bounds the run
51
+ if not response.is_success:
52
+ raise Exception(f"{table}: {path} answered {response.status_code}")
53
+ try:
54
+ rows = response.json()
55
+ except ValueError:
56
+ rows = None
57
+ if not isinstance(rows, list) or not all(isinstance(r, dict) for r in rows):
58
+ raise Exception(f"{table}: {path} must answer a JSON array of flat objects")
59
+ if paged and not rows:
60
+ if page == 1: # an empty table still needs its columns
61
+ yield pa.RecordBatch.from_pylist([], schema=schema)
62
+ return
63
+ if rows == previous: # an API that ignores the page would otherwise be asked until max_seconds
64
+ raise Exception(f"{table}: {path} answered page {page} like page {page - 1}, so it does not page")
65
+ previous = rows
66
+ for name in columns:
67
+ if rows and not any(name in r for r in rows):
68
+ raise Exception(f"{table}: column {name} is absent from every row of the answer")
69
+ arrays = []
70
+ for name, type in columns.items():
71
+ try:
72
+ values = [r.get(name) for r in rows]
73
+ for v in values: # pyarrow would truncate 1.5 and take True for a number
74
+ if (type == "int8" and v is not None and v.__class__ is not int) or (
75
+ type == "float8" and v.__class__ is bool):
76
+ raise ValueError
77
+ parse = PARSE.get(type)
78
+ arrays.append(pa.array([None if v is None else parse(v) for v in values] if parse else values,
79
+ TYPES[type]))
80
+ except Exception: # the value stays out of the reason: it would end up in the status view and the log
81
+ raise Exception(f"{table}: column {name} is not {type}") from None
82
+ yield pa.RecordBatch.from_arrays(arrays, schema=schema)
83
+ if not paged:
84
+ return
85
+ page += 1
rowvioli/contract.py ADDED
@@ -0,0 +1,16 @@
1
+ """The connector contract (decision 27): a module with `read(connection, table, options)` that returns an iterator of
2
+ Arrow batches, and optionally `tables(connection)` for `rowvioli list`. Connectors register as modules in the entry
3
+ point group `rowvioli.connectors` under their `type`; the shipped ones do the same."""
4
+ from importlib.metadata import entry_points
5
+
6
+
7
+ def connector(type, where=""):
8
+ """The connector module registered under `type`. Refuses if none is installed; `where` names the worker."""
9
+ found = entry_points(group="rowvioli.connectors", name=type)
10
+ if not found:
11
+ installed = ", ".join(sorted({e.name for e in entry_points(group="rowvioli.connectors")}))
12
+ raise SystemExit(f'connector "{type}" is not installed{where}; installed: {installed}')
13
+ try:
14
+ return next(iter(found)).load()
15
+ except (ImportError, OSError) as e: # a shipped connector without its extra, or without the library it loads
16
+ raise SystemExit(f'connector "{type}" cannot be loaded{where}: {e}')
rowvioli/definition.py ADDED
@@ -0,0 +1,192 @@
1
+ """Read and check the source files, fill in placeholders."""
2
+ import hashlib
3
+ import json
4
+ import re
5
+ import tomllib
6
+ from pathlib import Path
7
+ from urllib.parse import quote
8
+ from zoneinfo import ZoneInfo
9
+
10
+ UNITS = {"s": "seconds", "m": "minutes", "h": "hours", "d": "days"}
11
+ # a table may override these; its row carries them with the source's filled in
12
+ TABLE_SETTINGS = {"max_age": str, "log": bool, "max_delete_share": (int, float), "max_seconds": int, "at": list,
13
+ "timezone": str, "target_schema": str}
14
+ # settings of a table only, not of the source and not a connector option
15
+ TABLE_ONLY = {"key": list, "target_table": str}
16
+ DEFAULTS = {"log": True, "max_delete_share": 0.5, "max_seconds": 3600}
17
+ SOURCE_SETTINGS = {"type": str, "connection": dict, "tables": dict, "parallel": int,
18
+ **TABLE_SETTINGS}
19
+ RANGES = {"parallel": (lambda v: v >= 1, "at least 1"), "max_delete_share": (lambda v: 0 <= v <= 1, "between 0 and 1"),
20
+ "max_seconds": (lambda v: v > 0, "greater than 0")}
21
+ KINDS = {str: "a string", bool: "true or false", int: "a whole number", (int, float): "a number", dict: "a table",
22
+ list: "a list"}
23
+ # `$NAME`, `$$`, or a `$` that is neither (group 1 is None)
24
+ PLACEHOLDER = re.compile(r"\$(\$|[A-Z][A-Z0-9_]*)?")
25
+ # group 1 is the user info of a URL: `scheme://user:password@`
26
+ USERINFO = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://(?:([^/?#]*)@)?")
27
+
28
+
29
+ def read(directory, env=None, only=None):
30
+ """Reads and checks every *.toml in the directory. With env, the placeholders of each connection are filled in
31
+ from it; without, they are only checked. With `only`, just that source's are filled: `list` needs no other
32
+ source's secrets. Returns (sources, tables, hash of the definition)."""
33
+ if not Path(directory).is_dir():
34
+ raise SystemExit(f"{directory} is not a directory")
35
+ whole, sources, tables, names = {}, [], [], {}
36
+ files = sorted(Path(directory).glob("*.toml"))
37
+ if not files:
38
+ raise SystemExit(f"{directory} holds no source files (*.toml); applying it would disable every table")
39
+ for path in files:
40
+ try:
41
+ data = tomllib.loads(path.read_text())
42
+ except tomllib.TOMLDecodeError as e:
43
+ raise SystemExit(f"{path.name}: {e}")
44
+ for name, s in data.items():
45
+ where = f"{path.name}: {name}"
46
+ if name in whole:
47
+ raise SystemExit(f"{where}: source {name} is defined twice")
48
+ whole[name] = s
49
+ settings(s, SOURCE_SETTINGS, where)
50
+ typos(s, where)
51
+ for k in TABLE_ONLY.keys() & s.keys(): # one table's, so no default for all, no connector option either
52
+ raise SystemExit(f"{where}: {k} belongs to a table, not to the source")
53
+ for required in ("type", "max_age", "connection", "tables"):
54
+ if required not in s:
55
+ raise SystemExit(f"{where}: {required} is missing")
56
+ interval(s["max_age"], where)
57
+ sources.append({"name": name, "type": s["type"], "parallel": s.get("parallel", 2),
58
+ "options": ordered({k: v for k, v in s.items() if k not in SOURCE_SETTINGS}),
59
+ "connection": fill(s["connection"], f"{where}: connection",
60
+ env if only in (None, name) else None)})
61
+ for table, t in s["tables"].items():
62
+ spot = f"{where}: {table}"
63
+ settings(t, {**TABLE_SETTINGS, **TABLE_ONLY}, spot)
64
+ typos(t, spot)
65
+ for k in (SOURCE_SETTINGS.keys() - TABLE_SETTINGS.keys()) & t.keys():
66
+ raise SystemExit(f"{spot}: {k} belongs to the source, not to a table")
67
+ key = t.get("key")
68
+ if key is not None and not (key and all(isinstance(c, str) and c for c in key)):
69
+ raise SystemExit(f"{spot}: key must be a list of column names")
70
+ # a file's extension is not a name part: `customers.csv` becomes `customers`
71
+ target = t["target_table"] if "target_table" in t else (
72
+ Path(table).stem if s["type"] == "file" else table.rsplit(".", 1)[-1]).lower()
73
+ schema = t.get("target_schema", s.get("target_schema", f"raw_{name}"))
74
+ # Postgres cuts names at 63 bytes; the change log and the history view add `__log`, `__history`
75
+ if len(schema.encode()) > 63:
76
+ raise SystemExit(f"{spot}: target_schema {schema} is longer than 63 bytes")
77
+ if len(target.encode()) > 54:
78
+ raise SystemExit(f"{spot}: target table {target} is longer than 54 bytes")
79
+ if not schema or not target or schema == "_rowvioli":
80
+ raise SystemExit(f"{spot}: target_schema and target_table must not be empty, "
81
+ "and the schema must not be _rowvioli")
82
+ for suffix in ("", "__log", "__history"):
83
+ if (schema, target + suffix) in names:
84
+ first_path, first = names[schema, target + suffix]
85
+ both = (f"{first_path}: {first} and {path.name}: {name}: {table}" if first_path != path.name
86
+ else f"{path.name}: {first} and {name}: {table}")
87
+ raise SystemExit(f"{both} both become {schema}.{target + suffix}; "
88
+ "set target_table at one of them")
89
+ names[schema, target + suffix] = path.name, f"{name}: {table}"
90
+ if s["type"] == "rest" and not str(t.get("path", s.get("path", "/"))).startswith("/"):
91
+ raise SystemExit(f"{spot}: path must start with /")
92
+ # no default zone: 14:00 that silently means UTC is found in production (decision 36)
93
+ times = {k: t.get(k, s.get(k)) for k in ("at", "timezone")}
94
+ if times["at"] and not times["timezone"]:
95
+ raise SystemExit(f"{spot}: at needs a timezone, from the table or the source")
96
+ tables.append({"source": name, "source_table": table, "target_schema": schema,
97
+ "target_table": target, "key": key,
98
+ "max_age": interval(t.get("max_age", s["max_age"]), spot),
99
+ **{k: t.get(k, s.get(k, DEFAULTS[k])) for k in DEFAULTS},
100
+ **times,
101
+ "options": ordered({k: v for k, v in t.items()
102
+ if k not in TABLE_SETTINGS and k not in TABLE_ONLY})})
103
+ # of the text as written: placeholders stay unfilled, so the hash says nothing about a secret
104
+ digest = hashlib.sha256(json.dumps(whole, sort_keys=True, default=str).encode()).hexdigest()
105
+ return sources, tables, digest
106
+
107
+
108
+ def ordered(options):
109
+ """jsonb forgets the order of a table's keys, so `columns` is stored as pairs (the REST mirror follows its order)."""
110
+ return {k: list(v.items()) if k == "columns" and isinstance(v, dict) else v for k, v in options.items()}
111
+
112
+
113
+ def settings(block, known, where):
114
+ """Checks that a source or table is a table and that its known settings have the right kind of value."""
115
+ if not isinstance(block, dict):
116
+ raise SystemExit(f"{where}: must be a table")
117
+ for k, kind in known.items():
118
+ # bool is a subclass of int, so true must not pass as a whole number and 1 not as true
119
+ if k in block and (not isinstance(block[k], kind) or isinstance(block[k], bool) != (kind is bool)):
120
+ raise SystemExit(f"{where}: {k} must be {KINDS[kind]}")
121
+ if k in block and k in RANGES and not RANGES[k][0](block[k]):
122
+ raise SystemExit(f"{where}: {k} must be {RANGES[k][1]}")
123
+ for time in block.get("at", []):
124
+ if not (isinstance(time, str) and re.fullmatch(r"([01]\d|2[0-3]):[0-5]\d", time)):
125
+ raise SystemExit(f"{where}: at {time!r} is not a time HH:MM from 00:00 to 23:59")
126
+ if "timezone" in block:
127
+ try:
128
+ ZoneInfo(block["timezone"])
129
+ except (KeyError, ValueError, OSError): # not found, not a zone's name, a directory
130
+ raise SystemExit(f"{where}: timezone {block['timezone']!r} is not a time zone")
131
+
132
+
133
+ def edits(a, b):
134
+ """Edit distance of two words (insert, delete, replace)."""
135
+ row = list(range(len(b) + 1))
136
+ for i, x in enumerate(a, 1):
137
+ prev, row[0] = row[0], i
138
+ for j, y in enumerate(b, 1):
139
+ prev, row[j] = row[j], min(row[j] + 1, row[j - 1] + 1, prev + (x != y))
140
+ return row[-1]
141
+
142
+
143
+ def typos(block, where):
144
+ """A key that is no setting but a typo of one would silently go to the connector as an option. Short words
145
+ are left alone (`sep` is one edit from nothing but two from `key`): shortcut: one edit under 6 letters."""
146
+ known = SOURCE_SETTINGS.keys() | TABLE_ONLY.keys()
147
+ for k in block.keys() - known:
148
+ for setting in known:
149
+ if len(k) >= 4 and edits(k, setting) <= (2 if len(setting) >= 6 else 1):
150
+ raise SystemExit(f"{where}: {k} is not a setting; did you mean {setting}? "
151
+ "(other keys go to the connector as options)")
152
+
153
+
154
+ def interval(value, where):
155
+ m = re.fullmatch(r"([1-9]\d*)([smhd])", value)
156
+ if not m:
157
+ raise SystemExit(f"{where}: max_age {value!r} is not a whole number with s, m, h or d")
158
+ return f"{m[1]} {UNITS[m[2]]}"
159
+
160
+
161
+ def duration(seconds):
162
+ """Seconds as a person reads them, the largest two units: `5m`, `1h 3m`. For the status, not for the definition."""
163
+ parts = []
164
+ for unit, size in (("d", 86400), ("h", 3600), ("m", 60), ("s", 1)):
165
+ if seconds >= size and len(parts) < 2:
166
+ parts.append(f"{seconds // size}{unit}")
167
+ seconds %= size
168
+ return " ".join(parts) or "0s"
169
+
170
+
171
+ def fill(template, where, env):
172
+ """`$NAME` from env, `$$` a literal `$`, in every string of the connection, also nested ones. A value landing
173
+ in the user info of a URL is percent-encoded (decision 32). Without env, the placeholders are only checked. No
174
+ value ever appears in an error."""
175
+ if isinstance(template, dict):
176
+ return {k: fill(v, f"{where}.{k}", env) for k, v in template.items()}
177
+ if isinstance(template, list):
178
+ return [fill(v, where, env) for v in template]
179
+ if not isinstance(template, str):
180
+ return template
181
+ url = USERINFO.match(template)
182
+ userinfo_end = url.end(1) if url and url[1] is not None else -1
183
+
184
+ def sub(m):
185
+ if m[1] is None:
186
+ raise SystemExit(f"{where}: a $ must start a placeholder like $NAME or be written $$")
187
+ if m[1] == "$" or env is None:
188
+ return "$"
189
+ if m[1] not in env:
190
+ raise SystemExit(f"{where}: variable {m[1]} is not set")
191
+ return quote(env[m[1]], safe="") if m.start() < userinfo_end else env[m[1]]
192
+ return PLACEHOLDER.sub(sub, template)