rowvioli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rowvioli/__init__.py +0 -0
- rowvioli/__main__.py +3 -0
- rowvioli/apply.py +45 -0
- rowvioli/cli.py +119 -0
- rowvioli/connectors/__init__.py +0 -0
- rowvioli/connectors/file.py +54 -0
- rowvioli/connectors/odbc.py +43 -0
- rowvioli/connectors/postgres.py +69 -0
- rowvioli/connectors/rest.py +85 -0
- rowvioli/contract.py +16 -0
- rowvioli/definition.py +192 -0
- rowvioli/load.py +387 -0
- rowvioli/seal.py +49 -0
- rowvioli/sql/apply.sql +36 -0
- rowvioli/sql/claimable.sql +21 -0
- rowvioli/sql/schema.sql +95 -0
- rowvioli/target.py +73 -0
- rowvioli/work.py +180 -0
- rowvioli-0.1.0.dist-info/METADATA +1047 -0
- rowvioli-0.1.0.dist-info/RECORD +23 -0
- rowvioli-0.1.0.dist-info/WHEEL +4 -0
- rowvioli-0.1.0.dist-info/entry_points.txt +9 -0
- rowvioli-0.1.0.dist-info/licenses/LICENSE +21 -0
rowvioli/__init__.py
ADDED
|
File without changes
|
rowvioli/__main__.py
ADDED
rowvioli/apply.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Write the definition into the target. Needs no source and no driver for one."""
|
|
2
|
+
import base64
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
from importlib.resources import files
|
|
6
|
+
|
|
7
|
+
import adbc_driver_postgresql.dbapi as adbc
|
|
8
|
+
|
|
9
|
+
from . import definition, seal, target
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def apply(directory, key_file, init=False):
|
|
13
|
+
sources, tables, digest = definition.read(directory, os.environ)
|
|
14
|
+
try:
|
|
15
|
+
public = seal.public_key(key_file)
|
|
16
|
+
except OSError as e:
|
|
17
|
+
raise SystemExit(f"public key {key_file}: {e.strerror}")
|
|
18
|
+
except Exception:
|
|
19
|
+
raise SystemExit(f"{key_file} holds no public key")
|
|
20
|
+
for s in sources:
|
|
21
|
+
s["connection"] = base64.b64encode(seal.seal(public, s["name"], s["connection"])).decode()
|
|
22
|
+
s["key_id"] = seal.key_id(public)
|
|
23
|
+
try:
|
|
24
|
+
conn = target.connect()
|
|
25
|
+
except SystemExit as e:
|
|
26
|
+
if not init or e.code != target.NO_SCHEMA:
|
|
27
|
+
raise
|
|
28
|
+
conn = target.connect(init=True)
|
|
29
|
+
print("the schema _rowvioli is ready in the target")
|
|
30
|
+
# a changed target name moves nothing: say so, so that no stale mirror is left behind unknowingly
|
|
31
|
+
old = {(src, tbl): f"{schema}.{name}" for src, tbl, schema, name in target.sql(
|
|
32
|
+
conn, "SELECT source, source_table, target_schema, target_table FROM _rowvioli.tables")}
|
|
33
|
+
changed = []
|
|
34
|
+
for t in tables:
|
|
35
|
+
was, now = old.get((t["source"], t["source_table"])), f"{t['target_schema']}.{t['target_table']}"
|
|
36
|
+
if was not in (None, now):
|
|
37
|
+
changed.append(f"{t['source']}: {t['source_table']}: target name changes from {was} to {now}; the old "
|
|
38
|
+
f"tables ({was}, its __log and __history) stay where they are")
|
|
39
|
+
payload = json.dumps({"sources": sources, "tables": tables, "definition": digest}, default=str)
|
|
40
|
+
try:
|
|
41
|
+
target.transaction(conn, files(__package__).joinpath("sql/apply.sql").read_text(), [payload])
|
|
42
|
+
except adbc.Error as e: # Postgres names what it refused, for example a time zone it does not know
|
|
43
|
+
raise SystemExit(f"the target refused the definition, nothing was changed: {e}")
|
|
44
|
+
print(*changed, sep="\n", end="\n" if changed else "")
|
|
45
|
+
print(f"applied {len(sources)} sources with {len(tables)} tables")
|
rowvioli/cli.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""The commands: arguments, the answer on stdout, the exit code."""
|
|
2
|
+
import argparse
|
|
3
|
+
import importlib.metadata
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
import time
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import adbc_driver_postgresql.dbapi as adbc
|
|
11
|
+
|
|
12
|
+
from . import apply, contract, definition, load, seal, target, work
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main():
|
|
16
|
+
p = argparse.ArgumentParser(prog="rowvioli")
|
|
17
|
+
p.add_argument("--version", action="version", version=importlib.metadata.version("rowvioli"))
|
|
18
|
+
sub = p.add_subparsers(dest="command", required=True)
|
|
19
|
+
sub.add_parser("init", help="create the schema _rowvioli in the target")
|
|
20
|
+
sub.add_parser("key", help="create a key pair").add_argument("action", choices=["new"])
|
|
21
|
+
lp = sub.add_parser("list", help="print the tables of a source")
|
|
22
|
+
lp.add_argument("source")
|
|
23
|
+
lp.add_argument("--definition", default="sources", help="directory of the definition")
|
|
24
|
+
ap = sub.add_parser("apply", help="write the definition into the target")
|
|
25
|
+
ap.add_argument("directory")
|
|
26
|
+
ap.add_argument("--check", action="store_true", help="only read and check the definition")
|
|
27
|
+
ap.add_argument("--init", action="store_true", help="create the schema _rowvioli first if it is missing")
|
|
28
|
+
ap.add_argument("--key", help="public key file (default: worker.pub next to the directory)")
|
|
29
|
+
rp = sub.add_parser("run", help="load what is due, once")
|
|
30
|
+
rp.add_argument("source", nargs="?")
|
|
31
|
+
rp.add_argument("table", nargs="?")
|
|
32
|
+
rp.add_argument("--slots", type=int, default=4)
|
|
33
|
+
sp = sub.add_parser("status", help="print the state of the enabled tables")
|
|
34
|
+
sp.add_argument("source", nargs="?")
|
|
35
|
+
sp.add_argument("table", nargs="?")
|
|
36
|
+
sub.add_parser("work", help="load what is due, permanently").add_argument("--slots", type=int, default=4)
|
|
37
|
+
a = p.parse_args()
|
|
38
|
+
if getattr(a, "slots", 1) < 1: # with no slot `run` would wait forever for what is due
|
|
39
|
+
p.error("--slots must be at least 1")
|
|
40
|
+
try:
|
|
41
|
+
command(a)
|
|
42
|
+
except adbc.Error as e: # the target refused or is unreachable: its message, not a traceback
|
|
43
|
+
raise SystemExit(load.scrub(load.target_secrets(), str(e)))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def command(a):
|
|
47
|
+
needs = {"init": ["ROWVIOLI_TARGET_URL"], "status": ["ROWVIOLI_TARGET_URL"],
|
|
48
|
+
"run": ["ROWVIOLI_TARGET_URL", "ROWVIOLI_KEY_FILE"], "work": ["ROWVIOLI_TARGET_URL", "ROWVIOLI_KEY_FILE"]}
|
|
49
|
+
missing = [v for v in needs.get(a.command, []) if not os.environ.get(v)]
|
|
50
|
+
if missing:
|
|
51
|
+
raise SystemExit(f"{' and '.join(missing)} {'is' if len(missing) == 1 else 'are'} not set")
|
|
52
|
+
if a.command == "init":
|
|
53
|
+
target.connect(init=True)
|
|
54
|
+
print("the schema _rowvioli is ready in the target")
|
|
55
|
+
elif a.command == "key":
|
|
56
|
+
seal.new_key()
|
|
57
|
+
print("created worker.key (keep it secret) and worker.pub")
|
|
58
|
+
elif a.command == "apply" and a.check:
|
|
59
|
+
definition.read(a.directory)
|
|
60
|
+
print("the definition is valid")
|
|
61
|
+
elif a.command == "apply":
|
|
62
|
+
apply.apply(a.directory, a.key or str(Path(a.directory).resolve().parent / "worker.pub"), a.init)
|
|
63
|
+
elif a.command == "list":
|
|
64
|
+
print_tables(a.definition, a.source)
|
|
65
|
+
elif a.command == "status":
|
|
66
|
+
print_status(a.source or "", a.table or "")
|
|
67
|
+
elif a.command in ("run", "work"):
|
|
68
|
+
if a.command == "work":
|
|
69
|
+
work.loop(a.slots)
|
|
70
|
+
return
|
|
71
|
+
started = time.monotonic()
|
|
72
|
+
answers, crashed = work.run(a.source or "", a.table or "", a.slots)
|
|
73
|
+
if not answers and not a.source:
|
|
74
|
+
work.say_waiting()
|
|
75
|
+
print(json.dumps({"seconds": round(time.monotonic() - started, 1), "tables": answers}))
|
|
76
|
+
sys.exit(1 if crashed or any(answer["status"] != "ok" for answer in answers.values()) else 0)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
STATUS = f"""SELECT source, source_table, target_table, extract(epoch FROM max_age)::bigint,
|
|
80
|
+
extract(epoch FROM age)::bigint,
|
|
81
|
+
CASE WHEN due_at <= now() THEN NULL ELSE ceil(extract(epoch FROM due_at - now()))::bigint END,
|
|
82
|
+
status, reason, rows_written, rows_last_run
|
|
83
|
+
FROM _rowvioli.status
|
|
84
|
+
WHERE {work.SCOPE}
|
|
85
|
+
ORDER BY source, source_table"""
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def print_status(source, table):
|
|
89
|
+
"""The status view as an aligned table. It formats; status, due time and progress are the view's."""
|
|
90
|
+
conn = target.connect(versioned=False)
|
|
91
|
+
lines = [["source", "table", "target", "max_age", "age", "due", "status", "reason"]]
|
|
92
|
+
for src, name, target_name, max_age, age, wait, status, reason, written, last in target.sql(
|
|
93
|
+
conn, STATUS, [source, table]):
|
|
94
|
+
if status == "loading": # progress in place of a reason, none yet: no reason
|
|
95
|
+
reason = "" if written is None else f"{written:,} of about {last:,} rows" if last else f"{written:,} rows"
|
|
96
|
+
lines.append([src, name, target_name, definition.duration(max_age),
|
|
97
|
+
"" if age is None else definition.duration(age),
|
|
98
|
+
"now" if wait is None else "in " + definition.duration(wait), status, reason or ""])
|
|
99
|
+
widths = [max(len(line[i]) for line in lines) for i in range(7)]
|
|
100
|
+
for line in lines:
|
|
101
|
+
print((" ".join(c.ljust(w) for c, w in zip(line, widths)) + " " + line[7]).rstrip())
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def print_tables(directory, name):
|
|
105
|
+
"""The tables of a source from its connector's `tables`, as lines for the source's tables block."""
|
|
106
|
+
sources, _, _ = definition.read(directory, os.environ, only=name)
|
|
107
|
+
s = next((s for s in sources if s["name"] == name), None)
|
|
108
|
+
if s is None:
|
|
109
|
+
raise SystemExit(f"{directory} has no source {name}")
|
|
110
|
+
connector = contract.connector(s["type"])
|
|
111
|
+
if not hasattr(connector, "tables"):
|
|
112
|
+
raise SystemExit(f"connector {s['type']} of source {name} cannot list tables")
|
|
113
|
+
try:
|
|
114
|
+
tables = connector.tables(s["connection"])
|
|
115
|
+
except Exception as e: # a driver may quote the connection in its message
|
|
116
|
+
raise SystemExit(load.scrub(load.connection_values(s["connection"]), str(e) or type(e).__name__))
|
|
117
|
+
for table, key in tables.items():
|
|
118
|
+
# a JSON string and a JSON list of strings are TOML as well
|
|
119
|
+
print(f"{json.dumps(table)} = {{ key = {json.dumps(key)} }}" if key else f"{json.dumps(table)} = {{}}")
|
|
File without changes
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""The file connector: expects `path`, a directory; a table is a file name in it. Reads CSV, Excel and Parquet
|
|
2
|
+
through DuckDB by extension. Parquet keeps its own types; CSV and Excel arrive as text, except the columns the option
|
|
3
|
+
`columns` names with a DuckDB type. Every other option is a named parameter of DuckDB's reader, such as `sep` or
|
|
4
|
+
`sheet`; a name that belongs to rowvioli, such as `parallel`, reaches no connector."""
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import duckdb
|
|
9
|
+
import pyarrow as pa
|
|
10
|
+
|
|
11
|
+
from ..target import quote
|
|
12
|
+
|
|
13
|
+
READERS = {".csv": "read_csv", ".xlsx": "read_xlsx", ".parquet": "read_parquet"}
|
|
14
|
+
# a type name with an optional precision, since it is written into the query: `integer`, `decimal(10,2)`
|
|
15
|
+
TYPE = re.compile(r"[A-Za-z][A-Za-z0-9_ ]*(\(\d+(,\s*\d+)?\))?")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def read(connection, table, options):
|
|
19
|
+
file = (Path(connection["path"]) / table).resolve()
|
|
20
|
+
if not file.is_relative_to(Path(connection["path"]).resolve()):
|
|
21
|
+
raise Exception(f"{table}: a table must be a file inside {connection['path']}")
|
|
22
|
+
reader = READERS.get(file.suffix.lower())
|
|
23
|
+
if reader is None:
|
|
24
|
+
raise Exception(f"{table}: the file connector reads {', '.join(READERS)} files")
|
|
25
|
+
if not file.is_file():
|
|
26
|
+
raise Exception(f"{table}: no such file in {connection['path']}")
|
|
27
|
+
options = dict(options)
|
|
28
|
+
columns = options.pop("columns", {})
|
|
29
|
+
if reader != "read_parquet":
|
|
30
|
+
options["all_varchar"] = True
|
|
31
|
+
for name in options:
|
|
32
|
+
if not name.isidentifier():
|
|
33
|
+
raise Exception(f"{table}: {name} is not an option of DuckDB's {reader}")
|
|
34
|
+
for type in columns.values():
|
|
35
|
+
if not TYPE.fullmatch(type):
|
|
36
|
+
raise Exception(f"{table}: {type} is not a column type")
|
|
37
|
+
casts = ", ".join(f'CAST({quote(c)} AS {t}) AS {quote(c)}' for c, t in columns.items())
|
|
38
|
+
db = duckdb.connect()
|
|
39
|
+
result = db.sql(f"SELECT *{f' REPLACE ({casts})' if casts else ''} FROM {reader}(?"
|
|
40
|
+
+ "".join(f", {name} = ?" for name in options) + ")", params=[str(file), *options.values()])
|
|
41
|
+
batches = result.to_arrow_reader()
|
|
42
|
+
|
|
43
|
+
def keep_open(): # the reader needs the connection until the last batch
|
|
44
|
+
try:
|
|
45
|
+
yield from batches
|
|
46
|
+
finally:
|
|
47
|
+
db.close()
|
|
48
|
+
return pa.RecordBatchReader.from_batches(batches.schema, keep_open())
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def tables(connection):
|
|
52
|
+
"""The readable files of the directory, without keys."""
|
|
53
|
+
return {p.name: None for p in sorted(Path(connection["path"]).iterdir()) if p.suffix.lower() in READERS}
|
|
54
|
+
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""The odbc connector: expects `string`, an ODBC connection string; reads the whole table through arrow-odbc as a
|
|
2
|
+
stream of batches whose schema follows the driver's column descriptions. The driver is installed by the operator."""
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
import arrow_odbc
|
|
6
|
+
|
|
7
|
+
# not quoted, since quoting differs between sources, and a plain name needs none on any of them
|
|
8
|
+
PLAIN = re.compile(r"\w+(\.\w+)*")
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def read(connection, table, options):
|
|
12
|
+
if not PLAIN.fullmatch(table):
|
|
13
|
+
raise Exception(f"{table}: an odbc table name is dotted parts of letters, digits and underscores")
|
|
14
|
+
# shortcut: 1 MiB per text value, a longer one fails the run; when sources hold more, add an option
|
|
15
|
+
return arrow_odbc.read_arrow_batches_from_odbc(
|
|
16
|
+
"SELECT * FROM " + table, connection["string"],
|
|
17
|
+
max_bytes_per_batch=16 * 2**20, max_text_size=2**20).into_pyarrow_record_batch_reader()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def tables(connection):
|
|
21
|
+
"""Every base table as `schema.table`, with its primary key in key order or None, from INFORMATION_SCHEMA."""
|
|
22
|
+
conn = arrow_odbc.connect(connection["string"])
|
|
23
|
+
try:
|
|
24
|
+
reader = conn.read_arrow_batches("""
|
|
25
|
+
SELECT t.TABLE_SCHEMA, t.TABLE_NAME, k.COLUMN_NAME
|
|
26
|
+
FROM INFORMATION_SCHEMA.TABLES t
|
|
27
|
+
LEFT JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS c
|
|
28
|
+
ON c.TABLE_SCHEMA = t.TABLE_SCHEMA AND c.TABLE_NAME = t.TABLE_NAME AND c.CONSTRAINT_TYPE = 'PRIMARY KEY'
|
|
29
|
+
LEFT JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE k ON k.TABLE_SCHEMA = c.TABLE_SCHEMA
|
|
30
|
+
AND k.TABLE_NAME = c.TABLE_NAME AND k.CONSTRAINT_NAME = c.CONSTRAINT_NAME
|
|
31
|
+
WHERE t.TABLE_TYPE = 'BASE TABLE'
|
|
32
|
+
AND UPPER(t.TABLE_SCHEMA) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG', 'SYS')
|
|
33
|
+
ORDER BY k.ORDINAL_POSITION""")
|
|
34
|
+
except arrow_odbc.Error as e:
|
|
35
|
+
raise Exception(f"the tables could not be read from INFORMATION_SCHEMA, write the tables block by hand;"
|
|
36
|
+
f" loading is not affected ({e})")
|
|
37
|
+
found = {}
|
|
38
|
+
for batch in reader:
|
|
39
|
+
for schema, table, column in zip(*batch.to_pydict().values()):
|
|
40
|
+
key = found.setdefault(f"{schema}.{table}", [])
|
|
41
|
+
if column is not None:
|
|
42
|
+
key.append(column)
|
|
43
|
+
return {table: key or None for table, key in sorted(found.items())}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""The postgres connector: expects `url`, reads the whole table through ADBC as a stream of batches."""
|
|
2
|
+
import adbc_driver_postgresql.dbapi as adbc
|
|
3
|
+
import pyarrow as pa
|
|
4
|
+
|
|
5
|
+
from ..target import quote
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def read(connection, table, options):
|
|
9
|
+
conn = adbc.connect(connection["url"])
|
|
10
|
+
cur = conn.cursor()
|
|
11
|
+
name = ".".join(quote(p) for p in table.split("."))
|
|
12
|
+
# the schema without running the query; each column is read as the type that arrives faithfully
|
|
13
|
+
columns = ", ".join(_read_as(f) for f in cur.adbc_execute_schema("SELECT * FROM " + name))
|
|
14
|
+
cur.execute(f"SELECT {columns} FROM {name}")
|
|
15
|
+
reader = cur.fetch_record_batch()
|
|
16
|
+
schema = pa.schema([pa.field(f.name, _decimal(f)) if _is_numeric(f) else f for f in reader.schema])
|
|
17
|
+
|
|
18
|
+
def batches():
|
|
19
|
+
try:
|
|
20
|
+
for b in reader:
|
|
21
|
+
yield pa.record_batch([c.storage.cast(f.type) if _is_numeric(r) else c
|
|
22
|
+
for c, f, r in zip(b.columns, schema, reader.schema)], schema=schema)
|
|
23
|
+
finally:
|
|
24
|
+
cur.close()
|
|
25
|
+
conn.close()
|
|
26
|
+
return pa.RecordBatchReader.from_batches(schema, batches())
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def tables(connection):
|
|
30
|
+
"""Every table outside the system schemas as `schema.table`, with its primary key in key order or None."""
|
|
31
|
+
with adbc.connect(connection["url"]) as conn, conn.cursor() as cur:
|
|
32
|
+
cur.execute(r"""
|
|
33
|
+
SELECT n.nspname || '.' || c.relname,
|
|
34
|
+
(SELECT array_agg(a.attname::text ORDER BY k.ord)
|
|
35
|
+
FROM pg_index i, unnest(i.indkey) WITH ORDINALITY k(attnum, ord), pg_attribute a
|
|
36
|
+
WHERE i.indrelid = c.oid AND i.indisprimary AND a.attrelid = c.oid AND a.attnum = k.attnum)
|
|
37
|
+
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
38
|
+
WHERE c.relkind IN ('r', 'p') AND NOT c.relispartition
|
|
39
|
+
AND n.nspname <> 'information_schema' AND n.nspname NOT LIKE 'pg\_%'
|
|
40
|
+
ORDER BY 1""")
|
|
41
|
+
return dict(cur.fetchall())
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _read_as(field):
|
|
45
|
+
"""The column in the select list. `money` arrives as cents in an integer, so it is read as `numeric`. A type
|
|
46
|
+
the driver cannot name arrives as its wire bytes (Arrow's opaque over binary), so it is read as text; an
|
|
47
|
+
array of such a type arrives as one text value, not as an array. Every other type is read as it is (measured in tests/test_run.py)."""
|
|
48
|
+
column, type = quote(field.name), (field.metadata or {}).get(b"POSTGRESQL:type")
|
|
49
|
+
if type in (b"money", b"_money"):
|
|
50
|
+
return f"{column}::numeric{'[]' if type == b'_money' else ''} AS {column}"
|
|
51
|
+
item = field.type.value_type if pa.types.is_list(field.type) else field.type
|
|
52
|
+
if isinstance(item, pa.OpaqueType) and pa.types.is_binary(item.storage_type):
|
|
53
|
+
return f"{column}::text AS {column}"
|
|
54
|
+
return column
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _is_numeric(field):
|
|
58
|
+
return (field.metadata or {}).get(b"POSTGRESQL:type") == b"numeric"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _decimal(field):
|
|
62
|
+
"""The driver reads `numeric` as text and cannot write text into `numeric` (README "Types"), so it becomes a
|
|
63
|
+
decimal with the column's precision and scale, and the target gets `numeric` again."""
|
|
64
|
+
typmod = int((field.metadata or {}).get(b"POSTGRESQL:typmod", b"-1"))
|
|
65
|
+
if typmod < 0:
|
|
66
|
+
# unconstrained: 38 digits on either side of the point; more fails the run (README "Types")
|
|
67
|
+
return pa.decimal256(76, 38)
|
|
68
|
+
p, s = (typmod - 4) >> 16, (typmod - 4) & 0xFFFF
|
|
69
|
+
return pa.decimal128(p, s) if p <= 38 else pa.decimal256(p, s)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""The rest connector: expects `url` and, optionally, `token` (sent as a Bearer header). A table is an endpoint: the
|
|
2
|
+
option `path` is appended to `url`, `columns` names every column with a type (text, int8, float8, bool, date, timestamp,
|
|
3
|
+
timestamptz) and `page` names a query parameter that counts from 1 until a page comes back empty; a page equal to the one
|
|
4
|
+
before fails the run. Each response is a JSON array of flat objects and becomes one batch."""
|
|
5
|
+
from datetime import date, datetime, timezone
|
|
6
|
+
from urllib.parse import urlsplit
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
import pyarrow as pa
|
|
10
|
+
|
|
11
|
+
TYPES = {"text": pa.string(), "int8": pa.int64(), "float8": pa.float64(), "bool": pa.bool_(), "date": pa.date32(),
|
|
12
|
+
"timestamp": pa.timestamp("us"), "timestamptz": pa.timestamp("us", tz="UTC")}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _timestamptz(text):
|
|
16
|
+
t = datetime.fromisoformat(text)
|
|
17
|
+
return t if t.tzinfo else t.replace(tzinfo=timezone.utc) # shortcut: no zone means UTC, when sources differ add an option
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _timestamp(text):
|
|
21
|
+
t = datetime.fromisoformat(text)
|
|
22
|
+
if t.tzinfo:
|
|
23
|
+
raise ValueError(f"{text} carries a zone, use timestamptz")
|
|
24
|
+
return t
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
PARSE = {"date": date.fromisoformat, "timestamp": _timestamp, "timestamptz": _timestamptz}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def read(connection, table, options):
|
|
31
|
+
for option in ("path", "columns"):
|
|
32
|
+
if option not in options:
|
|
33
|
+
raise Exception(f"{table}: the option {option} is missing")
|
|
34
|
+
columns = options["columns"]
|
|
35
|
+
for name, type in columns.items():
|
|
36
|
+
if type not in TYPES:
|
|
37
|
+
raise Exception(f"{table}: column {name}: {type} is not a type, use {', '.join(TYPES)}")
|
|
38
|
+
schema = pa.schema([(name, TYPES[type]) for name, type in columns.items()])
|
|
39
|
+
if not connection.get("token", "").isascii(): # httpx's own error would show the character
|
|
40
|
+
raise Exception(f"{table}: the token holds a character outside ASCII")
|
|
41
|
+
headers = {"Authorization": f"Bearer {connection['token']}"} if connection.get("token") else {}
|
|
42
|
+
path, paged, page, previous = options["path"], "page" in options, 1, None
|
|
43
|
+
address = connection["url"] + path
|
|
44
|
+
here, there = urlsplit(connection["url"]), urlsplit(address)
|
|
45
|
+
# the token goes to the connection's host only: `.evil.com/x` or `@host/x` would move it
|
|
46
|
+
if not path.startswith("/") or (here.scheme, here.hostname, here.port) != (there.scheme, there.hostname, there.port):
|
|
47
|
+
raise Exception(f"{table}: the option path must start with / and stay on the connection's host")
|
|
48
|
+
while True:
|
|
49
|
+
response = httpx.get(address, params={options["page"]: page} if paged else None,
|
|
50
|
+
headers=headers, timeout=None) # max_seconds bounds the run
|
|
51
|
+
if not response.is_success:
|
|
52
|
+
raise Exception(f"{table}: {path} answered {response.status_code}")
|
|
53
|
+
try:
|
|
54
|
+
rows = response.json()
|
|
55
|
+
except ValueError:
|
|
56
|
+
rows = None
|
|
57
|
+
if not isinstance(rows, list) or not all(isinstance(r, dict) for r in rows):
|
|
58
|
+
raise Exception(f"{table}: {path} must answer a JSON array of flat objects")
|
|
59
|
+
if paged and not rows:
|
|
60
|
+
if page == 1: # an empty table still needs its columns
|
|
61
|
+
yield pa.RecordBatch.from_pylist([], schema=schema)
|
|
62
|
+
return
|
|
63
|
+
if rows == previous: # an API that ignores the page would otherwise be asked until max_seconds
|
|
64
|
+
raise Exception(f"{table}: {path} answered page {page} like page {page - 1}, so it does not page")
|
|
65
|
+
previous = rows
|
|
66
|
+
for name in columns:
|
|
67
|
+
if rows and not any(name in r for r in rows):
|
|
68
|
+
raise Exception(f"{table}: column {name} is absent from every row of the answer")
|
|
69
|
+
arrays = []
|
|
70
|
+
for name, type in columns.items():
|
|
71
|
+
try:
|
|
72
|
+
values = [r.get(name) for r in rows]
|
|
73
|
+
for v in values: # pyarrow would truncate 1.5 and take True for a number
|
|
74
|
+
if (type == "int8" and v is not None and v.__class__ is not int) or (
|
|
75
|
+
type == "float8" and v.__class__ is bool):
|
|
76
|
+
raise ValueError
|
|
77
|
+
parse = PARSE.get(type)
|
|
78
|
+
arrays.append(pa.array([None if v is None else parse(v) for v in values] if parse else values,
|
|
79
|
+
TYPES[type]))
|
|
80
|
+
except Exception: # the value stays out of the reason: it would end up in the status view and the log
|
|
81
|
+
raise Exception(f"{table}: column {name} is not {type}") from None
|
|
82
|
+
yield pa.RecordBatch.from_arrays(arrays, schema=schema)
|
|
83
|
+
if not paged:
|
|
84
|
+
return
|
|
85
|
+
page += 1
|
rowvioli/contract.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""The connector contract (decision 27): a module with `read(connection, table, options)` that returns an iterator of
|
|
2
|
+
Arrow batches, and optionally `tables(connection)` for `rowvioli list`. Connectors register as modules in the entry
|
|
3
|
+
point group `rowvioli.connectors` under their `type`; the shipped ones do the same."""
|
|
4
|
+
from importlib.metadata import entry_points
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def connector(type, where=""):
|
|
8
|
+
"""The connector module registered under `type`. Refuses if none is installed; `where` names the worker."""
|
|
9
|
+
found = entry_points(group="rowvioli.connectors", name=type)
|
|
10
|
+
if not found:
|
|
11
|
+
installed = ", ".join(sorted({e.name for e in entry_points(group="rowvioli.connectors")}))
|
|
12
|
+
raise SystemExit(f'connector "{type}" is not installed{where}; installed: {installed}')
|
|
13
|
+
try:
|
|
14
|
+
return next(iter(found)).load()
|
|
15
|
+
except (ImportError, OSError) as e: # a shipped connector without its extra, or without the library it loads
|
|
16
|
+
raise SystemExit(f'connector "{type}" cannot be loaded{where}: {e}')
|
rowvioli/definition.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Read and check the source files, fill in placeholders."""
|
|
2
|
+
import hashlib
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
import tomllib
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from urllib.parse import quote
|
|
8
|
+
from zoneinfo import ZoneInfo
|
|
9
|
+
|
|
10
|
+
UNITS = {"s": "seconds", "m": "minutes", "h": "hours", "d": "days"}
|
|
11
|
+
# a table may override these; its row carries them with the source's filled in
|
|
12
|
+
TABLE_SETTINGS = {"max_age": str, "log": bool, "max_delete_share": (int, float), "max_seconds": int, "at": list,
|
|
13
|
+
"timezone": str, "target_schema": str}
|
|
14
|
+
# settings of a table only, not of the source and not a connector option
|
|
15
|
+
TABLE_ONLY = {"key": list, "target_table": str}
|
|
16
|
+
DEFAULTS = {"log": True, "max_delete_share": 0.5, "max_seconds": 3600}
|
|
17
|
+
SOURCE_SETTINGS = {"type": str, "connection": dict, "tables": dict, "parallel": int,
|
|
18
|
+
**TABLE_SETTINGS}
|
|
19
|
+
RANGES = {"parallel": (lambda v: v >= 1, "at least 1"), "max_delete_share": (lambda v: 0 <= v <= 1, "between 0 and 1"),
|
|
20
|
+
"max_seconds": (lambda v: v > 0, "greater than 0")}
|
|
21
|
+
KINDS = {str: "a string", bool: "true or false", int: "a whole number", (int, float): "a number", dict: "a table",
|
|
22
|
+
list: "a list"}
|
|
23
|
+
# `$NAME`, `$$`, or a `$` that is neither (group 1 is None)
|
|
24
|
+
PLACEHOLDER = re.compile(r"\$(\$|[A-Z][A-Z0-9_]*)?")
|
|
25
|
+
# group 1 is the user info of a URL: `scheme://user:password@`
|
|
26
|
+
USERINFO = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://(?:([^/?#]*)@)?")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def read(directory, env=None, only=None):
|
|
30
|
+
"""Reads and checks every *.toml in the directory. With env, the placeholders of each connection are filled in
|
|
31
|
+
from it; without, they are only checked. With `only`, just that source's are filled: `list` needs no other
|
|
32
|
+
source's secrets. Returns (sources, tables, hash of the definition)."""
|
|
33
|
+
if not Path(directory).is_dir():
|
|
34
|
+
raise SystemExit(f"{directory} is not a directory")
|
|
35
|
+
whole, sources, tables, names = {}, [], [], {}
|
|
36
|
+
files = sorted(Path(directory).glob("*.toml"))
|
|
37
|
+
if not files:
|
|
38
|
+
raise SystemExit(f"{directory} holds no source files (*.toml); applying it would disable every table")
|
|
39
|
+
for path in files:
|
|
40
|
+
try:
|
|
41
|
+
data = tomllib.loads(path.read_text())
|
|
42
|
+
except tomllib.TOMLDecodeError as e:
|
|
43
|
+
raise SystemExit(f"{path.name}: {e}")
|
|
44
|
+
for name, s in data.items():
|
|
45
|
+
where = f"{path.name}: {name}"
|
|
46
|
+
if name in whole:
|
|
47
|
+
raise SystemExit(f"{where}: source {name} is defined twice")
|
|
48
|
+
whole[name] = s
|
|
49
|
+
settings(s, SOURCE_SETTINGS, where)
|
|
50
|
+
typos(s, where)
|
|
51
|
+
for k in TABLE_ONLY.keys() & s.keys(): # one table's, so no default for all, no connector option either
|
|
52
|
+
raise SystemExit(f"{where}: {k} belongs to a table, not to the source")
|
|
53
|
+
for required in ("type", "max_age", "connection", "tables"):
|
|
54
|
+
if required not in s:
|
|
55
|
+
raise SystemExit(f"{where}: {required} is missing")
|
|
56
|
+
interval(s["max_age"], where)
|
|
57
|
+
sources.append({"name": name, "type": s["type"], "parallel": s.get("parallel", 2),
|
|
58
|
+
"options": ordered({k: v for k, v in s.items() if k not in SOURCE_SETTINGS}),
|
|
59
|
+
"connection": fill(s["connection"], f"{where}: connection",
|
|
60
|
+
env if only in (None, name) else None)})
|
|
61
|
+
for table, t in s["tables"].items():
|
|
62
|
+
spot = f"{where}: {table}"
|
|
63
|
+
settings(t, {**TABLE_SETTINGS, **TABLE_ONLY}, spot)
|
|
64
|
+
typos(t, spot)
|
|
65
|
+
for k in (SOURCE_SETTINGS.keys() - TABLE_SETTINGS.keys()) & t.keys():
|
|
66
|
+
raise SystemExit(f"{spot}: {k} belongs to the source, not to a table")
|
|
67
|
+
key = t.get("key")
|
|
68
|
+
if key is not None and not (key and all(isinstance(c, str) and c for c in key)):
|
|
69
|
+
raise SystemExit(f"{spot}: key must be a list of column names")
|
|
70
|
+
# a file's extension is not a name part: `customers.csv` becomes `customers`
|
|
71
|
+
target = t["target_table"] if "target_table" in t else (
|
|
72
|
+
Path(table).stem if s["type"] == "file" else table.rsplit(".", 1)[-1]).lower()
|
|
73
|
+
schema = t.get("target_schema", s.get("target_schema", f"raw_{name}"))
|
|
74
|
+
# Postgres cuts names at 63 bytes; the change log and the history view add `__log`, `__history`
|
|
75
|
+
if len(schema.encode()) > 63:
|
|
76
|
+
raise SystemExit(f"{spot}: target_schema {schema} is longer than 63 bytes")
|
|
77
|
+
if len(target.encode()) > 54:
|
|
78
|
+
raise SystemExit(f"{spot}: target table {target} is longer than 54 bytes")
|
|
79
|
+
if not schema or not target or schema == "_rowvioli":
|
|
80
|
+
raise SystemExit(f"{spot}: target_schema and target_table must not be empty, "
|
|
81
|
+
"and the schema must not be _rowvioli")
|
|
82
|
+
for suffix in ("", "__log", "__history"):
|
|
83
|
+
if (schema, target + suffix) in names:
|
|
84
|
+
first_path, first = names[schema, target + suffix]
|
|
85
|
+
both = (f"{first_path}: {first} and {path.name}: {name}: {table}" if first_path != path.name
|
|
86
|
+
else f"{path.name}: {first} and {name}: {table}")
|
|
87
|
+
raise SystemExit(f"{both} both become {schema}.{target + suffix}; "
|
|
88
|
+
"set target_table at one of them")
|
|
89
|
+
names[schema, target + suffix] = path.name, f"{name}: {table}"
|
|
90
|
+
if s["type"] == "rest" and not str(t.get("path", s.get("path", "/"))).startswith("/"):
|
|
91
|
+
raise SystemExit(f"{spot}: path must start with /")
|
|
92
|
+
# no default zone: 14:00 that silently means UTC is found in production (decision 36)
|
|
93
|
+
times = {k: t.get(k, s.get(k)) for k in ("at", "timezone")}
|
|
94
|
+
if times["at"] and not times["timezone"]:
|
|
95
|
+
raise SystemExit(f"{spot}: at needs a timezone, from the table or the source")
|
|
96
|
+
tables.append({"source": name, "source_table": table, "target_schema": schema,
|
|
97
|
+
"target_table": target, "key": key,
|
|
98
|
+
"max_age": interval(t.get("max_age", s["max_age"]), spot),
|
|
99
|
+
**{k: t.get(k, s.get(k, DEFAULTS[k])) for k in DEFAULTS},
|
|
100
|
+
**times,
|
|
101
|
+
"options": ordered({k: v for k, v in t.items()
|
|
102
|
+
if k not in TABLE_SETTINGS and k not in TABLE_ONLY})})
|
|
103
|
+
# of the text as written: placeholders stay unfilled, so the hash says nothing about a secret
|
|
104
|
+
digest = hashlib.sha256(json.dumps(whole, sort_keys=True, default=str).encode()).hexdigest()
|
|
105
|
+
return sources, tables, digest
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def ordered(options):
|
|
109
|
+
"""jsonb forgets the order of a table's keys, so `columns` is stored as pairs (the REST mirror follows its order)."""
|
|
110
|
+
return {k: list(v.items()) if k == "columns" and isinstance(v, dict) else v for k, v in options.items()}
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def settings(block, known, where):
|
|
114
|
+
"""Checks that a source or table is a table and that its known settings have the right kind of value."""
|
|
115
|
+
if not isinstance(block, dict):
|
|
116
|
+
raise SystemExit(f"{where}: must be a table")
|
|
117
|
+
for k, kind in known.items():
|
|
118
|
+
# bool is a subclass of int, so true must not pass as a whole number and 1 not as true
|
|
119
|
+
if k in block and (not isinstance(block[k], kind) or isinstance(block[k], bool) != (kind is bool)):
|
|
120
|
+
raise SystemExit(f"{where}: {k} must be {KINDS[kind]}")
|
|
121
|
+
if k in block and k in RANGES and not RANGES[k][0](block[k]):
|
|
122
|
+
raise SystemExit(f"{where}: {k} must be {RANGES[k][1]}")
|
|
123
|
+
for time in block.get("at", []):
|
|
124
|
+
if not (isinstance(time, str) and re.fullmatch(r"([01]\d|2[0-3]):[0-5]\d", time)):
|
|
125
|
+
raise SystemExit(f"{where}: at {time!r} is not a time HH:MM from 00:00 to 23:59")
|
|
126
|
+
if "timezone" in block:
|
|
127
|
+
try:
|
|
128
|
+
ZoneInfo(block["timezone"])
|
|
129
|
+
except (KeyError, ValueError, OSError): # not found, not a zone's name, a directory
|
|
130
|
+
raise SystemExit(f"{where}: timezone {block['timezone']!r} is not a time zone")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def edits(a, b):
|
|
134
|
+
"""Edit distance of two words (insert, delete, replace)."""
|
|
135
|
+
row = list(range(len(b) + 1))
|
|
136
|
+
for i, x in enumerate(a, 1):
|
|
137
|
+
prev, row[0] = row[0], i
|
|
138
|
+
for j, y in enumerate(b, 1):
|
|
139
|
+
prev, row[j] = row[j], min(row[j] + 1, row[j - 1] + 1, prev + (x != y))
|
|
140
|
+
return row[-1]
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def typos(block, where):
|
|
144
|
+
"""A key that is no setting but a typo of one would silently go to the connector as an option. Short words
|
|
145
|
+
are left alone (`sep` is one edit from nothing but two from `key`): shortcut: one edit under 6 letters."""
|
|
146
|
+
known = SOURCE_SETTINGS.keys() | TABLE_ONLY.keys()
|
|
147
|
+
for k in block.keys() - known:
|
|
148
|
+
for setting in known:
|
|
149
|
+
if len(k) >= 4 and edits(k, setting) <= (2 if len(setting) >= 6 else 1):
|
|
150
|
+
raise SystemExit(f"{where}: {k} is not a setting; did you mean {setting}? "
|
|
151
|
+
"(other keys go to the connector as options)")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def interval(value, where):
|
|
155
|
+
m = re.fullmatch(r"([1-9]\d*)([smhd])", value)
|
|
156
|
+
if not m:
|
|
157
|
+
raise SystemExit(f"{where}: max_age {value!r} is not a whole number with s, m, h or d")
|
|
158
|
+
return f"{m[1]} {UNITS[m[2]]}"
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def duration(seconds):
|
|
162
|
+
"""Seconds as a person reads them, the largest two units: `5m`, `1h 3m`. For the status, not for the definition."""
|
|
163
|
+
parts = []
|
|
164
|
+
for unit, size in (("d", 86400), ("h", 3600), ("m", 60), ("s", 1)):
|
|
165
|
+
if seconds >= size and len(parts) < 2:
|
|
166
|
+
parts.append(f"{seconds // size}{unit}")
|
|
167
|
+
seconds %= size
|
|
168
|
+
return " ".join(parts) or "0s"
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def fill(template, where, env):
|
|
172
|
+
"""`$NAME` from env, `$$` a literal `$`, in every string of the connection, also nested ones. A value landing
|
|
173
|
+
in the user info of a URL is percent-encoded (decision 32). Without env, the placeholders are only checked. No
|
|
174
|
+
value ever appears in an error."""
|
|
175
|
+
if isinstance(template, dict):
|
|
176
|
+
return {k: fill(v, f"{where}.{k}", env) for k, v in template.items()}
|
|
177
|
+
if isinstance(template, list):
|
|
178
|
+
return [fill(v, where, env) for v in template]
|
|
179
|
+
if not isinstance(template, str):
|
|
180
|
+
return template
|
|
181
|
+
url = USERINFO.match(template)
|
|
182
|
+
userinfo_end = url.end(1) if url and url[1] is not None else -1
|
|
183
|
+
|
|
184
|
+
def sub(m):
|
|
185
|
+
if m[1] is None:
|
|
186
|
+
raise SystemExit(f"{where}: a $ must start a placeholder like $NAME or be written $$")
|
|
187
|
+
if m[1] == "$" or env is None:
|
|
188
|
+
return "$"
|
|
189
|
+
if m[1] not in env:
|
|
190
|
+
raise SystemExit(f"{where}: variable {m[1]} is not set")
|
|
191
|
+
return quote(env[m[1]], safe="") if m.start() < userinfo_end else env[m[1]]
|
|
192
|
+
return PLACEHOLDER.sub(sub, template)
|