continuo-postgres-adapter 0.3.2__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuo_postgres_adapter-0.4.0/PKG-INFO +102 -0
- continuo_postgres_adapter-0.4.0/README.md +87 -0
- {continuo_postgres_adapter-0.3.2 → continuo_postgres_adapter-0.4.0}/continuo_postgres_adapter/adapter.py +170 -1
- {continuo_postgres_adapter-0.3.2 → continuo_postgres_adapter-0.4.0}/pyproject.toml +2 -2
- {continuo_postgres_adapter-0.3.2 → continuo_postgres_adapter-0.4.0}/tests/test_adapter_postgres_validation.py +22 -1
- {continuo_postgres_adapter-0.3.2 → continuo_postgres_adapter-0.4.0}/tests/test_integration_postgres_conformance.py +5 -0
- continuo_postgres_adapter-0.4.0/tests/test_integration_postgres_dependents.py +701 -0
- continuo_postgres_adapter-0.4.0/tests/test_integration_postgres_entrypoint.py +209 -0
- continuo_postgres_adapter-0.4.0/tests/test_integration_postgres_seed_preflight.py +141 -0
- continuo_postgres_adapter-0.3.2/PKG-INFO +0 -58
- continuo_postgres_adapter-0.3.2/README.md +0 -43
- {continuo_postgres_adapter-0.3.2 → continuo_postgres_adapter-0.4.0}/.gitignore +0 -0
- {continuo_postgres_adapter-0.3.2 → continuo_postgres_adapter-0.4.0}/continuo_postgres_adapter/__init__.py +0 -0
- {continuo_postgres_adapter-0.3.2 → continuo_postgres_adapter-0.4.0}/tests/__init__.py +0 -0
- {continuo_postgres_adapter-0.3.2 → continuo_postgres_adapter-0.4.0}/tests/conftest.py +0 -0
- {continuo_postgres_adapter-0.3.2 → continuo_postgres_adapter-0.4.0}/tests/test_adapter_postgres_connect.py +0 -0
- {continuo_postgres_adapter-0.3.2 → continuo_postgres_adapter-0.4.0}/tests/test_adapter_runtime_postgres.py +0 -0
- {continuo_postgres_adapter-0.3.2 → continuo_postgres_adapter-0.4.0}/tests/test_integration_postgres_validation.py +0 -0
- {continuo_postgres_adapter-0.3.2 → continuo_postgres_adapter-0.4.0}/tests/test_integration_runtime_postgres.py +0 -0
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: continuo-postgres-adapter
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Postgres warehouse adapter for Continuo: validation and python-node runtime.
|
|
5
|
+
Author: Simone Carolini
|
|
6
|
+
Maintainer: Simone Carolini
|
|
7
|
+
Classifier: Development Status :: 4 - Beta
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
10
|
+
Requires-Python: >=3.14
|
|
11
|
+
Requires-Dist: continuo-engine-contract==0.10.0
|
|
12
|
+
Requires-Dist: psycopg2-binary==2.9.13
|
|
13
|
+
Requires-Dist: pyarrow==26.0.0
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# continuo-postgres-adapter
|
|
17
|
+
|
|
18
|
+
Postgres engine-adapter library for Continuo python nodes. `PostgresAdapter`
|
|
19
|
+
implements `continuo_engine_contract.port.WarehouseAdapter` (from the
|
|
20
|
+
`continuo-engine-contract` package) — one class covering both the data plane
|
|
21
|
+
(`fetch` / `ensure_table` / `load`) and release-time validation
|
|
22
|
+
(`ensure_schema` / `drop_schema` / `build_empty_from_sql` /
|
|
23
|
+
`build_empty_from_columns` / `clone_empty_from_prod` / `check_binds`) — and
|
|
24
|
+
registers itself under entry-point group `continuo_engine.adapters` as
|
|
25
|
+
`postgres`.
|
|
26
|
+
|
|
27
|
+
Connection env: `POSTGRES_HOST`, `POSTGRES_DB`, `POSTGRES_USER` (required);
|
|
28
|
+
`POSTGRES_PORT` (default 5432), `POSTGRES_PASSWORD` (default empty).
|
|
29
|
+
|
|
30
|
+
`from_env` tries a connection that no server answered (every address of the
|
|
31
|
+
host refused, had no route or sat on an unreachable network) up to 5 times,
|
|
32
|
+
2 s apart, logging each retry: a Job pod's first connection can be refused
|
|
33
|
+
for a few seconds after the pod starts, until the cluster's NetworkPolicy
|
|
34
|
+
engine has admitted the pod. Every other failure, an authentication failure or
|
|
35
|
+
a connect timeout on any address among them, is raised at once.
|
|
36
|
+
|
|
37
|
+
The connection runs with `autocommit = True`; every method owns its
|
|
38
|
+
transaction explicitly. `load()` and `ensure_table()` open an explicit
|
|
39
|
+
`BEGIN` around their statements and `COMMIT`/`ROLLBACK` themselves, so a
|
|
40
|
+
mid-sequence failure leaves no partial change. `fetch()` executes a single
|
|
41
|
+
read and commits (or rolls back on error), never leaving an open transaction
|
|
42
|
+
dangling.
|
|
43
|
+
|
|
44
|
+
`PostgresAdapter` declares `drop_semantics = "cascade"` and is a
|
|
45
|
+
`DependentsIntrospection`: `dependents(schema, table)` lists every object a
|
|
46
|
+
`DROP TABLE ... CASCADE` of the table destroys or changes, with
|
|
47
|
+
`completeness="exact"`. It walks `pg_depend` from the table through every normal
|
|
48
|
+
dependency, so it lists views and materialized views (found through their
|
|
49
|
+
`_RETURN` rule, at any depth: a view of a view of the table is depth 2),
|
|
50
|
+
SQL-standard functions and procedures, foreign keys and policies on other
|
|
51
|
+
tables, rules, inheriting tables, tables with a column of the table's row type
|
|
52
|
+
or of an array of it, the types built on it (a domain, a range, a composite
|
|
53
|
+
type, a domain over an array of it) and what reads them, including a column of
|
|
54
|
+
the array or multirange type of such a type, a generated column over a dropped
|
|
55
|
+
column and what reads it, and what reads a partition of the table or a sequence
|
|
56
|
+
of it (owned or identity). A table that only loses a column is listed, and from
|
|
57
|
+
it only what reads that column is walked on.
|
|
58
|
+
`information_schema.view_table_usage` misses the materialized views, the
|
|
59
|
+
functions and every view past the first level. The table's own indexes,
|
|
60
|
+
constraints (a self-referencing foreign key included), policies, rules, column
|
|
61
|
+
defaults, sequences, partitions and triggers are parts of the table, and the
|
|
62
|
+
array and multirange types, constructor functions and casts a type internally
|
|
63
|
+
owns (the constructors of a range or multirange type and the range-to-multirange
|
|
64
|
+
cast) are parts of the type: none is listed. A cast a user created, such as
|
|
65
|
+
`CREATE CAST (seed AS text)`, is dropped by the cascade and is listed as `other`
|
|
66
|
+
(`cast from <schema>.seed to text`). Each object appears once, at its shallowest depth.
|
|
67
|
+
The walk follows at most 32 dependency levels; an object deeper than that is not
|
|
68
|
+
listed. The table is found case-insensitively, so a seed loaded under another
|
|
69
|
+
letter case never makes the walk skip it, and a schema holding two tables that
|
|
70
|
+
differ only by letter case lists the dependants of both; a missing table or
|
|
71
|
+
schema gives no objects. Each `kind` is one of `view`, `materialized_view`,
|
|
72
|
+
`table`, `function`, `procedure`, `constraint`, `policy`, `rule` or `other`.
|
|
73
|
+
`other` is anything else: a domain, a range type or another type (named
|
|
74
|
+
`type <name>`), a composite type (its own name), an index, a user cast, a column
|
|
75
|
+
default of another table (named `default value for column ...`) or a trigger.
|
|
76
|
+
A name that Postgres describes (a type, a cast, a column default) is
|
|
77
|
+
schema-qualified unless the object is visible on the connection's `search_path`
|
|
78
|
+
(a domain in `public` is named `type my_domain`). The
|
|
79
|
+
name of a function carries its argument types (`seed_count()`), and that of a
|
|
80
|
+
constraint, policy or rule the table or domain it sits on
|
|
81
|
+
(`orders_seed_id_fkey on orders`). A listed object that has no table of its own
|
|
82
|
+
(a type, a column default, a trigger) has an empty schema and an empty owner.
|
|
83
|
+
CI runs the dependents suite against the postgres:16 test stack; it was also
|
|
84
|
+
verified against a postgres:18.3 server by pointing the suite at it with
|
|
85
|
+
`VR_IT_PG_PORT` (host `localhost`, database `warehouse`, user and password
|
|
86
|
+
`continuo`).
|
|
87
|
+
|
|
88
|
+
The `config:` index vocabulary is the union of the two vocabularies this
|
|
89
|
+
adapter inherited from the merge: `columns`, `unique`, `type`, and `name`,
|
|
90
|
+
with the strictest check from each side applied. The derived index name is
|
|
91
|
+
built from the table and the column list only — `type` and `unique` are not
|
|
92
|
+
folded into it, so two index entries on the same columns that differ only in
|
|
93
|
+
`type` (or only in `unique`) collide on that derived name and are rejected as
|
|
94
|
+
a duplicate. Give one of them an explicit `name:` when that is genuinely
|
|
95
|
+
intended.
|
|
96
|
+
|
|
97
|
+
Verification tier: unit tests are mock-free pure-logic tests (type-grammar
|
|
98
|
+
validation, Arrow table construction from rows); DDL/transactional behavior
|
|
99
|
+
(schema/table creation, load truncate+insert+rollback, validation DDL) is
|
|
100
|
+
verified against a live postgres:16 engine in
|
|
101
|
+
`tests/test_integration_runtime_postgres.py` and
|
|
102
|
+
`tests/test_integration_postgres_validation.py`.
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# continuo-postgres-adapter
|
|
2
|
+
|
|
3
|
+
Postgres engine-adapter library for Continuo python nodes. `PostgresAdapter`
|
|
4
|
+
implements `continuo_engine_contract.port.WarehouseAdapter` (from the
|
|
5
|
+
`continuo-engine-contract` package) — one class covering both the data plane
|
|
6
|
+
(`fetch` / `ensure_table` / `load`) and release-time validation
|
|
7
|
+
(`ensure_schema` / `drop_schema` / `build_empty_from_sql` /
|
|
8
|
+
`build_empty_from_columns` / `clone_empty_from_prod` / `check_binds`) — and
|
|
9
|
+
registers itself under entry-point group `continuo_engine.adapters` as
|
|
10
|
+
`postgres`.
|
|
11
|
+
|
|
12
|
+
Connection env: `POSTGRES_HOST`, `POSTGRES_DB`, `POSTGRES_USER` (required);
|
|
13
|
+
`POSTGRES_PORT` (default 5432), `POSTGRES_PASSWORD` (default empty).
|
|
14
|
+
|
|
15
|
+
`from_env` tries a connection that no server answered (every address of the
|
|
16
|
+
host refused, had no route or sat on an unreachable network) up to 5 times,
|
|
17
|
+
2 s apart, logging each retry: a Job pod's first connection can be refused
|
|
18
|
+
for a few seconds after the pod starts, until the cluster's NetworkPolicy
|
|
19
|
+
engine has admitted the pod. Every other failure, an authentication failure or
|
|
20
|
+
a connect timeout on any address among them, is raised at once.
|
|
21
|
+
|
|
22
|
+
The connection runs with `autocommit = True`; every method owns its
|
|
23
|
+
transaction explicitly. `load()` and `ensure_table()` open an explicit
|
|
24
|
+
`BEGIN` around their statements and `COMMIT`/`ROLLBACK` themselves, so a
|
|
25
|
+
mid-sequence failure leaves no partial change. `fetch()` executes a single
|
|
26
|
+
read and commits (or rolls back on error), never leaving an open transaction
|
|
27
|
+
dangling.
|
|
28
|
+
|
|
29
|
+
`PostgresAdapter` declares `drop_semantics = "cascade"` and is a
|
|
30
|
+
`DependentsIntrospection`: `dependents(schema, table)` lists every object a
|
|
31
|
+
`DROP TABLE ... CASCADE` of the table destroys or changes, with
|
|
32
|
+
`completeness="exact"`. It walks `pg_depend` from the table through every normal
|
|
33
|
+
dependency, so it lists views and materialized views (found through their
|
|
34
|
+
`_RETURN` rule, at any depth: a view of a view of the table is depth 2),
|
|
35
|
+
SQL-standard functions and procedures, foreign keys and policies on other
|
|
36
|
+
tables, rules, inheriting tables, tables with a column of the table's row type
|
|
37
|
+
or of an array of it, the types built on it (a domain, a range, a composite
|
|
38
|
+
type, a domain over an array of it) and what reads them, including a column of
|
|
39
|
+
the array or multirange type of such a type, a generated column over a dropped
|
|
40
|
+
column and what reads it, and what reads a partition of the table or a sequence
|
|
41
|
+
of it (owned or identity). A table that only loses a column is listed, and from
|
|
42
|
+
it only what reads that column is walked on.
|
|
43
|
+
`information_schema.view_table_usage` misses the materialized views, the
|
|
44
|
+
functions and every view past the first level. The table's own indexes,
|
|
45
|
+
constraints (a self-referencing foreign key included), policies, rules, column
|
|
46
|
+
defaults, sequences, partitions and triggers are parts of the table, and the
|
|
47
|
+
array and multirange types, constructor functions and casts a type internally
|
|
48
|
+
owns (the constructors of a range or multirange type and the range-to-multirange
|
|
49
|
+
cast) are parts of the type: none is listed. A cast a user created, such as
|
|
50
|
+
`CREATE CAST (seed AS text)`, is dropped by the cascade and is listed as `other`
|
|
51
|
+
(`cast from <schema>.seed to text`). Each object appears once, at its shallowest depth.
|
|
52
|
+
The walk follows at most 32 dependency levels; an object deeper than that is not
|
|
53
|
+
listed. The table is found case-insensitively, so a seed loaded under another
|
|
54
|
+
letter case never makes the walk skip it, and a schema holding two tables that
|
|
55
|
+
differ only by letter case lists the dependants of both; a missing table or
|
|
56
|
+
schema gives no objects. Each `kind` is one of `view`, `materialized_view`,
|
|
57
|
+
`table`, `function`, `procedure`, `constraint`, `policy`, `rule` or `other`.
|
|
58
|
+
`other` is anything else: a domain, a range type or another type (named
|
|
59
|
+
`type <name>`), a composite type (its own name), an index, a user cast, a column
|
|
60
|
+
default of another table (named `default value for column ...`) or a trigger.
|
|
61
|
+
A name that Postgres describes (a type, a cast, a column default) is
|
|
62
|
+
schema-qualified unless the object is visible on the connection's `search_path`
|
|
63
|
+
(a domain in `public` is named `type my_domain`). The
|
|
64
|
+
name of a function carries its argument types (`seed_count()`), and that of a
|
|
65
|
+
constraint, policy or rule the table or domain it sits on
|
|
66
|
+
(`orders_seed_id_fkey on orders`). A listed object that has no table of its own
|
|
67
|
+
(a type, a column default, a trigger) has an empty schema and an empty owner.
|
|
68
|
+
CI runs the dependents suite against the postgres:16 test stack; it was also
|
|
69
|
+
verified against a postgres:18.3 server by pointing the suite at it with
|
|
70
|
+
`VR_IT_PG_PORT` (host `localhost`, database `warehouse`, user and password
|
|
71
|
+
`continuo`).
|
|
72
|
+
|
|
73
|
+
The `config:` index vocabulary is the union of the two vocabularies this
|
|
74
|
+
adapter inherited from the merge: `columns`, `unique`, `type`, and `name`,
|
|
75
|
+
with the strictest check from each side applied. The derived index name is
|
|
76
|
+
built from the table and the column list only — `type` and `unique` are not
|
|
77
|
+
folded into it, so two index entries on the same columns that differ only in
|
|
78
|
+
`type` (or only in `unique`) collide on that derived name and are rejected as
|
|
79
|
+
a duplicate. Give one of them an explicit `name:` when that is genuinely
|
|
80
|
+
intended.
|
|
81
|
+
|
|
82
|
+
Verification tier: unit tests are mock-free pure-logic tests (type-grammar
|
|
83
|
+
validation, Arrow table construction from rows); DDL/transactional behavior
|
|
84
|
+
(schema/table creation, load truncate+insert+rollback, validation DDL) is
|
|
85
|
+
verified against a live postgres:16 engine in
|
|
86
|
+
`tests/test_integration_runtime_postgres.py` and
|
|
87
|
+
`tests/test_integration_postgres_validation.py`.
|
|
@@ -12,6 +12,8 @@ One class covers both roles Continuo asks of an engine:
|
|
|
12
12
|
its ``PRIMARY KEY``); ``load`` atomically replaces its contents, ``append``
|
|
13
13
|
adds rows and ``_upsert`` inserts or overwrites by key, all through one
|
|
14
14
|
insert helper; ``table_columns`` reads its shape.
|
|
15
|
+
* **Drop impact** — ``dependents`` lists every object a drop of a table would
|
|
16
|
+
cascade to, from ``pg_depend``.
|
|
15
17
|
|
|
16
18
|
The connection runs with ``autocommit = True``: every method owns its
|
|
17
19
|
transaction explicitly rather than relying on psycopg2's implicit
|
|
@@ -40,6 +42,9 @@ import pyarrow as pa # type: ignore[import-untyped]
|
|
|
40
42
|
from continuo_engine_contract.config import ensure_known_keys # type: ignore[import-untyped]
|
|
41
43
|
from continuo_engine_contract.port import ( # type: ignore[import-untyped]
|
|
42
44
|
AppendCapable,
|
|
45
|
+
Dependent,
|
|
46
|
+
DependentsIntrospection,
|
|
47
|
+
Dependents,
|
|
43
48
|
LiveColumn,
|
|
44
49
|
UpsertCapable,
|
|
45
50
|
WarehouseAdapter,
|
|
@@ -443,9 +448,136 @@ def _arrow_table_from_rows(colnames: list[str], rows: list[tuple[Any, ...]]) ->
|
|
|
443
448
|
return pa.table({name: pa.array(values) for name, values in zip(colnames, by_column)})
|
|
444
449
|
|
|
445
450
|
|
|
446
|
-
|
|
451
|
+
# Every object that depends on a table, found by following pg_depend's normal
|
|
452
|
+
# dependencies (deptype 'n'), the ones DROP ... CASCADE follows. The walk's
|
|
453
|
+
# nodes are (class, object, subid): subid 0 is the whole object, a positive
|
|
454
|
+
# subid one column of a table or composite type. A column a cascade drops leaves
|
|
455
|
+
# its table in place, so from a column node only the dependencies on that column
|
|
456
|
+
# are followed, and the table itself is still listed. A view's dependency is
|
|
457
|
+
# recorded on its _RETURN rewrite rule, so the walk steps from the rule to the
|
|
458
|
+
# view it belongs to. The expression of a generated column (a pg_attrdef row) is
|
|
459
|
+
# dropped together with its column, so the walk steps from it to that column. A
|
|
460
|
+
# table that has a column of the walked table's row type, or of an array of it,
|
|
461
|
+
# depends on that type (pg_type), not on the table, so a whole-table step also
|
|
462
|
+
# follows the dependencies on its reltype and typarray. What a type the walk
|
|
463
|
+
# reaches (a domain, a range) internally owns (deptype 'i': its array type, its
|
|
464
|
+
# multirange type, its constructor functions and casts) is dropped with it, so a
|
|
465
|
+
# pg_type step also follows every internal dependency on it. A partition, a
|
|
466
|
+
# sequence owned by a column (deptype 'a'), an identity sequence and an
|
|
467
|
+
# internally owned object are parts of their owner, yet what reads them is
|
|
468
|
+
# dropped with it: the walk steps through them without adding a level, and lists
|
|
469
|
+
# none of them. The table's own indexes and triggers are automatic dependencies
|
|
470
|
+
# that nothing reads, so the walk does not follow them. The depth bound of 32
|
|
471
|
+
# keeps a dependency cycle from running forever, and ends a chain deeper than
|
|
472
|
+
# that unlisted.
|
|
473
|
+
_DEPENDENTS_SQL = """
|
|
474
|
+
WITH RECURSIVE walk(classid, objid, subid, depth) AS (
|
|
475
|
+
SELECT 'pg_class'::regclass::oid, c.oid, 0, 0
|
|
476
|
+
FROM pg_class c
|
|
477
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
478
|
+
WHERE lower(n.nspname) = lower(%(schema)s)
|
|
479
|
+
AND lower(c.relname) = lower(%(table)s)
|
|
480
|
+
AND c.relkind IN ('r', 'p')
|
|
481
|
+
UNION
|
|
482
|
+
SELECT CASE WHEN rw.oid IS NULL AND gad.oid IS NULL THEN dep.classid ELSE 'pg_class'::regclass::oid END,
|
|
483
|
+
CASE WHEN rw.oid IS NOT NULL THEN rw.ev_class
|
|
484
|
+
WHEN gad.oid IS NOT NULL THEN gad.adrelid
|
|
485
|
+
ELSE dep.objid END,
|
|
486
|
+
CASE WHEN rw.oid IS NOT NULL THEN 0
|
|
487
|
+
WHEN gad.oid IS NOT NULL THEN gad.adnum::int
|
|
488
|
+
ELSE dep.objsubid END,
|
|
489
|
+
w.depth + CASE WHEN dep.deptype IN ('a', 'i') THEN 0 ELSE 1 END
|
|
490
|
+
FROM walk w
|
|
491
|
+
LEFT JOIN pg_class wc
|
|
492
|
+
ON w.classid = 'pg_class'::regclass::oid AND w.subid = 0 AND wc.oid = w.objid
|
|
493
|
+
LEFT JOIN pg_type rowtype ON rowtype.oid = wc.reltype
|
|
494
|
+
JOIN pg_depend dep
|
|
495
|
+
ON (dep.refclassid = w.classid AND dep.refobjid = w.objid
|
|
496
|
+
AND (w.subid = 0 OR dep.refobjsubid = w.subid))
|
|
497
|
+
OR (dep.refclassid = 'pg_type'::regclass::oid
|
|
498
|
+
AND dep.refobjid IN (rowtype.oid, rowtype.typarray))
|
|
499
|
+
LEFT JOIN pg_class dc
|
|
500
|
+
ON dep.classid = 'pg_class'::regclass::oid AND dc.oid = dep.objid
|
|
501
|
+
LEFT JOIN pg_rewrite rw
|
|
502
|
+
ON dep.classid = 'pg_rewrite'::regclass::oid AND rw.oid = dep.objid AND rw.rulename = '_RETURN'
|
|
503
|
+
LEFT JOIN pg_attrdef gad
|
|
504
|
+
ON dep.classid = 'pg_attrdef'::regclass::oid AND gad.oid = dep.objid
|
|
505
|
+
AND EXISTS (SELECT 1 FROM pg_attribute ga
|
|
506
|
+
WHERE ga.attrelid = gad.adrelid AND ga.attnum = gad.adnum AND ga.attgenerated <> '')
|
|
507
|
+
WHERE w.depth < 32
|
|
508
|
+
AND (dep.deptype = 'n'
|
|
509
|
+
OR (dep.deptype IN ('a', 'i') AND dc.relkind = 'S')
|
|
510
|
+
OR (dep.deptype = 'a' AND dc.relkind IN ('r', 'p', 'f') AND dc.relispartition)
|
|
511
|
+
OR (dep.deptype = 'i' AND w.classid = 'pg_type'::regclass::oid AND dep.refclassid = w.classid))
|
|
512
|
+
AND (rw.oid IS NULL OR rw.ev_class <> w.objid)
|
|
513
|
+
)
|
|
514
|
+
SELECT found.catalog,
|
|
515
|
+
CASE found.catalog WHEN 'pg_class' THEN c.relkind::text WHEN 'pg_proc' THEN p.prokind::text ELSE '' END,
|
|
516
|
+
COALESCE(n.nspname, ''),
|
|
517
|
+
CASE found.catalog
|
|
518
|
+
WHEN 'pg_class' THEN c.relname
|
|
519
|
+
WHEN 'pg_proc' THEN p.proname || '(' || pg_get_function_identity_arguments(p.oid) || ')'
|
|
520
|
+
WHEN 'pg_constraint' THEN
|
|
521
|
+
CASE WHEN k.conrelid = 0 THEN k.conname || ' on ' || format_type(k.contypid, NULL)
|
|
522
|
+
ELSE k.conname || ' on ' || t.relname END
|
|
523
|
+
WHEN 'pg_policy' THEN pol.polname || ' on ' || t.relname
|
|
524
|
+
WHEN 'pg_rewrite' THEN rw.rulename || ' on ' || t.relname
|
|
525
|
+
ELSE pg_describe_object(found.classid, found.objid, 0)
|
|
526
|
+
END,
|
|
527
|
+
COALESCE(pg_get_userbyid(COALESCE(c.relowner, p.proowner, t.relowner)), ''),
|
|
528
|
+
found.depth
|
|
529
|
+
FROM (
|
|
530
|
+
SELECT classid, objid, classid::regclass::text AS catalog, min(depth) AS depth
|
|
531
|
+
FROM walk
|
|
532
|
+
GROUP BY classid, objid
|
|
533
|
+
HAVING min(depth) > 0
|
|
534
|
+
) found
|
|
535
|
+
LEFT JOIN pg_class c ON found.catalog = 'pg_class' AND c.oid = found.objid
|
|
536
|
+
LEFT JOIN pg_proc p ON found.catalog = 'pg_proc' AND p.oid = found.objid
|
|
537
|
+
LEFT JOIN pg_constraint k ON found.catalog = 'pg_constraint' AND k.oid = found.objid
|
|
538
|
+
LEFT JOIN pg_policy pol ON found.catalog = 'pg_policy' AND pol.oid = found.objid
|
|
539
|
+
LEFT JOIN pg_rewrite rw ON found.catalog = 'pg_rewrite' AND rw.oid = found.objid
|
|
540
|
+
LEFT JOIN pg_attrdef ad ON found.catalog = 'pg_attrdef' AND ad.oid = found.objid
|
|
541
|
+
LEFT JOIN pg_trigger tg ON found.catalog = 'pg_trigger' AND tg.oid = found.objid
|
|
542
|
+
LEFT JOIN pg_class t ON t.oid = COALESCE(k.conrelid, pol.polrelid, rw.ev_class)
|
|
543
|
+
LEFT JOIN pg_namespace n ON n.oid = COALESCE(c.relnamespace, p.pronamespace, t.relnamespace)
|
|
544
|
+
-- a sequence, a partition and the per-partition copy of a foreign key are parts of their owner
|
|
545
|
+
WHERE NOT (found.catalog = 'pg_class'
|
|
546
|
+
AND (c.relkind = 'S' OR (c.relkind IN ('r', 'p', 'f') AND c.relispartition)))
|
|
547
|
+
-- so is whatever a type internally owns (its array and multirange types, its constructor functions and
|
|
548
|
+
-- casts), except a composite type's own pg_class row
|
|
549
|
+
AND NOT (found.catalog <> 'pg_class'
|
|
550
|
+
AND EXISTS (SELECT 1 FROM pg_depend own
|
|
551
|
+
WHERE own.classid = found.classid AND own.objid = found.objid
|
|
552
|
+
AND own.deptype = 'i' AND own.refclassid = 'pg_type'::regclass::oid))
|
|
553
|
+
AND NOT (found.catalog = 'pg_constraint' AND k.conparentid <> 0)
|
|
554
|
+
-- a constraint, default, policy, rule or trigger of the table itself is part of the table
|
|
555
|
+
AND NOT COALESCE(COALESCE(k.conrelid, ad.adrelid, pol.polrelid, rw.ev_class, tg.tgrelid) IN (
|
|
556
|
+
SELECT objid FROM walk WHERE depth = 0 AND classid = 'pg_class'::regclass::oid), false)
|
|
557
|
+
ORDER BY found.depth, 3, 4
|
|
558
|
+
"""
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def _dependent_kind(catalog: str, subkind: str) -> str:
|
|
562
|
+
"""The contract's kind for a catalog row: *subkind* is ``relkind`` for ``pg_class``, ``prokind`` for ``pg_proc``.
|
|
563
|
+
|
|
564
|
+
A ``pg_class`` row that is no table, view or materialized view (a composite
|
|
565
|
+
type, an index) is ``other``.
|
|
566
|
+
"""
|
|
567
|
+
if catalog == "pg_class":
|
|
568
|
+
return {"v": "view", "m": "materialized_view", "r": "table", "p": "table", "f": "table"}.get(
|
|
569
|
+
subkind, "other"
|
|
570
|
+
)
|
|
571
|
+
if catalog == "pg_proc":
|
|
572
|
+
return "procedure" if subkind == "p" else "function"
|
|
573
|
+
return {"pg_constraint": "constraint", "pg_policy": "policy", "pg_rewrite": "rule"}.get(catalog, "other")
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
class PostgresAdapter(WarehouseAdapter, AppendCapable, UpsertCapable, DependentsIntrospection):
|
|
447
577
|
"""WarehouseAdapter speaking postgres over a psycopg2 connection."""
|
|
448
578
|
|
|
579
|
+
drop_semantics = "cascade"
|
|
580
|
+
|
|
449
581
|
def __init__(self, conn: "psycopg2.extensions.connection") -> None:
|
|
450
582
|
self._conn = conn
|
|
451
583
|
self._conn.autocommit = True
|
|
@@ -716,6 +848,43 @@ class PostgresAdapter(WarehouseAdapter, AppendCapable, UpsertCapable):
|
|
|
716
848
|
raise LookupError(f"table {schema}.{table} does not exist")
|
|
717
849
|
return [LiveColumn(name=name, nullable=is_nullable == "YES") for name, is_nullable in rows]
|
|
718
850
|
|
|
851
|
+
def dependents(self, schema: str, table: str) -> Dependents:
|
|
852
|
+
"""Every object a ``DROP TABLE schema.table CASCADE`` destroys or changes, exactly.
|
|
853
|
+
|
|
854
|
+
Lists views, materialized views, SQL-standard functions and procedures,
|
|
855
|
+
foreign keys and policies on other tables, rules, inheriting tables,
|
|
856
|
+
tables with a column of this table's row type, or of an array of it,
|
|
857
|
+
and the types built on it (a domain, a range, a composite type, and a
|
|
858
|
+
domain over an array of it), each once at its shallowest depth, with
|
|
859
|
+
what reads them: a column of an array or multirange type of such a
|
|
860
|
+
type, a generated column over a dropped column, and what reads a
|
|
861
|
+
partition of the table or a sequence of it (owned or identity). A table
|
|
862
|
+
that only loses a column is listed, and only what reads that column is
|
|
863
|
+
walked on from it. The table's own indexes, constraints (a
|
|
864
|
+
self-referencing foreign key included), policies, rules, column
|
|
865
|
+
defaults, sequences, partitions and triggers, and what a type owns
|
|
866
|
+
internally (its array and multirange types, constructor functions and
|
|
867
|
+
casts), are parts of their owner and are not listed.
|
|
868
|
+
|
|
869
|
+
The walk follows at most 32 dependency levels: an object deeper than
|
|
870
|
+
that is not listed.
|
|
871
|
+
|
|
872
|
+
The table is found case-insensitively, so a seed loaded under another
|
|
873
|
+
letter case still has its dependants listed, and the walk never
|
|
874
|
+
under-reports; a schema holding two tables that differ only by case
|
|
875
|
+
lists the dependants of both. A missing table or schema gives no objects.
|
|
876
|
+
"""
|
|
877
|
+
with self._conn.cursor() as cur:
|
|
878
|
+
cur.execute(_DEPENDENTS_SQL, {"schema": schema, "table": table})
|
|
879
|
+
rows = cur.fetchall()
|
|
880
|
+
return Dependents(
|
|
881
|
+
objects=tuple(
|
|
882
|
+
Dependent(dep_schema, name, _dependent_kind(catalog, subkind), owner or "", depth)
|
|
883
|
+
for catalog, subkind, dep_schema, name, owner, depth in rows
|
|
884
|
+
),
|
|
885
|
+
completeness="exact",
|
|
886
|
+
)
|
|
887
|
+
|
|
719
888
|
@classmethod
|
|
720
889
|
def validate_config(
|
|
721
890
|
cls, config: dict[str, Any] | None, column_names: list[str]
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
[project]
|
|
2
|
-
version = "0.
|
|
2
|
+
version = "0.4.0"
|
|
3
3
|
name = "continuo-postgres-adapter"
|
|
4
4
|
description = "Postgres warehouse adapter for Continuo: validation and python-node runtime."
|
|
5
5
|
authors = [{ name = "Simone Carolini" }]
|
|
@@ -12,7 +12,7 @@ dependencies = [
|
|
|
12
12
|
# of this adapter would otherwise accept whatever contract release is
|
|
13
13
|
# newest. This adapter subclasses the contract's WarehouseAdapter port
|
|
14
14
|
# directly, so the pair must stay the tested one.
|
|
15
|
-
"continuo-engine-contract==0.
|
|
15
|
+
"continuo-engine-contract==0.10.0",
|
|
16
16
|
"psycopg2-binary==2.9.13",
|
|
17
17
|
"pyarrow==26.0.0",
|
|
18
18
|
]
|
|
@@ -7,7 +7,12 @@ cursors/connections here.
|
|
|
7
7
|
"""
|
|
8
8
|
import pytest
|
|
9
9
|
|
|
10
|
-
from continuo_postgres_adapter.adapter import
|
|
10
|
+
from continuo_postgres_adapter.adapter import (
|
|
11
|
+
PostgresAdapter,
|
|
12
|
+
_dependent_kind,
|
|
13
|
+
_index_name,
|
|
14
|
+
_validated_indexes,
|
|
15
|
+
)
|
|
11
16
|
|
|
12
17
|
|
|
13
18
|
def test_required_env_names_connection_vars():
|
|
@@ -17,6 +22,10 @@ def test_required_env_names_connection_vars():
|
|
|
17
22
|
]
|
|
18
23
|
|
|
19
24
|
|
|
25
|
+
def test_dropping_a_table_cascades_to_its_dependents():
|
|
26
|
+
assert PostgresAdapter.drop_semantics == "cascade"
|
|
27
|
+
|
|
28
|
+
|
|
20
29
|
def test_build_empty_from_columns_rejects_bad_type_before_touching_db() -> None:
|
|
21
30
|
"""Test that a malformed type string is rejected before any DB access."""
|
|
22
31
|
adapter = PostgresAdapter.__new__(PostgresAdapter) # no connection needed
|
|
@@ -197,3 +206,15 @@ def test_an_explicit_name_admits_two_access_methods_over_one_column_set():
|
|
|
197
206
|
]
|
|
198
207
|
entries = _validated_indexes({"indexes": indexes}, "t", ["id"])
|
|
199
208
|
assert [e["name"] for e in entries] == ["ix_t_id", "ix_t_id_hash"]
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
@pytest.mark.parametrize(("catalog", "subkind", "kind"), [
|
|
212
|
+
("pg_class", "v", "view"), ("pg_class", "m", "materialized_view"), ("pg_class", "r", "table"),
|
|
213
|
+
("pg_class", "p", "table"), ("pg_class", "f", "table"), ("pg_proc", "f", "function"),
|
|
214
|
+
("pg_proc", "p", "procedure"), ("pg_constraint", "", "constraint"), ("pg_policy", "", "policy"),
|
|
215
|
+
("pg_rewrite", "", "rule"), ("pg_trigger", "", "other"), ("pg_class", "c", "other"),
|
|
216
|
+
("pg_class", "i", "other"), ("pg_class", "I", "other"), ("pg_class", "S", "other"),
|
|
217
|
+
("pg_class", "t", "other"),
|
|
218
|
+
])
|
|
219
|
+
def test_dependent_kind_maps_each_catalog_row(catalog, subkind, kind):
|
|
220
|
+
assert _dependent_kind(catalog, subkind) == kind
|
|
@@ -17,6 +17,7 @@ from continuo_engine_contract.testing.conformance import (
|
|
|
17
17
|
NAMES,
|
|
18
18
|
AppendConformance,
|
|
19
19
|
ConformanceBase,
|
|
20
|
+
DependentsConformance,
|
|
20
21
|
IntrospectionConformance,
|
|
21
22
|
StartupConformance,
|
|
22
23
|
UpsertConformance,
|
|
@@ -83,6 +84,10 @@ class TestPostgresUpsert(UpsertConformance):
|
|
|
83
84
|
pass
|
|
84
85
|
|
|
85
86
|
|
|
87
|
+
class TestPostgresDependents(DependentsConformance):
|
|
88
|
+
pass
|
|
89
|
+
|
|
90
|
+
|
|
86
91
|
def _primary_key_columns(schema: str, table: str) -> list[str]:
|
|
87
92
|
conn = psycopg2.connect(**PG)
|
|
88
93
|
try:
|