continuo-postgres-adapter 0.3.1__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (19) hide show
  1. continuo_postgres_adapter-0.4.0/PKG-INFO +102 -0
  2. continuo_postgres_adapter-0.4.0/README.md +87 -0
  3. {continuo_postgres_adapter-0.3.1 → continuo_postgres_adapter-0.4.0}/continuo_postgres_adapter/adapter.py +238 -12
  4. {continuo_postgres_adapter-0.3.1 → continuo_postgres_adapter-0.4.0}/pyproject.toml +6 -6
  5. continuo_postgres_adapter-0.4.0/tests/test_adapter_postgres_connect.py +209 -0
  6. {continuo_postgres_adapter-0.3.1 → continuo_postgres_adapter-0.4.0}/tests/test_adapter_postgres_validation.py +22 -1
  7. {continuo_postgres_adapter-0.3.1 → continuo_postgres_adapter-0.4.0}/tests/test_integration_postgres_conformance.py +28 -0
  8. continuo_postgres_adapter-0.4.0/tests/test_integration_postgres_dependents.py +701 -0
  9. continuo_postgres_adapter-0.4.0/tests/test_integration_postgres_entrypoint.py +209 -0
  10. continuo_postgres_adapter-0.4.0/tests/test_integration_postgres_seed_preflight.py +141 -0
  11. {continuo_postgres_adapter-0.3.1 → continuo_postgres_adapter-0.4.0}/tests/test_integration_runtime_postgres.py +29 -0
  12. continuo_postgres_adapter-0.3.1/PKG-INFO +0 -51
  13. continuo_postgres_adapter-0.3.1/README.md +0 -36
  14. {continuo_postgres_adapter-0.3.1 → continuo_postgres_adapter-0.4.0}/.gitignore +0 -0
  15. {continuo_postgres_adapter-0.3.1 → continuo_postgres_adapter-0.4.0}/continuo_postgres_adapter/__init__.py +0 -0
  16. {continuo_postgres_adapter-0.3.1 → continuo_postgres_adapter-0.4.0}/tests/__init__.py +0 -0
  17. {continuo_postgres_adapter-0.3.1 → continuo_postgres_adapter-0.4.0}/tests/conftest.py +0 -0
  18. {continuo_postgres_adapter-0.3.1 → continuo_postgres_adapter-0.4.0}/tests/test_adapter_runtime_postgres.py +0 -0
  19. {continuo_postgres_adapter-0.3.1 → continuo_postgres_adapter-0.4.0}/tests/test_integration_postgres_validation.py +0 -0
@@ -0,0 +1,102 @@
1
+ Metadata-Version: 2.5
2
+ Name: continuo-postgres-adapter
3
+ Version: 0.4.0
4
+ Summary: Postgres warehouse adapter for Continuo: validation and python-node runtime.
5
+ Author: Simone Carolini
6
+ Maintainer: Simone Carolini
7
+ Classifier: Development Status :: 4 - Beta
8
+ Classifier: Intended Audience :: Developers
9
+ Classifier: Programming Language :: Python :: 3.14
10
+ Requires-Python: >=3.14
11
+ Requires-Dist: continuo-engine-contract==0.10.0
12
+ Requires-Dist: psycopg2-binary==2.9.13
13
+ Requires-Dist: pyarrow==26.0.0
14
+ Description-Content-Type: text/markdown
15
+
16
+ # continuo-postgres-adapter
17
+
18
+ Postgres engine-adapter library for Continuo python nodes. `PostgresAdapter`
19
+ implements `continuo_engine_contract.port.WarehouseAdapter` (from the
20
+ `continuo-engine-contract` package) — one class covering both the data plane
21
+ (`fetch` / `ensure_table` / `load`) and release-time validation
22
+ (`ensure_schema` / `drop_schema` / `build_empty_from_sql` /
23
+ `build_empty_from_columns` / `clone_empty_from_prod` / `check_binds`) — and
24
+ registers itself under entry-point group `continuo_engine.adapters` as
25
+ `postgres`.
26
+
27
+ Connection env: `POSTGRES_HOST`, `POSTGRES_DB`, `POSTGRES_USER` (required);
28
+ `POSTGRES_PORT` (default 5432), `POSTGRES_PASSWORD` (default empty).
29
+
30
+ `from_env` tries a connection that no server answered (every address of the
31
+ host refused, had no route or sat on an unreachable network) up to 5 times,
32
+ 2 s apart, logging each retry: a Job pod's first connection can be refused
33
+ for a few seconds after the pod starts, until the cluster's NetworkPolicy
34
+ engine has admitted the pod. Every other failure, an authentication failure or
35
+ a connect timeout on any address among them, is raised at once.
36
+
37
+ The connection runs with `autocommit = True`; every method owns its
38
+ transaction explicitly. `load()` and `ensure_table()` open an explicit
39
+ `BEGIN` around their statements and `COMMIT`/`ROLLBACK` themselves, so a
40
+ mid-sequence failure leaves no partial change. `fetch()` executes a single
41
+ read and commits (or rolls back on error), never leaving an open transaction
42
+ dangling.
43
+
44
+ `PostgresAdapter` declares `drop_semantics = "cascade"` and is a
45
+ `DependentsIntrospection`: `dependents(schema, table)` lists every object a
46
+ `DROP TABLE ... CASCADE` of the table destroys or changes, with
47
+ `completeness="exact"`. It walks `pg_depend` from the table through every normal
48
+ dependency, so it lists views and materialized views (found through their
49
+ `_RETURN` rule, at any depth: a view of a view of the table is depth 2),
50
+ SQL-standard functions and procedures, foreign keys and policies on other
51
+ tables, rules, inheriting tables, tables with a column of the table's row type
52
+ or of an array of it, the types built on it (a domain, a range, a composite
53
+ type, a domain over an array of it) and what reads them, including a column of
54
+ the array or multirange type of such a type, a generated column over a dropped
55
+ column and what reads it, and what reads a partition of the table or a sequence
56
+ of it (owned or identity). A table that only loses a column is listed, and from
57
+ it only what reads that column is walked on.
58
+ `information_schema.view_table_usage` misses the materialized views, the
59
+ functions and every view past the first level. The table's own indexes,
60
+ constraints (a self-referencing foreign key included), policies, rules, column
61
+ defaults, sequences, partitions and triggers are parts of the table, and the
62
+ array and multirange types, constructor functions and casts a type internally
63
+ owns (the constructors of a range or multirange type and the range-to-multirange
64
+ cast) are parts of the type: none is listed. A cast a user created, such as
65
+ `CREATE CAST (seed AS text)`, is dropped by the cascade and is listed as `other`
66
+ (`cast from <schema>.seed to text`). Each object appears once, at its shallowest depth.
67
+ The walk follows at most 32 dependency levels; an object deeper than that is not
68
+ listed. The table is found case-insensitively, so a seed loaded under another
69
+ letter case never makes the walk skip it, and a schema holding two tables that
70
+ differ only by letter case lists the dependants of both; a missing table or
71
+ schema gives no objects. Each `kind` is one of `view`, `materialized_view`,
72
+ `table`, `function`, `procedure`, `constraint`, `policy`, `rule` or `other`.
73
+ `other` is anything else: a domain, a range type or another type (named
74
+ `type <name>`), a composite type (its own name), an index, a user cast, a column
75
+ default of another table (named `default value for column ...`) or a trigger.
76
+ A name that Postgres describes (a type, a cast, a column default) is
77
+ schema-qualified unless the object is visible on the connection's `search_path`
78
+ (a domain in `public` is named `type my_domain`). The
79
+ name of a function carries its argument types (`seed_count()`), and that of a
80
+ constraint, policy or rule the table or domain it sits on
81
+ (`orders_seed_id_fkey on orders`). A listed object that has no table of its own
82
+ (a type, a column default, a trigger) has an empty schema and an empty owner.
83
+ CI runs the dependents suite against the postgres:16 test stack; it was also
84
+ verified against a postgres:18.3 server by pointing the suite at it with
85
+ `VR_IT_PG_PORT` (host `localhost`, database `warehouse`, user and password
86
+ `continuo`).
87
+
88
+ The `config:` index vocabulary is the union of the two vocabularies this
89
+ adapter inherited from the merge: `columns`, `unique`, `type`, and `name`,
90
+ with the strictest check from each side applied. The derived index name is
91
+ built from the table and the column list only — `type` and `unique` are not
92
+ folded into it, so two index entries on the same columns that differ only in
93
+ `type` (or only in `unique`) collide on that derived name and are rejected as
94
+ a duplicate. Give one of them an explicit `name:` when that is genuinely
95
+ intended.
96
+
97
+ Verification tier: unit tests are mock-free pure-logic tests (type-grammar
98
+ validation, Arrow table construction from rows); DDL/transactional behavior
99
+ (schema/table creation, load truncate+insert+rollback, validation DDL) is
100
+ verified against a live postgres:16 engine in
101
+ `tests/test_integration_runtime_postgres.py` and
102
+ `tests/test_integration_postgres_validation.py`.
@@ -0,0 +1,87 @@
1
+ # continuo-postgres-adapter
2
+
3
+ Postgres engine-adapter library for Continuo python nodes. `PostgresAdapter`
4
+ implements `continuo_engine_contract.port.WarehouseAdapter` (from the
5
+ `continuo-engine-contract` package) — one class covering both the data plane
6
+ (`fetch` / `ensure_table` / `load`) and release-time validation
7
+ (`ensure_schema` / `drop_schema` / `build_empty_from_sql` /
8
+ `build_empty_from_columns` / `clone_empty_from_prod` / `check_binds`) — and
9
+ registers itself under entry-point group `continuo_engine.adapters` as
10
+ `postgres`.
11
+
12
+ Connection env: `POSTGRES_HOST`, `POSTGRES_DB`, `POSTGRES_USER` (required);
13
+ `POSTGRES_PORT` (default 5432), `POSTGRES_PASSWORD` (default empty).
14
+
15
+ `from_env` tries a connection that no server answered (every address of the
16
+ host refused, had no route or sat on an unreachable network) up to 5 times,
17
+ 2 s apart, logging each retry: a Job pod's first connection can be refused
18
+ for a few seconds after the pod starts, until the cluster's NetworkPolicy
19
+ engine has admitted the pod. Every other failure, an authentication failure or
20
+ a connect timeout on any address among them, is raised at once.
21
+
22
+ The connection runs with `autocommit = True`; every method owns its
23
+ transaction explicitly. `load()` and `ensure_table()` open an explicit
24
+ `BEGIN` around their statements and `COMMIT`/`ROLLBACK` themselves, so a
25
+ mid-sequence failure leaves no partial change. `fetch()` executes a single
26
+ read and commits (or rolls back on error), never leaving an open transaction
27
+ dangling.
28
+
29
+ `PostgresAdapter` declares `drop_semantics = "cascade"` and is a
30
+ `DependentsIntrospection`: `dependents(schema, table)` lists every object a
31
+ `DROP TABLE ... CASCADE` of the table destroys or changes, with
32
+ `completeness="exact"`. It walks `pg_depend` from the table through every normal
33
+ dependency, so it lists views and materialized views (found through their
34
+ `_RETURN` rule, at any depth: a view of a view of the table is depth 2),
35
+ SQL-standard functions and procedures, foreign keys and policies on other
36
+ tables, rules, inheriting tables, tables with a column of the table's row type
37
+ or of an array of it, the types built on it (a domain, a range, a composite
38
+ type, a domain over an array of it) and what reads them, including a column of
39
+ the array or multirange type of such a type, a generated column over a dropped
40
+ column and what reads it, and what reads a partition of the table or a sequence
41
+ of it (owned or identity). A table that only loses a column is listed, and from
42
+ it only what reads that column is walked on.
43
+ `information_schema.view_table_usage` misses the materialized views, the
44
+ functions and every view past the first level. The table's own indexes,
45
+ constraints (a self-referencing foreign key included), policies, rules, column
46
+ defaults, sequences, partitions and triggers are parts of the table, and the
47
+ array and multirange types, constructor functions and casts a type internally
48
+ owns (the constructors of a range or multirange type and the range-to-multirange
49
+ cast) are parts of the type: none is listed. A cast a user created, such as
50
+ `CREATE CAST (seed AS text)`, is dropped by the cascade and is listed as `other`
51
+ (`cast from <schema>.seed to text`). Each object appears once, at its shallowest depth.
52
+ The walk follows at most 32 dependency levels; an object deeper than that is not
53
+ listed. The table is found case-insensitively, so a seed loaded under another
54
+ letter case never makes the walk skip it, and a schema holding two tables that
55
+ differ only by letter case lists the dependants of both; a missing table or
56
+ schema gives no objects. Each `kind` is one of `view`, `materialized_view`,
57
+ `table`, `function`, `procedure`, `constraint`, `policy`, `rule` or `other`.
58
+ `other` is anything else: a domain, a range type or another type (named
59
+ `type <name>`), a composite type (its own name), an index, a user cast, a column
60
+ default of another table (named `default value for column ...`) or a trigger.
61
+ A name that Postgres describes (a type, a cast, a column default) is
62
+ schema-qualified unless the object is visible on the connection's `search_path`
63
+ (a domain in `public` is named `type my_domain`). The
64
+ name of a function carries its argument types (`seed_count()`), and that of a
65
+ constraint, policy or rule the table or domain it sits on
66
+ (`orders_seed_id_fkey on orders`). A listed object that has no table of its own
67
+ (a type, a column default, a trigger) has an empty schema and an empty owner.
68
+ CI runs the dependents suite against the postgres:16 test stack; it was also
69
+ verified against a postgres:18.3 server by pointing the suite at it with
70
+ `VR_IT_PG_PORT` (host `localhost`, database `warehouse`, user and password
71
+ `continuo`).
72
+
73
+ The `config:` index vocabulary is the union of the two vocabularies this
74
+ adapter inherited from the merge: `columns`, `unique`, `type`, and `name`,
75
+ with the strictest check from each side applied. The derived index name is
76
+ built from the table and the column list only — `type` and `unique` are not
77
+ folded into it, so two index entries on the same columns that differ only in
78
+ `type` (or only in `unique`) collide on that derived name and are rejected as
79
+ a duplicate. Give one of them an explicit `name:` when that is genuinely
80
+ intended.
81
+
82
+ Verification tier: unit tests are mock-free pure-logic tests (type-grammar
83
+ validation, Arrow table construction from rows); DDL/transactional behavior
84
+ (schema/table creation, load truncate+insert+rollback, validation DDL) is
85
+ verified against a live postgres:16 engine in
86
+ `tests/test_integration_runtime_postgres.py` and
87
+ `tests/test_integration_postgres_validation.py`.
@@ -12,6 +12,8 @@ One class covers both roles Continuo asks of an engine:
12
12
  its ``PRIMARY KEY``); ``load`` atomically replaces its contents, ``append``
13
13
  adds rows and ``_upsert`` inserts or overwrites by key, all through one
14
14
  insert helper; ``table_columns`` reads its shape.
15
+ * **Drop impact** — ``dependents`` lists every object a drop of a table would
16
+ cascade to, from ``pg_depend``.
15
17
 
16
18
  The connection runs with ``autocommit = True``: every method owns its
17
19
  transaction explicitly rather than relying on psycopg2's implicit
@@ -28,8 +30,10 @@ import contextlib
28
30
  import hashlib
29
31
  import logging
30
32
  import os
33
+ import re
34
+ import time
31
35
 
32
- from collections.abc import Iterator, Sequence
36
+ from collections.abc import Callable, Iterator, Sequence
33
37
  from typing import Any
34
38
 
35
39
  import psycopg2 # type: ignore[import-untyped]
@@ -38,6 +42,9 @@ import pyarrow as pa # type: ignore[import-untyped]
38
42
  from continuo_engine_contract.config import ensure_known_keys # type: ignore[import-untyped]
39
43
  from continuo_engine_contract.port import ( # type: ignore[import-untyped]
40
44
  AppendCapable,
45
+ Dependent,
46
+ DependentsIntrospection,
47
+ Dependents,
41
48
  LiveColumn,
42
49
  UpsertCapable,
43
50
  WarehouseAdapter,
@@ -79,6 +86,42 @@ _DEFAULT_INDEX_TYPE = "btree"
79
86
  # postgres actually stores.
80
87
  _MAX_IDENTIFIER_BYTES = 63
81
88
 
89
+ # A Job pod's first connection can be refused for a few seconds after the pod
90
+ # starts, until the cluster's NetworkPolicy engine has admitted the pod: an
91
+ # engine that rejects (k3s's kube-router) refuses at once instead of dropping
92
+ # the packets. So a connection the server never answered, refused or
93
+ # unroutable, is tried again. A connect timeout is not: an engine that drops
94
+ # packets only delays the first connection, which TCP's own retransmissions
95
+ # carry through, so a timeout means the server stayed silent for the whole
96
+ # timeout and every retry would wait that long again.
97
+ _CONNECT_ATTEMPTS = 5
98
+ _CONNECT_RETRY_SECONDS = 2.0
99
+ # libpq writes one line per address it tried, the reason after "failed: "
100
+ # (libpq 14 and later) or after "could not connect to server: " (libpq 13 and
101
+ # earlier); pgcode is None for every connection failure, so the text is all
102
+ # there is.
103
+ _ADDRESS_FAILURE = re.compile(r"(?:failed|could not connect to server): ([^\n]*)")
104
+ _UNREACHABLE_REASONS = frozenset({"Connection refused", "No route to host", "Network is unreachable"})
105
+
106
+
107
+ def _is_unreachable(exc: "psycopg2.OperationalError") -> bool:
108
+ """Whether *exc* says that every address of the host refused or could not be routed to.
109
+
110
+ A host name can have several addresses, and libpq reports each one it
111
+ tried. One that timed out means a retry would wait out its timeout again;
112
+ a FATAL line came from a server that answered (a failed authentication, an
113
+ unknown database). libpq 13 and earlier print a timeout as a bare
114
+ "timeout expired" line, with no prefix.
115
+ """
116
+ message = str(exc)
117
+ reasons = [reason.strip() for reason in _ADDRESS_FAILURE.findall(message)]
118
+ return (
119
+ bool(reasons)
120
+ and all(reason in _UNREACHABLE_REASONS for reason in reasons)
121
+ and "FATAL:" not in message
122
+ and "timeout expired" not in message
123
+ )
124
+
82
125
 
83
126
  def _truncate_explicit_identifier(name: str) -> str:
84
127
  """Truncate an explicit, author-given identifier to at most 63 bytes.
@@ -405,9 +448,136 @@ def _arrow_table_from_rows(colnames: list[str], rows: list[tuple[Any, ...]]) ->
405
448
  return pa.table({name: pa.array(values) for name, values in zip(colnames, by_column)})
406
449
 
407
450
 
408
- class PostgresAdapter(WarehouseAdapter, AppendCapable, UpsertCapable):
451
+ # Every object that depends on a table, found by following pg_depend's normal
452
+ # dependencies (deptype 'n'), the ones DROP ... CASCADE follows. The walk's
453
+ # nodes are (class, object, subid): subid 0 is the whole object, a positive
454
+ # subid one column of a table or composite type. A column a cascade drops leaves
455
+ # its table in place, so from a column node only the dependencies on that column
456
+ # are followed, and the table itself is still listed. A view's dependency is
457
+ # recorded on its _RETURN rewrite rule, so the walk steps from the rule to the
458
+ # view it belongs to. The expression of a generated column (a pg_attrdef row) is
459
+ # dropped together with its column, so the walk steps from it to that column. A
460
+ # table that has a column of the walked table's row type, or of an array of it,
461
+ # depends on that type (pg_type), not on the table, so a whole-table step also
462
+ # follows the dependencies on its reltype and typarray. What a type the walk
463
+ # reaches (a domain, a range) internally owns (deptype 'i': its array type, its
464
+ # multirange type, its constructor functions and casts) is dropped with it, so a
465
+ # pg_type step also follows every internal dependency on it. A partition, a
466
+ # sequence owned by a column (deptype 'a'), an identity sequence and an
467
+ # internally owned object are parts of their owner, yet what reads them is
468
+ # dropped with it: the walk steps through them without adding a level, and lists
469
+ # none of them. The table's own indexes and triggers are automatic dependencies
470
+ # that nothing reads, so the walk does not follow them. The depth bound of 32
471
+ # keeps a dependency cycle from running forever, and ends a chain deeper than
472
+ # that unlisted.
473
+ _DEPENDENTS_SQL = """
474
+ WITH RECURSIVE walk(classid, objid, subid, depth) AS (
475
+ SELECT 'pg_class'::regclass::oid, c.oid, 0, 0
476
+ FROM pg_class c
477
+ JOIN pg_namespace n ON n.oid = c.relnamespace
478
+ WHERE lower(n.nspname) = lower(%(schema)s)
479
+ AND lower(c.relname) = lower(%(table)s)
480
+ AND c.relkind IN ('r', 'p')
481
+ UNION
482
+ SELECT CASE WHEN rw.oid IS NULL AND gad.oid IS NULL THEN dep.classid ELSE 'pg_class'::regclass::oid END,
483
+ CASE WHEN rw.oid IS NOT NULL THEN rw.ev_class
484
+ WHEN gad.oid IS NOT NULL THEN gad.adrelid
485
+ ELSE dep.objid END,
486
+ CASE WHEN rw.oid IS NOT NULL THEN 0
487
+ WHEN gad.oid IS NOT NULL THEN gad.adnum::int
488
+ ELSE dep.objsubid END,
489
+ w.depth + CASE WHEN dep.deptype IN ('a', 'i') THEN 0 ELSE 1 END
490
+ FROM walk w
491
+ LEFT JOIN pg_class wc
492
+ ON w.classid = 'pg_class'::regclass::oid AND w.subid = 0 AND wc.oid = w.objid
493
+ LEFT JOIN pg_type rowtype ON rowtype.oid = wc.reltype
494
+ JOIN pg_depend dep
495
+ ON (dep.refclassid = w.classid AND dep.refobjid = w.objid
496
+ AND (w.subid = 0 OR dep.refobjsubid = w.subid))
497
+ OR (dep.refclassid = 'pg_type'::regclass::oid
498
+ AND dep.refobjid IN (rowtype.oid, rowtype.typarray))
499
+ LEFT JOIN pg_class dc
500
+ ON dep.classid = 'pg_class'::regclass::oid AND dc.oid = dep.objid
501
+ LEFT JOIN pg_rewrite rw
502
+ ON dep.classid = 'pg_rewrite'::regclass::oid AND rw.oid = dep.objid AND rw.rulename = '_RETURN'
503
+ LEFT JOIN pg_attrdef gad
504
+ ON dep.classid = 'pg_attrdef'::regclass::oid AND gad.oid = dep.objid
505
+ AND EXISTS (SELECT 1 FROM pg_attribute ga
506
+ WHERE ga.attrelid = gad.adrelid AND ga.attnum = gad.adnum AND ga.attgenerated <> '')
507
+ WHERE w.depth < 32
508
+ AND (dep.deptype = 'n'
509
+ OR (dep.deptype IN ('a', 'i') AND dc.relkind = 'S')
510
+ OR (dep.deptype = 'a' AND dc.relkind IN ('r', 'p', 'f') AND dc.relispartition)
511
+ OR (dep.deptype = 'i' AND w.classid = 'pg_type'::regclass::oid AND dep.refclassid = w.classid))
512
+ AND (rw.oid IS NULL OR rw.ev_class <> w.objid)
513
+ )
514
+ SELECT found.catalog,
515
+ CASE found.catalog WHEN 'pg_class' THEN c.relkind::text WHEN 'pg_proc' THEN p.prokind::text ELSE '' END,
516
+ COALESCE(n.nspname, ''),
517
+ CASE found.catalog
518
+ WHEN 'pg_class' THEN c.relname
519
+ WHEN 'pg_proc' THEN p.proname || '(' || pg_get_function_identity_arguments(p.oid) || ')'
520
+ WHEN 'pg_constraint' THEN
521
+ CASE WHEN k.conrelid = 0 THEN k.conname || ' on ' || format_type(k.contypid, NULL)
522
+ ELSE k.conname || ' on ' || t.relname END
523
+ WHEN 'pg_policy' THEN pol.polname || ' on ' || t.relname
524
+ WHEN 'pg_rewrite' THEN rw.rulename || ' on ' || t.relname
525
+ ELSE pg_describe_object(found.classid, found.objid, 0)
526
+ END,
527
+ COALESCE(pg_get_userbyid(COALESCE(c.relowner, p.proowner, t.relowner)), ''),
528
+ found.depth
529
+ FROM (
530
+ SELECT classid, objid, classid::regclass::text AS catalog, min(depth) AS depth
531
+ FROM walk
532
+ GROUP BY classid, objid
533
+ HAVING min(depth) > 0
534
+ ) found
535
+ LEFT JOIN pg_class c ON found.catalog = 'pg_class' AND c.oid = found.objid
536
+ LEFT JOIN pg_proc p ON found.catalog = 'pg_proc' AND p.oid = found.objid
537
+ LEFT JOIN pg_constraint k ON found.catalog = 'pg_constraint' AND k.oid = found.objid
538
+ LEFT JOIN pg_policy pol ON found.catalog = 'pg_policy' AND pol.oid = found.objid
539
+ LEFT JOIN pg_rewrite rw ON found.catalog = 'pg_rewrite' AND rw.oid = found.objid
540
+ LEFT JOIN pg_attrdef ad ON found.catalog = 'pg_attrdef' AND ad.oid = found.objid
541
+ LEFT JOIN pg_trigger tg ON found.catalog = 'pg_trigger' AND tg.oid = found.objid
542
+ LEFT JOIN pg_class t ON t.oid = COALESCE(k.conrelid, pol.polrelid, rw.ev_class)
543
+ LEFT JOIN pg_namespace n ON n.oid = COALESCE(c.relnamespace, p.pronamespace, t.relnamespace)
544
+ -- a sequence, a partition and the per-partition copy of a foreign key are parts of their owner
545
+ WHERE NOT (found.catalog = 'pg_class'
546
+ AND (c.relkind = 'S' OR (c.relkind IN ('r', 'p', 'f') AND c.relispartition)))
547
+ -- so is whatever a type internally owns (its array and multirange types, its constructor functions and
548
+ -- casts), except a composite type's own pg_class row
549
+ AND NOT (found.catalog <> 'pg_class'
550
+ AND EXISTS (SELECT 1 FROM pg_depend own
551
+ WHERE own.classid = found.classid AND own.objid = found.objid
552
+ AND own.deptype = 'i' AND own.refclassid = 'pg_type'::regclass::oid))
553
+ AND NOT (found.catalog = 'pg_constraint' AND k.conparentid <> 0)
554
+ -- a constraint, default, policy, rule or trigger of the table itself is part of the table
555
+ AND NOT COALESCE(COALESCE(k.conrelid, ad.adrelid, pol.polrelid, rw.ev_class, tg.tgrelid) IN (
556
+ SELECT objid FROM walk WHERE depth = 0 AND classid = 'pg_class'::regclass::oid), false)
557
+ ORDER BY found.depth, 3, 4
558
+ """
559
+
560
+
561
+ def _dependent_kind(catalog: str, subkind: str) -> str:
562
+ """The contract's kind for a catalog row: *subkind* is ``relkind`` for ``pg_class``, ``prokind`` for ``pg_proc``.
563
+
564
+ A ``pg_class`` row that is no table, view or materialized view (a composite
565
+ type, an index) is ``other``.
566
+ """
567
+ if catalog == "pg_class":
568
+ return {"v": "view", "m": "materialized_view", "r": "table", "p": "table", "f": "table"}.get(
569
+ subkind, "other"
570
+ )
571
+ if catalog == "pg_proc":
572
+ return "procedure" if subkind == "p" else "function"
573
+ return {"pg_constraint": "constraint", "pg_policy": "policy", "pg_rewrite": "rule"}.get(catalog, "other")
574
+
575
+
576
+ class PostgresAdapter(WarehouseAdapter, AppendCapable, UpsertCapable, DependentsIntrospection):
409
577
  """WarehouseAdapter speaking postgres over a psycopg2 connection."""
410
578
 
579
+ drop_semantics = "cascade"
580
+
411
581
  def __init__(self, conn: "psycopg2.extensions.connection") -> None:
412
582
  self._conn = conn
413
583
  self._conn.autocommit = True
@@ -418,16 +588,35 @@ class PostgresAdapter(WarehouseAdapter, AppendCapable, UpsertCapable):
418
588
  return ["POSTGRES_HOST", "POSTGRES_DB", "POSTGRES_USER"]
419
589
 
420
590
  @classmethod
421
- def from_env(cls) -> "PostgresAdapter":
422
- """Connect from POSTGRES_* env (port defaults 5432, password empty)."""
423
- conn = psycopg2.connect(
424
- host=os.environ["POSTGRES_HOST"],
425
- port=os.environ.get("POSTGRES_PORT", "5432"),
426
- dbname=os.environ["POSTGRES_DB"],
427
- user=os.environ["POSTGRES_USER"],
428
- password=os.environ.get("POSTGRES_PASSWORD", ""),
429
- )
430
- return cls(conn)
591
+ def from_env(cls, *, sleep: Callable[[float], None] = time.sleep) -> "PostgresAdapter":
592
+ """Connect from POSTGRES_* env (port defaults 5432, password empty).
593
+
594
+ A connection no server answered (refused, no route, network
595
+ unreachable) is tried again, up to ``_CONNECT_ATTEMPTS`` times in all,
596
+ ``_CONNECT_RETRY_SECONDS`` apart, each retry logged; the last failure
597
+ is raised. Every other failure is raised at once.
598
+ """
599
+ params = {
600
+ "host": os.environ["POSTGRES_HOST"],
601
+ "port": os.environ.get("POSTGRES_PORT", "5432"),
602
+ "dbname": os.environ["POSTGRES_DB"],
603
+ "user": os.environ["POSTGRES_USER"],
604
+ "password": os.environ.get("POSTGRES_PASSWORD", ""),
605
+ }
606
+ for attempt in range(1, _CONNECT_ATTEMPTS + 1):
607
+ try:
608
+ conn = psycopg2.connect(**params)
609
+ except psycopg2.OperationalError as exc:
610
+ if attempt == _CONNECT_ATTEMPTS or not _is_unreachable(exc):
611
+ raise
612
+ logger.warning(
613
+ "postgres not reachable yet (attempt %d of %d), retrying in %.0f s: %s",
614
+ attempt, _CONNECT_ATTEMPTS, _CONNECT_RETRY_SECONDS, str(exc).strip(),
615
+ )
616
+ sleep(_CONNECT_RETRY_SECONDS)
617
+ continue
618
+ return cls(conn)
619
+ raise AssertionError("unreachable") # pragma: no cover
431
620
 
432
621
  # --- Schema lifecycle ---------------------------------------------------
433
622
 
@@ -659,6 +848,43 @@ class PostgresAdapter(WarehouseAdapter, AppendCapable, UpsertCapable):
659
848
  raise LookupError(f"table {schema}.{table} does not exist")
660
849
  return [LiveColumn(name=name, nullable=is_nullable == "YES") for name, is_nullable in rows]
661
850
 
851
+ def dependents(self, schema: str, table: str) -> Dependents:
852
+ """Every object a ``DROP TABLE schema.table CASCADE`` destroys or changes, exactly.
853
+
854
+ Lists views, materialized views, SQL-standard functions and procedures,
855
+ foreign keys and policies on other tables, rules, inheriting tables,
856
+ tables with a column of this table's row type, or of an array of it,
857
+ and the types built on it (a domain, a range, a composite type, and a
858
+ domain over an array of it), each once at its shallowest depth, with
859
+ what reads them: a column of an array or multirange type of such a
860
+ type, a generated column over a dropped column, and what reads a
861
+ partition of the table or a sequence of it (owned or identity). A table
862
+ that only loses a column is listed, and only what reads that column is
863
+ walked on from it. The table's own indexes, constraints (a
864
+ self-referencing foreign key included), policies, rules, column
865
+ defaults, sequences, partitions and triggers, and what a type owns
866
+ internally (its array and multirange types, constructor functions and
867
+ casts), are parts of their owner and are not listed.
868
+
869
+ The walk follows at most 32 dependency levels: an object deeper than
870
+ that is not listed.
871
+
872
+ The table is found case-insensitively, so a seed loaded under another
873
+ letter case still has its dependants listed, and the walk never
874
+ under-reports; a schema holding two tables that differ only by case
875
+ lists the dependants of both. A missing table or schema gives no objects.
876
+ """
877
+ with self._conn.cursor() as cur:
878
+ cur.execute(_DEPENDENTS_SQL, {"schema": schema, "table": table})
879
+ rows = cur.fetchall()
880
+ return Dependents(
881
+ objects=tuple(
882
+ Dependent(dep_schema, name, _dependent_kind(catalog, subkind), owner or "", depth)
883
+ for catalog, subkind, dep_schema, name, owner, depth in rows
884
+ ),
885
+ completeness="exact",
886
+ )
887
+
662
888
  @classmethod
663
889
  def validate_config(
664
890
  cls, config: dict[str, Any] | None, column_names: list[str]
@@ -1,5 +1,5 @@
1
1
  [project]
2
- version = "0.3.1"
2
+ version = "0.4.0"
3
3
  name = "continuo-postgres-adapter"
4
4
  description = "Postgres warehouse adapter for Continuo: validation and python-node runtime."
5
5
  authors = [{ name = "Simone Carolini" }]
@@ -12,9 +12,9 @@ dependencies = [
12
12
  # of this adapter would otherwise accept whatever contract release is
13
13
  # newest. This adapter subclasses the contract's WarehouseAdapter port
14
14
  # directly, so the pair must stay the tested one.
15
- "continuo-engine-contract==0.8.1",
16
- "psycopg2-binary==2.9.12",
17
- "pyarrow==25.0.1",
15
+ "continuo-engine-contract==0.10.0",
16
+ "psycopg2-binary==2.9.13",
17
+ "pyarrow==26.0.0",
18
18
  ]
19
19
  classifiers = [
20
20
  "Development Status :: 4 - Beta",
@@ -27,8 +27,8 @@ postgres = "continuo_postgres_adapter.adapter:PostgresAdapter"
27
27
 
28
28
  [dependency-groups]
29
29
  dev = [
30
- "ruff==0.16.5",
31
- "mypy==2.3.1",
30
+ "ruff==0.17.1",
31
+ "mypy==2.4.0",
32
32
  ]
33
33
  test = [
34
34
  "pytest==9.1.1",