feather-cdc-postgres 0.1.0__py3-none-any.whl

feather_cdc_postgres/__init__.py

@@ -0,0 +1,6 @@
+"""feather_cdc_postgres — declarative PostgreSQL → MotherDuck CDC runtime (#3079).
+
+One code artifact, one isolated service instance per registration (`<source_id>.yml`). The
+registrations are the DEPLOYER's, not this package's. Mechanism: slot-based logical replication
+(pgoutput) drained cron-shaped into a bronze database, source namespaces preserved.
+"""

feather_cdc_postgres/__main__.py

@@ -0,0 +1,9 @@
+"""`python -m feather_cdc_postgres` entry point."""
+from __future__ import annotations
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

feather_cdc_postgres/adapters/__init__.py

@@ -0,0 +1,44 @@
+"""Discovery adapters: source-specific POLICY that narrows the generic set of relations.
+
+An adapter may only answer "which of the source's relations are in scope, and what does the
+source call them" — it never supplies types, keys, or load behaviour. That keeps the CDC
+mechanism single-implementation (#3079 D4); a fork of the mechanism per source is the failure
+this design exists to prevent.
+"""
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, NamedTuple
+
+
+# How an adapter came to offer a relation. `user_created` = the source's own users declared
+# it (always application data); `named_system` = one of the source's platform tables the
+# manifest names explicitly. None = the adapter applied no such distinction.
+USER_CREATED = "user_created"
+NAMED_SYSTEM = "named_system"
+
+
+class AdapterRelation(NamedTuple):
+    """(schema, table, module) — plus a description the source itself owns, when it has one,
+    and the `origin` the adapter selected it under (see USER_CREATED / NAMED_SYSTEM)."""
+
+    schema: str
+    table: str
+    module: str | None = None
+    description: str | None = None
+    origin: str | None = None
+
+
+AdapterFn = Callable[[str, Dict[str, Any]], List[AdapterRelation]]
+
+
+def get(name: str) -> AdapterFn:
+    """Resolve an adapter by manifest name. Unknown names fail loudly at validate time."""
+    if name == "featherbase":
+        from .featherbase import select as featherbase_select
+
+        return featherbase_select
+    raise KeyError(name)
+
+
+def known() -> tuple:
+    return ("featherbase",)

feather_cdc_postgres/adapters/featherbase.py

@@ -0,0 +1,126 @@
+"""Featherbase catalog adapter — `table_def` decides which relations exist (#2209, #3079 D4/D-13).
+
+Constraints (each one is a live behaviour of Featherbase, not a preference):
+  - platform physical name = lower(Table name) with spaces → underscores
+    ('Access Log' → access_log) in the catalog schema;
+  - current app Tables name their physical schema/relation in the catalog;
+  - only kind in (table, sub_table) — settings Tables are EAV rows, not relations;
+  - data_source IS NULL — a row with one is a reflection of an external source, not ours;
+  - a declared Table with no physical relation yet is SKIPPED, never an error;
+  - `options.system_tables` given ⇒ two-way rule: `table_def."system" = true` rows only when
+    named there, `false` rows (user-created) always; absent ⇒ everything, catalog included.
+"""
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+import psycopg
+
+from . import NAMED_SYSTEM, USER_CREATED, AdapterRelation
+
+# The catalog itself. Descriptions match the retired scripts/generate_featherbase_sources.py, which this
+# adapter supersedes — the generated dbt file must not churn on the switch.
+CATALOG_TABLES: Dict[str, str] = {
+    "table_def": "Featherbase catalog — one row per declared Table (the generation input for this file).",
+    "column_def": "Featherbase catalog — one row per declared Column.",
+}
+
+
+def select(dsn: str, options: Dict[str, Any] | None = None) -> List[AdapterRelation]:
+    """Relations Featherbase's own catalog declares. `options.modules` is an allow-list of
+    module names; null/absent means every module. `options.schema` defaults to public.
+    `options.system_tables` (list of physical names) switches on the two-way rule: a platform
+    table (`"system" = true`) is offered ONLY when named there — `table_def`/`column_def`
+    included — and a user-created table (`"system" = false`) is always offered."""
+    options = options or {}
+    schema = str(options.get("schema") or "public")
+    raw_modules = options.get("modules")
+    modules = None if raw_modules is None else {str(m) for m in raw_modules}
+    raw_system = options.get("system_tables")
+    system_tables = None if raw_system is None else {str(t) for t in raw_system}
+
+    with psycopg.connect(dsn, connect_timeout=10) as conn:
+        conn.read_only = True
+        cols = {
+            r[0]
+            for r in conn.execute(
+                "select column_name from information_schema.columns "
+                "where table_schema = %s and table_name = 'table_def'",
+                (schema,),
+            ).fetchall()
+        }
+        if not cols:
+            raise RuntimeError(f"{schema}.table_def not found — is this a Featherbase database?")
+        if system_tables is not None and "system" not in cols:
+            # Without the flag every row would read as user-created and the allow-list would
+            # admit every platform table — the opposite of what the option promises.
+            raise RuntimeError(
+                f"{schema}.table_def has no \"system\" column — options.system_tables needs it"
+            )
+        physical_cols = {"physical_schema", "physical_relation"}
+        if physical_cols & cols and not physical_cols <= cols:
+            raise RuntimeError(
+                f"{schema}.table_def needs both physical_schema and physical_relation"
+            )
+        has_physical = physical_cols <= cols
+        desc = '"description"' if "description" in cols else "null"
+        system = '"system"' if "system" in cols else "false"
+        physical_schema = "physical_schema" if has_physical else "null"
+        physical_relation = "physical_relation" if has_physical else "null"
+        rows = conn.execute(
+            f'select "name", module, data_source, kind, {desc}, {system}, '
+            f'{physical_schema}, {physical_relation} from "{schema}".table_def'
+        ).fetchall()
+        physical = {
+            (r[0], r[1])
+            for r in conn.execute(
+                "select schemaname, tablename from pg_tables"
+            ).fetchall()
+        }
+
+    out: Dict[tuple[str, str], AdapterRelation] = {}
+    for name, module, data_source, kind, description, is_system, physical_schema, physical_relation in rows:
+        derived_table = str(name).lower().replace(" ", "_")
+        if kind not in ("table", "sub_table"):
+            continue
+        if data_source is not None:
+            continue
+        if derived_table in CATALOG_TABLES:
+            continue  # the catalog's own rows are handled below
+        if modules is not None and (module or "") not in modules:
+            continue
+        relation_schema = schema
+        relation_table = derived_table
+        if has_physical and not bool(is_system):
+            if physical_schema and physical_relation:
+                relation_schema = str(physical_schema)
+                relation_table = str(physical_relation)
+            elif "." in str(name):
+                # #4113: Featherbase's tableRelation() — a qualified (app-owned) Table is never
+                # located by its spelling; a plain one lives at the platform default above.
+                raise RuntimeError(
+                    f"{schema}.table_def row {name!r} has no "
+                    "physical_schema/physical_relation"
+                )
+        if (relation_schema, relation_table) not in physical:
+            continue  # declared but not materialised yet
+        origin = None
+        if system_tables is not None:
+            if bool(is_system) and relation_table not in system_tables:
+                continue
+            origin = NAMED_SYSTEM if bool(is_system) else USER_CREATED
+        out[(relation_schema, relation_table)] = AdapterRelation(
+            relation_schema, relation_table, module, description or None, origin
+        )
+    for tbl, description in CATALOG_TABLES.items():
+        if (schema, tbl) not in physical:
+            continue
+        if system_tables is None:
+            out[(schema, tbl)] = AdapterRelation(
+                schema, tbl, None, description
+            )  # unconditional (legacy)
+        elif tbl in system_tables:
+            out[(schema, tbl)] = AdapterRelation(
+                schema, tbl, None, description, NAMED_SYSTEM
+            )
+    return [out[k] for k in sorted(out)]

feather_cdc_postgres/cli.py

@@ -0,0 +1,247 @@
+"""`python -m feather_cdc_postgres <command> <manifest>` (#3079 D1).
+
+Constraints:
+  - `validate` and `plan` MUTATE NOTHING — they open read-only connections and print. Anything
+    that would change either database belongs to `run`.
+  - no environment VALUE is ever printed; only the NAME of the variable is echoed.
+  - `run`/`status`/`resnapshot` import the runtime lazily, so the compiler half stays usable
+    (and testable) without dlt credentials or a destination.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import pathlib
+import sys
+from typing import Dict, List, Optional, Tuple
+
+from . import discovery, generate_dbt, manifest as mf
+from .adapters import AdapterRelation, get as get_adapter, known as known_adapters
+from .selectors import CompiledPlan, compile as compile_selection
+
+def _env(name: str) -> Optional[str]:
+    val = os.environ.get(name)
+    return val or None
+
+
+def _resolve(m: mf.Manifest) -> Tuple[Optional[str], List[mf.Problem]]:
+    """The source DSN, or a problem naming the variable that is unset — never its value."""
+    dsn = _env(m.source.dsn_env)
+    if not dsn:
+        return None, [mf.Problem("env_not_set", "error", f"environment variable {m.source.dsn_env} is not set")]
+    return dsn, []
+
+
+def _adapter_relations(m: mf.Manifest, dsn: str) -> Tuple[List[AdapterRelation], List[mf.Problem]]:
+    if m.source.discovery is None:
+        return [], []
+    try:
+        fn = get_adapter(m.source.discovery.adapter)
+    except KeyError:
+        return [], [
+            mf.Problem(
+                "adapter_unknown",
+                "error",
+                f"discovery adapter {m.source.discovery.adapter!r} is unknown (known: {', '.join(known_adapters())})",
+            )
+        ]
+    return list(fn(dsn, m.source.discovery.options)), []
+
+
+def _compile(m: mf.Manifest, *, with_publication: bool) -> Tuple[Optional[CompiledPlan], List[mf.Problem], Dict]:
+    problems = list(mf.validate_static(m))
+    dsn, dsn_problems = _resolve(m)
+    problems += dsn_problems
+    if dsn is None:
+        return None, problems, {}
+    relations = discovery.discover(dsn)
+    adapter_rels, adapter_problems = _adapter_relations(m, dsn)
+    problems += adapter_problems
+    published = discovery.publication_relations(dsn, m.replication.publication) if with_publication else None
+    plan = compile_selection(m, relations, adapter_rels, published)
+    extra: Dict = {}
+    if with_publication:
+        extra["publication_exists"] = published is not None
+        extra["slot"] = discovery.slot_state(dsn, m.replication.slot)
+    return plan, problems + list(plan.problems), extra
+
+
+def _emit(payload: Dict, lines: List[str], as_json: bool) -> None:
+    text = json.dumps(payload, indent=2, default=str) if as_json else "\n".join(lines)
+    print(mf.redact(text))
+
+
+def _problem_lines(problems: List[mf.Problem]) -> List[str]:
+    if not problems:
+        return ["problems: none"]
+    return ["problems:"] + [f"  {p}" for p in problems]
+
+
+def _relation_lines(plan: CompiledPlan) -> List[str]:
+    width = max((len(r.qualified) for r in plan.relations), default=10)
+    out = [f"relations ({len(plan.relations)}):"]
+    for r in plan.relations:
+        key = ",".join(r.key) or "-"
+        idx = f" via {r.key_index}" if r.key_index else ""
+        origin = f"  origin={r.origin}" if r.origin else ""
+        # The column projection (#4346): these never leave the CDC process.
+        deny = f"  deny={','.join(r.deny_columns)}" if r.deny_columns else ""
+        out.append(
+            f"  {r.qualified:<{width}}  -> {r.target:<{width}}  {r.key_policy:<12} key={key}{idx}  "
+            f"class={r.table_class}{origin}{deny}"
+        )
+    return out
+
+
+def cmd_validate(args) -> int:
+    m = mf.load(args.manifest)
+    plan, problems, _ = _compile(m, with_publication=False)
+    lines = [f"manifest: {args.manifest}", f"source_id: {m.source_id}", f"dsn from: ${m.source.dsn_env}"]
+    payload: Dict = {"source_id": m.source_id, "problems": [p.__dict__ for p in problems]}
+    if plan is not None:
+        lines += _relation_lines(plan) + [f"selection_hash: {plan.selection_hash}"]
+        payload["relations"] = [r.__dict__ for r in plan.relations]
+        payload["selection_hash"] = plan.selection_hash
+    lines += _problem_lines(problems)
+    _emit(payload, lines, args.json)
+    return 1 if mf.errors(problems) else 0
+
+
+def cmd_plan(args) -> int:
+    m = mf.load(args.manifest)
+    plan, problems, extra = _compile(m, with_publication=True)
+    lines = [f"manifest: {args.manifest}", f"source_id: {m.source_id}", "PLAN — nothing is mutated by this command"]
+    payload: Dict = {"source_id": m.source_id, "problems": [p.__dict__ for p in problems], **extra}
+    if plan is not None:
+        lines += _relation_lines(plan)
+        lines.append(f"publication {m.replication.publication}: " + ("exists" if extra.get("publication_exists") else "MISSING — would be created"))
+        lines.append(f"  add ({len(plan.publication_add)}): " + (", ".join(f"{s}.{t}" for s, t in plan.publication_add) or "-"))
+        lines.append(f"  drop ({len(plan.publication_drop)}): " + (", ".join(f"{s}.{t}" for s, t in plan.publication_drop) or "-"))
+        lines.append(
+            f"replica identity ALTERs ({len(plan.replica_identity_alters)}): "
+            + (", ".join(f"{s}.{t}" for s, t in plan.replica_identity_alters) or "-")
+        )
+        lines.append("denied hits: " + (", ".join(plan.denied_hits) or "-"))
+        lines.append(f"selection_hash: {plan.selection_hash}")
+        slot = extra.get("slot") or {}
+        lines.append(
+            f"slot {m.replication.slot}: "
+            + ("absent — would be created" if not slot.get("exists") else
+               f"wal_status={slot.get('wal_status')} lag_bytes={slot.get('lag_bytes')} active={slot.get('active')}")
+            + f" · server max_slot_wal_keep_size={slot.get('max_slot_wal_keep_size_bytes') or 'unlimited'}"
+        )
+        payload.update(
+            relations=[r.__dict__ for r in plan.relations],
+            publication_add=plan.publication_add,
+            publication_drop=plan.publication_drop,
+            replica_identity_alters=plan.replica_identity_alters,
+            denied_hits=plan.denied_hits,
+            selection_hash=plan.selection_hash,
+        )
+    lines += _problem_lines(problems)
+    _emit(payload, lines, args.json)
+    return 1 if mf.errors(problems) else 0
+
+
+def cmd_generate_dbt(args) -> int:
+    m = mf.load(args.manifest)
+    plan, problems, _ = _compile(m, with_publication=False)
+    if plan is None or mf.errors(problems):
+        _emit({"problems": [p.__dict__ for p in problems]}, _problem_lines(problems), args.json)
+        return 1
+    out = pathlib.Path(args.out)
+    generate_dbt.write(m, plan, out)
+    _emit(
+        {"out": str(out), "tables": len(plan.relations)},
+        [f"wrote {out} — {len(plan.relations)} tables + _dlt_loads"],
+        args.json,
+    )
+    return 0
+
+
+def _runner():
+    """The runtime half (#3079 Phase 2). Contract expected here:
+
+        runner.run(manifest: Manifest, *, dev: bool, manifest_path: str) -> int
+        runner.status(manifest: Manifest) -> int
+        runner.resnapshot(manifest: Manifest, *, tables, yes: bool, dev: bool) -> int
+    """
+    from . import runner  # noqa: PLC0415  # TODO(runner): owned by the runtime half
+
+    return runner
+
+
+def cmd_run(args) -> int:
+    return int(_runner().run(mf.load(args.manifest), dev=args.dev, manifest_path=args.manifest) or 0)
+
+
+def cmd_status(args) -> int:
+    return int(_runner().status(mf.load(args.manifest)) or 0)
+
+
+def cmd_resnapshot(args) -> int:
+    return int(_runner().resnapshot(mf.load(args.manifest), tables=args.table or None,
+                                    yes=args.yes, dev=args.dev) or 0)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="feather_cdc_postgres", description="Declarative PostgreSQL → MotherDuck CDC runtime")
+    p.add_argument("--json", action="store_true", help="machine-readable output")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    for name, fn, helptext in (
+        ("validate", cmd_validate, "static + read-only checks; prints the compiled relation set"),
+        ("plan", cmd_plan, "validate + every change a run would make; mutates nothing"),
+    ):
+        s = sub.add_parser(name, help=helptext)
+        s.add_argument("manifest")
+        s.set_defaults(func=fn)
+
+    s = sub.add_parser("generate-dbt", help="write dbt source declarations for this registration")
+    s.add_argument("manifest")
+    # REQUIRED, and it has no default on purpose: the default used to be
+    # `<repo root>/dbt_runner/dbt/models/<source_id>/...`, resolved four parents up from this
+    # file. That is one warehouse's directory layout, and a runtime installed from a wheel sits
+    # in site-packages, where four parents up is somebody's virtualenv. The caller owns the path.
+    s.add_argument("--out", required=True, help="output path for the generated sources YAML")
+    s.set_defaults(func=cmd_generate_dbt)
+
+    s = sub.add_parser("run", help="reconcile, snapshot and drain one tick")
+    s.add_argument("manifest")
+    s.add_argument("--dev", action="store_true", help="local destination, no control-plane writes")
+    s.set_defaults(func=cmd_run)
+
+    s = sub.add_parser("status", help="lifecycle, checkpoint and slot health")
+    s.add_argument("manifest")
+    s.set_defaults(func=cmd_status)
+
+    s = sub.add_parser("resnapshot", help="re-baseline selected tables")
+    s.add_argument("manifest")
+    # `--tables` is the spelling the drift remedy text prints; both names, one destination.
+    s.add_argument("--table", "--tables", dest="table", action="append",
+                   help="schema.table (repeatable); default every relation")
+    s.add_argument("--yes", action="store_true", help="skip the confirmation prompt")
+    s.add_argument("--dev", action="store_true", help="local destination, no control-plane writes")
+    s.set_defaults(func=cmd_resnapshot)
+    return p
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        return args.func(args)
+    except mf.ManifestError as exc:
+        print(f"[error] {mf.redact(str(exc))}", file=sys.stderr)
+        return 2
+    except Exception as exc:  # noqa: BLE001
+        # Anything else — an unreachable source, a bad adapter, a duckdb error. A raw traceback
+        # would print psycopg's `connection to server at ...` line (and dlt's config exceptions
+        # echo the value they resolved) straight into Railway's log; redact() is the only thing
+        # standing between that and a DSN in a log line. SystemExit/KeyboardInterrupt are not
+        # Exception, so `sys.exit("FATAL: …")` from the runtime still propagates untouched.
+        message = mf.redact(f"{type(exc).__name__}: {exc}")
+        print(f"[error] {message}", file=sys.stderr)
+        if getattr(args, "json", False):
+            print(json.dumps({"error": type(exc).__name__, "message": message}, indent=2))
+        return 1