feather-cdc-postgres 0.1.0__py3-none-any.whl

feather_cdc_postgres/schema_registry.py

@@ -0,0 +1,283 @@
+"""Source type → destination type, and drift classification (#3079 D7).
+
+Constraints:
+  - an unknown OID becomes text but NEVER silently: every fallback emits a Problem(warning)
+    naming schema.table.column and the OID.
+  - the integer family compiles to plain BIGINT on both the snapshot and the CDC path — dlt
+    renders precision 32 as INTEGER, and one live table already carries both spellings.
+  - drift kinds are a closed set; `severity == "breaking"` is what the runtime pauses on.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+from typing import Dict, List, Optional, Sequence, Tuple
+
+from pg_replication.schema_types import _PG_TYPES, _to_dlt_column_type
+
+from .discovery import SourceRelation
+from .manifest import Problem
+
+# How faithfully the destination type carries the source value.
+EXACT = "exact"
+WIDENED = "widened"  # a strictly larger destination type; no value is lost
+EXACT_AS_TEXT = "exact_as_text"  # round-trips through its text rendering (uuid)
+LOSSY_TEXT = "lossy_text"  # text rendering only; arithmetic/ordering is not preserved
+TEXT_FALLBACK = "text_fallback"  # no declared mapping — reported, never silent
+
+_MAPPING: Dict[int, str] = {
+    16: EXACT, 17: EXACT, 20: EXACT, 21: WIDENED, 23: WIDENED, 25: EXACT, 26: WIDENED,
+    114: EXACT, 700: WIDENED, 701: EXACT, 1042: EXACT, 1043: EXACT, 1082: EXACT,
+    1083: EXACT, 1114: EXACT, 1184: EXACT, 1700: EXACT, 3802: EXACT,
+    19: EXACT_AS_TEXT, 2950: EXACT_AS_TEXT,
+    1186: LOSSY_TEXT, 1266: LOSSY_TEXT,
+}
+
+
+@dataclass(frozen=True)
+class CompiledType:
+    dest_type: str  # dlt data_type: bigint | text | double | decimal | date | time | timestamp | json | bool | binary
+    precision: Optional[int] = None
+    scale: Optional[int] = None
+    mapping: str = EXACT
+    timezone: Optional[bool] = None
+
+
+@dataclass(frozen=True)
+class CompiledColumn:
+    name: str
+    type_oid: int
+    typmod: int
+    not_null: bool
+    key_role: str  # key | none
+    compiled: CompiledType
+
+
+@dataclass(frozen=True)
+class Drift:
+    kind: str
+    column: Optional[str]
+    detail: str
+    severity: str  # additive | warning | breaking | info
+
+
+def compile_column(type_oid: int, typmod: int, not_null: bool = False, is_array: bool = False) -> CompiledType:
+    """One source column's destination type. `not_null` is carried for the caller's schema hint;
+    it never changes the type (a NOT NULL column still lands nullable after a key-only delete)."""
+    if is_array:
+        # pgoutput ships arrays as their text rendering ('{a,b}'); nothing downstream parses it.
+        return CompiledType(dest_type="text", mapping=TEXT_FALLBACK)
+    dlt_type = _to_dlt_column_type(type_oid, typmod)
+    mapping = _MAPPING.get(type_oid, TEXT_FALLBACK)
+    return CompiledType(
+        dest_type=str(dlt_type["data_type"]),
+        precision=dlt_type.get("precision"),
+        scale=dlt_type.get("scale"),
+        mapping=mapping,
+        timezone=dlt_type.get("timezone"),
+    )
+
+
+def compile_relation(
+    relation: SourceRelation, key: Tuple[str, ...] = ()
+) -> Tuple[List[CompiledColumn], List[Problem]]:
+    """Compile every column of one relation; surface each fallback as a warning."""
+    cols: List[CompiledColumn] = []
+    problems: List[Problem] = []
+    for c in relation.columns:
+        ct = compile_column(c.type_oid, c.typmod, c.not_null, c.is_array)
+        if ct.mapping == TEXT_FALLBACK:
+            reason = "array type" if c.is_array else f"OID {c.type_oid} has no declared mapping"
+            problems.append(
+                Problem(
+                    "type_fallback",
+                    "warning",
+                    f"{relation.schema}.{relation.table}.{c.name} ({c.type_name}): {reason} — "
+                    "landing as text; add it to schema_registry._MAPPING to make the choice explicit",
+                )
+            )
+        cols.append(
+            CompiledColumn(
+                name=c.name,
+                type_oid=c.type_oid,
+                typmod=c.typmod,
+                not_null=c.not_null,
+                key_role="key" if c.name in key else "none",
+                compiled=ct,
+            )
+        )
+    return cols, problems
+
+
+def known_oids() -> Tuple[int, ...]:
+    return tuple(sorted(_PG_TYPES))
+
+
+# --------------------------------------------------------------------------- signatures
+
+
+def project(relation: SourceRelation, exclude: Sequence[str] = ()) -> SourceRelation:
+    """The relation as the destination sees it: denied columns (#4346) removed. Every
+    registry row, signature and drift diff runs on this, so a deny is never a `drop_column`."""
+    if not exclude:
+        return relation
+    denied = set(exclude)
+    return replace(relation, columns=[c for c in relation.columns if c.name not in denied])
+
+
+def project_signature(sig: dict, exclude: Sequence[str] = ()) -> dict:
+    """A STORED signature projected the same way — one written before the deny existed still
+    names the column, and diffing it raw would report the deny as a breaking drop."""
+    if not exclude or not sig:
+        return sig
+    denied = set(exclude)
+    return {**sig, "columns": {k: v for k, v in sig.get("columns", {}).items() if k not in denied}}
+
+
+def signature(relation: SourceRelation, key: Tuple[str, ...] = (),
+              exclude: Sequence[str] = ()) -> dict:
+    """The inspectable shape a drift comparison runs on. Plain dicts so it can be stored in
+    `_cdc_runtime.schema_registry` and diffed across runs without unpickling anything."""
+    relation = project(relation, exclude)
+    cols, _ = compile_relation(relation, key)
+    return {
+        "schema": relation.schema,
+        "table": relation.table,
+        "key": list(key),
+        "replica_identity": relation.replica_identity,
+        # Not compared by classify_drift: it keys the reviewed shape to the relation's OID, so
+        # the decoder's column guard (#4346) follows a table rename.
+        "oid": relation.oid,
+        "columns": {
+            c.name: {
+                "type_oid": c.type_oid,
+                "typmod": c.typmod,
+                "not_null": c.not_null,
+                "dest_type": c.compiled.dest_type,
+                "precision": c.compiled.precision,
+                "scale": c.compiled.scale,
+                "mapping": c.compiled.mapping,
+            }
+            for c in cols
+        },
+    }
+
+
+_INT_RANK = {21: 1, 23: 2, 20: 3}
+
+
+def _varchar_len(oid: int, typmod: int) -> Optional[int]:
+    if oid in (1043, 1042) and typmod != -1:
+        return typmod - 4
+    return None
+
+
+def _numeric_ps(typmod: int) -> Tuple[Optional[int], Optional[int]]:
+    if typmod == -1:
+        return None, None
+    return ((typmod - 4) >> 16) & 65535, (typmod - 4) & 65535
+
+
+def _is_widen(old: dict, new: dict) -> bool:
+    o, n = old["type_oid"], new["type_oid"]
+    if o == n:
+        if o in (1043, 1042):
+            ol, nl = _varchar_len(o, old["typmod"]), _varchar_len(n, new["typmod"])
+            return ol is not None and (nl is None or nl > ol)
+        if o == 1700:
+            op, os_ = _numeric_ps(old["typmod"])
+            np, ns = _numeric_ps(new["typmod"])
+            if op is None:
+                return False
+            return np is None or (np >= op and ns == os_ and np > op)
+        return False
+    if o in _INT_RANK and n in _INT_RANK:
+        return _INT_RANK[n] > _INT_RANK[o]
+    if o in (1043, 1042) and n == 25:  # varchar/bpchar -> text
+        return True
+    if o == 700 and n == 701:  # float4 -> float8
+        return True
+    return False
+
+
+def classify_drift(old_sig: dict, new_sig: dict) -> List[Drift]:
+    """Compare two signatures of the SAME relation. Ordered and deterministic."""
+    out: List[Drift] = []
+    old_cols: Dict[str, dict] = old_sig.get("columns", {})
+    new_cols: Dict[str, dict] = new_sig.get("columns", {})
+    dropped = sorted(set(old_cols) - set(new_cols))
+    added = sorted(set(new_cols) - set(old_cols))
+
+    # One dropped and one added of the same destination type is far more often a rename than a
+    # coincidental pair; report it as a rename so a human sees the real event, at warning level
+    # because the runtime cannot prove it (logical replication carries rows, not the DDL).
+    if len(dropped) == 1 and len(added) == 1 and old_cols[dropped[0]]["dest_type"] == new_cols[added[0]]["dest_type"]:
+        out.append(
+            Drift(
+                "rename_column",
+                added[0],
+                f"{dropped[0]!r} disappeared and {added[0]!r} appeared with the same type "
+                f"({new_cols[added[0]]['dest_type']}) — probably a rename",
+                "warning",
+            )
+        )
+        dropped, added = [], []
+
+    for name in added:
+        if new_cols[name]["not_null"]:
+            out.append(
+                Drift(
+                    "add_not_null_column",
+                    name,
+                    "added NOT NULL — rows already in bronze keep NULL there; backfill or resnapshot to align",
+                    "warning",
+                )
+            )
+        else:
+            out.append(Drift("add_nullable_column", name, "added nullable column", "additive"))
+    for name in dropped:
+        out.append(
+            Drift(
+                "drop_column",
+                name,
+                "dropped at the source — bronze keeps the column and its history (dlt never destroys one)",
+                "breaking",
+            )
+        )
+    for name in sorted(set(old_cols) & set(new_cols)):
+        o, n = old_cols[name], new_cols[name]
+        if o["type_oid"] == n["type_oid"] and o["typmod"] == n["typmod"]:
+            continue
+        if _is_widen(o, n):
+            out.append(
+                Drift("type_widen", name, f"OID {o['type_oid']}→{n['type_oid']} widened, applied automatically", "additive")
+            )
+        else:
+            out.append(
+                Drift(
+                    "type_incompatible",
+                    name,
+                    f"OID {o['type_oid']}→{n['type_oid']} is not a proven-safe widening",
+                    "breaking",
+                )
+            )
+
+    if list(old_sig.get("key", [])) != list(new_sig.get("key", [])):
+        out.append(
+            Drift(
+                "key_change",
+                None,
+                f"merge key {old_sig.get('key')} → {new_sig.get('key')}; a live key change double-loads "
+                "under dlt merge — the table is removed from the publication and needs `resnapshot`",
+                "breaking",
+            )
+        )
+    if old_sig.get("replica_identity") != new_sig.get("replica_identity"):
+        out.append(
+            Drift(
+                "replica_identity_change",
+                None,
+                f"REPLICA IDENTITY {old_sig.get('replica_identity')} → {new_sig.get('replica_identity')}",
+                "info",
+            )
+        )
+    return out

feather_cdc_postgres/selectors.py

@@ -0,0 +1,347 @@
+"""Desired-state compiler: manifest + catalog + adapter → one deterministic relation set.
+
+Constraints:
+  - deny is evaluated on the COMPILED set, not on the include list — an adapter or a wildcard
+    must not be able to smuggle a denied relation in (#3079 D12).
+  - ordering is sorted everywhere, so `selection_hash` is a function of the selection alone.
+  - a relation is always (schema, table); the target schema is derived per TABLE
+    (`destination.table_schemas`, then `schema_mapping`), never assumed equal (D-13).
+"""
+from __future__ import annotations
+
+import fnmatch
+import hashlib
+import re
+from dataclasses import dataclass, field
+from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple
+
+from .adapters import USER_CREATED, AdapterRelation
+from .discovery import SourceRelation
+from .manifest import Manifest, Problem, Selection
+
+# Key policies, in the order the compiler prefers them.
+PRIMARY_KEY = "primary_key"
+UNIQUE_INDEX = "unique_index"
+APPEND_ONLY = "append_only"
+# Rejected, never compiled: an all-column merge key cannot mirror an UPDATE — pgoutput emits
+# only the new tuple, so the old row is never matched and both versions stay (#3079 review).
+FULL_ROW = "full_row"
+
+# Featherbase's own platform tables. Physical-schema-faithful bronze keeps them where they are;
+# the class is what lets a consumer filter them out (#3079 D3).
+_SOURCE_SYSTEM = frozenset(
+    {
+        "table_def", "column_def", "person", "identifier_system", "role", "has_role",
+        "permission", "data_scope",
+        "share", "user_event", "access_log", "activity_log", "version", "migration", "site",
+    }
+)
+_SOURCE_OPERATIONAL = frozenset({"background_job", "job_execution"})
+# A selected column whose NAME looks secret-bearing and is not in `deny.columns` (#4346). A
+# warning, never an error: the name is a hint, and `validate` must stay green on a false hit.
+_SECRET_LIKE = re.compile(r"password|_hash$|token|secret", re.IGNORECASE)
+
+
+@dataclass(frozen=True)
+class Relation:
+    schema: str
+    table: str
+    key: Tuple[str, ...]
+    key_policy: str
+    table_class: str
+    module: Optional[str] = None
+    target_schema: str = ""
+    key_index: Optional[str] = None  # the unique index a UNIQUE_INDEX key came from
+    description: Optional[str] = None
+    origin: Optional[str] = None  # adapters.USER_CREATED | NAMED_SYSTEM | None
+    # Columns that never leave the CDC process (`source.deny.columns`, #4346), sorted.
+    deny_columns: Tuple[str, ...] = ()
+
+    @property
+    def qualified(self) -> str:
+        return f"{self.schema}.{self.table}"
+
+    @property
+    def target(self) -> str:
+        return f"{self.target_schema or self.schema}.{self.table}"
+
+
+@dataclass(frozen=True)
+class CompiledPlan:
+    relations: List[Relation] = field(default_factory=list)
+    selection_hash: str = ""
+    publication_add: List[Tuple[str, str]] = field(default_factory=list)
+    publication_drop: List[Tuple[str, str]] = field(default_factory=list)
+    replica_identity_alters: List[Tuple[str, str]] = field(default_factory=list)
+    denied_hits: List[str] = field(default_factory=list)
+    problems: List[Problem] = field(default_factory=list)
+
+
+def _matches(qualified: str, schema: str, sel: Selection) -> bool:
+    if any(fnmatch.fnmatchcase(schema, pat) for pat in sel.schemas):
+        return True
+    return any(fnmatch.fnmatchcase(qualified, pat) for pat in sel.tables)
+
+
+def _included(qualified: str, schema: str, sel: Selection) -> bool:
+    """Include is a conjunction: the schema must be allowed AND, when table patterns are given,
+    one must match. Table patterns alone (empty `schemas`) select by table pattern only."""
+    schema_ok = any(fnmatch.fnmatchcase(schema, pat) for pat in sel.schemas)
+    if not sel.tables:
+        return schema_ok
+    table_ok = any(fnmatch.fnmatchcase(qualified, pat) for pat in sel.tables)
+    return table_ok if not sel.schemas else (schema_ok and table_ok)
+
+
+def table_class(m: Manifest, schema: str, table: str, origin: Optional[str] = None) -> str:
+    """First matching manifest rule wins; otherwise the defaults below. Loader-internal is
+    checked first so a broad application rule cannot claim `_dlt_loads`. A relation the
+    adapter offered as USER_CREATED is application data whatever it is called — a user's
+    table named `share` is not the platform's `share`."""
+    qualified = f"{schema}.{table}"
+    for rule in m.classification.rules:
+        if fnmatch.fnmatchcase(qualified, rule.match):
+            return rule.table_class
+    if table.startswith("_dlt_") or schema == "_cdc_runtime":
+        return "loader_internal"
+    if origin == USER_CREATED:
+        return "application"
+    if m.source_id == "control_plane" and schema == "control":
+        return "warehouse_control"
+    if table in _SOURCE_OPERATIONAL:
+        return "source_operational"
+    if table in _SOURCE_SYSTEM:
+        return "source_system"
+    return "application"
+
+
+def _key_for(
+    m: Manifest, rel: SourceRelation, problems: List[Problem]
+) -> Tuple[Tuple[str, ...], str, Optional[str], bool]:
+    """(key, policy, index_name, keep) per D6."""
+    override = m.replication.key_overrides.get(rel.qualified)
+    if override == APPEND_ONLY:
+        return (), APPEND_ONLY, None, True
+    if override == FULL_ROW:
+        problems.append(
+            Problem(
+                "full_row_unsupported",
+                "error",
+                f"{rel.qualified}: full_row cannot mirror an UPDATE (the old row is never matched, "
+                "so both versions would stay) — declare append_only for a history, or add a key",
+            )
+        )
+        return (), FULL_ROW, None, False
+
+    if rel.primary_key:
+        return tuple(rel.primary_key), PRIMARY_KEY, None, True
+
+    # A unique index is a merge key only when it is also NOT NULL: dlt joins the destination to
+    # the staging table on key equality, and NULL = NULL is never true, so a nullable key row
+    # would insert a duplicate on every load instead of updating.
+    notnull = {c.name for c in rel.columns if c.not_null}
+    for idx in rel.unique_indexes:
+        if all(c in notnull for c in idx.columns):
+            if m.replication.replica_identity != "full" and rel.replica_identity not in ("f", "i"):
+                problems.append(
+                    Problem(
+                        "unique_key_needs_identity",
+                        "warning",
+                        f"{rel.qualified}: merging on unique index {idx.name!r}, but REPLICA IDENTITY is "
+                        f"{rel.replica_identity!r} — DELETE will not carry those columns; set "
+                        "replica_identity: full or ALTER ... REPLICA IDENTITY USING INDEX",
+                    )
+                )
+            return tuple(idx.columns), UNIQUE_INDEX, idx.name, True
+
+    if m.replication.tables_without_key == APPEND_ONLY:
+        problems.append(
+            Problem(
+                "no_key_append_only",
+                "warning",
+                f"{rel.qualified}: no primary key and no NOT NULL unique index — loading append-only; "
+                "updates and deletes are not promised at the destination",
+            )
+        )
+        return (), APPEND_ONLY, None, True
+
+    problems.append(
+        Problem(
+            "no_key_rejected",
+            "error",
+            f"{rel.qualified}: no primary key and no NOT NULL unique index; set "
+            "replication.tables_without_key: append_only or add key_overrides for this table",
+        )
+    )
+    return (), APPEND_ONLY, None, False
+
+
+def compile(
+    m: Manifest,
+    discovered: Sequence[SourceRelation],
+    adapter_relations: Optional[Sequence[AdapterRelation]] = None,
+    published: Optional[Set[Tuple[str, str]]] = None,
+) -> CompiledPlan:
+    """Compile the manifest against a discovered catalog. Pure — no connection, no mutation."""
+    problems: List[Problem] = []
+    by_qualified = {r.qualified: r for r in discovered}
+
+    adapter_allowed: Optional[Set[str]] = None
+    adapter_meta: Dict[str, AdapterRelation] = {}
+    if m.source.discovery is not None:
+        adapter_meta = {f"{a.schema}.{a.table}": a for a in (adapter_relations or [])}
+        adapter_allowed = set(adapter_meta)
+        for q in sorted(adapter_allowed - set(by_qualified)):
+            problems.append(
+                Problem("adapter_phantom", "warning", f"{q}: selected by the adapter but not physically present — skipped")
+            )
+
+    denied: List[str] = []
+    candidates: List[SourceRelation] = []
+    leaves_of: Dict[str, List[str]] = {}
+    for rel in discovered:
+        if rel.parent:
+            leaves_of.setdefault(rel.parent, []).append(rel.qualified)
+    for rel in sorted(discovered, key=lambda r: (r.schema, r.table)):
+        q = rel.qualified
+        if not _included(q, rel.schema, m.source.include):
+            continue
+        if _matches(q, rel.schema, m.source.exclude):
+            continue
+        if adapter_allowed is not None and q not in adapter_allowed:
+            continue
+        if rel.kind == "p":
+            # Leaves only: `pg_publication_tables` expands a parent to its leaves (so a parent
+            # in the desired set is "added" again every tick), and pgoutput emits leaf
+            # identity by default (so a parent's destination copy would stop receiving rows).
+            # The leaves are selected by the same globs; the parent is named so the operator
+            # can see what was skipped (#3079 review).
+            problems.append(
+                Problem(
+                    "partitioned_parent_skipped",
+                    "warning",
+                    f"{q}: partitioned parent — not replicated as one relation; its partitions "
+                    f"{', '.join(sorted(leaves_of.get(q, []))) or '(none yet)'} are selected "
+                    "individually when the include globs match them",
+                )
+            )
+            continue
+        # Deny last, on what selection actually produced: a deny rule is a tripwire, so it must
+        # fire only when something really would have been replicated (D12).
+        if _matches(q, rel.schema, m.source.deny):
+            denied.append(q)
+            continue
+        candidates.append(rel)
+
+    for q in denied:
+        problems.append(
+            Problem("deny_hit", "error", f"{q} matches a deny rule but was selected — refusing to replicate it")
+        )
+
+    relations: List[Relation] = []
+    for rel in candidates:
+        key, policy, idx_name, keep = _key_for(m, rel, problems)
+        if not keep:
+            continue
+        meta = adapter_meta.get(rel.qualified)
+        deny_cols = _deny_columns_for(m, rel, key, problems)
+        if deny_cols is None:
+            continue
+        relations.append(
+            Relation(
+                schema=rel.schema,
+                table=rel.table,
+                key=key,
+                key_policy=policy,
+                table_class=table_class(m, rel.schema, rel.table, meta.origin if meta else None),
+                module=meta.module if meta else None,
+                target_schema=m.destination.target_schema(rel.schema, rel.table),
+                key_index=idx_name,
+                description=meta.description if meta else None,
+                origin=meta.origin if meta else None,
+                deny_columns=deny_cols,
+            )
+        )
+
+    seen_target: Dict[str, str] = {}
+    for r in relations:
+        prior = seen_target.get(r.target)
+        if prior:
+            problems.append(
+                Problem(
+                    "target_collision",
+                    "error",
+                    f"{prior} and {r.qualified} both map to {m.destination.database}.{r.target}",
+                )
+            )
+        seen_target[r.target] = r.qualified
+
+    desired = {(r.schema, r.table) for r in relations}
+    add, drop = diff_publication(desired, published)
+    alters = (
+        sorted((r.schema, r.table) for r in relations if by_qualified[r.qualified].replica_identity != "f")
+        if m.replication.replica_identity == "full"
+        else []
+    )
+
+    return CompiledPlan(
+        relations=relations,
+        selection_hash=selection_hash(relations),
+        publication_add=add,
+        publication_drop=drop,
+        replica_identity_alters=alters,
+        denied_hits=denied,
+        problems=problems,
+    )
+
+
+def _deny_columns_for(
+    m: Manifest, rel: SourceRelation, key: Tuple[str, ...], problems: List[Problem]
+) -> Optional[Tuple[str, ...]]:
+    """The relation's column projection (#4346), or None when it cannot be honoured: a denied
+    merge-key column would leave the mirror unable to merge, so the relation is refused. A name
+    the catalogue does not carry is a warning (it may be dropped, or not yet added) and is still
+    withheld, so a column added later can never flow before the manifest is re-read."""
+    denied = sorted(set(m.source.deny_columns.get(rel.qualified, [])))
+    present = {c.name for c in rel.columns}
+    for col in denied:
+        if col in key:
+            problems.append(Problem(
+                "deny_column_is_key", "error",
+                f"{rel.qualified}.{col} is part of the merge key and cannot be withheld — "
+                "the mirror could not merge without it"))
+            return None
+        if col not in present:
+            problems.append(Problem(
+                "deny_column_absent", "warning",
+                f"{rel.qualified}.{col} is denied but not in the source catalogue — still withheld "
+                "if it appears"))
+    for col in sorted(present - set(denied)):
+        if _SECRET_LIKE.search(col):
+            problems.append(Problem(
+                "secret_like_column", "warning",
+                f"{rel.qualified}.{col} looks secret-bearing and is replicated — add it to "
+                "source.deny.columns if it should never leave the source"))
+    return tuple(denied)
+
+
+def selection_hash(relations: Iterable[Relation]) -> str:
+    """Stable over the selection AND its key policy — a table that silently changed from
+    merge to append-only must produce a different hash in the control plane. A column
+    projection (#4346) is part of the selection; it is appended only when present, so a
+    registration with no column deny keeps the hash it always had."""
+    lines = sorted(
+        f"{r.schema}.{r.table}|{','.join(r.key)}|{r.key_policy}"
+        + (f"|deny={','.join(r.deny_columns)}" if r.deny_columns else "")
+        for r in relations)
+    return hashlib.sha256("\n".join(lines).encode("utf-8")).hexdigest()
+
+
+def diff_publication(
+    desired: Set[Tuple[str, str]], published: Optional[Set[Tuple[str, str]]]
+) -> Tuple[List[Tuple[str, str]], List[Tuple[str, str]]]:
+    """(add, drop). `published is None` means the publication does not exist yet — everything
+    is an add and nothing can be dropped."""
+    if published is None:
+        return sorted(desired), []
+    return sorted(desired - published), sorted(published - desired)