rowproof 0.1.0__py3-none-any.whl

rowproof/cli/spec.py

@@ -0,0 +1,99 @@
+"""Parsing for the SOURCE/TARGET connection+table grammar (spec §7):
+
+    postgres://user:pw@host:5432/db/public.orders
+
+i.e. a normal DSN, plus one extra path segment naming `schema.table` (or
+just `table`, schema defaults to "public"). This is intentionally the only
+thing this module does — engine dispatch and the actual connection live in
+connectors/.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from urllib.parse import urlsplit, urlunsplit
+
+from rowproof.core.models import TableRef
+
+
+@dataclass(frozen=True)
+class ParsedSource:
+    engine: str
+    connect_dsn: str
+    table_ref: TableRef
+
+
+_ENGINE_DEFAULT_PORT = {"postgres": 5432, "postgresql": 5432}
+
+
+def parse_source_spec(spec: str) -> ParsedSource:
+    parts = urlsplit(spec)
+    engine = parts.scheme
+    if not engine:
+        raise ValueError(
+            f"'{spec}' doesn't look like a connection string "
+            "(expected engine://user:pw@host:port/database/table)"
+        )
+
+    path_segments = [seg for seg in parts.path.split("/") if seg]
+    if len(path_segments) < 2:
+        raise ValueError(
+            f"'{spec}' is missing the table: expected "
+            "engine://user:pw@host:port/database/[schema.]table"
+        )
+    database = path_segments[0]
+    table_spec = "/".join(path_segments[1:])
+
+    if "." in table_spec:
+        schema, table = table_spec.split(".", 1)
+    else:
+        schema, table = None, table_spec
+
+    netloc = parts.netloc
+    connect_dsn = f"{parts.scheme}://{netloc}/{database}"
+
+    return ParsedSource(
+        engine=engine,
+        connect_dsn=connect_dsn,
+        table_ref=TableRef(engine=engine, database=database, schema=schema, table=table),
+    )
+
+
+def redact_dsn(dsn: str) -> str:
+    """spec §8.3: "both DSNs with secrets redacted" — used by the HTML
+    report's "Reproduce" section (and anywhere else a DSN might be shown
+    to a human). Generated SQL never embeds a DSN, so this is the only
+    place redaction is needed.
+    """
+    parts = urlsplit(dsn)
+    if not parts.password:
+        return dsn
+    # `.password` returns the netloc's own (possibly percent-encoded) text
+    # verbatim, not unquoted -- so it can be matched back into `.netloc`
+    # directly with no re-encoding step.
+    redacted_netloc = parts.netloc.replace(f":{parts.password}@", ":***@")
+    return urlunsplit((parts.scheme, redacted_netloc, parts.path, parts.query, parts.fragment))
+
+
+def resolve_source_spec(spec: str, connections: dict[str, str] | None) -> ParsedSource:
+    """spec §7: SOURCE/TARGET is either a full connection string, "or a
+    named connection from the config file: prod_pg/public.orders" — used
+    by `rowproof run` (and `cli.main.cmd_diff`, so `rowproof diff` can
+    reference a config's connections too, once one is loaded). A named
+    connection's own value is a bare DSN with no table (validated by
+    config.load_config); this just splices the two together and reuses
+    parse_source_spec's own DSN+table grammar rather than duplicating it.
+    """
+    if "://" in spec:
+        return parse_source_spec(spec)
+    if not connections:
+        raise ValueError(
+            f"'{spec}' doesn't look like a connection string and no named "
+            "connections were loaded (expected engine://... or name/[schema.]table)"
+        )
+    name, _, rest = spec.partition("/")
+    if name not in connections:
+        raise ValueError(f"unknown connection '{name}' (known: {', '.join(sorted(connections))})")
+    if not rest:
+        raise ValueError(f"'{spec}' is missing the table: expected {name}/[schema.]table")
+    return parse_source_spec(f"{connections[name]}/{rest}")

rowproof/config/__init__.py

@@ -0,0 +1,167 @@
+# YAML config loading/validation for `rowproof run config.yaml` (spec §7,
+# §9 M1).
+"""Config file format (spec §7's `rowproof run CONFIG.yaml`):
+
+    connections:
+      prod_pg: postgres://user:${PROD_PG_PASSWORD}@host:5432/db
+      analytics_ch: clickhouse://user:${CH_PASSWORD}@host:8123/db
+
+    tables:
+      - source: prod_pg/public.orders
+        target: analytics_ch/orders
+        key: order_id
+        where: created_at > '2024-01-01'
+      - source: prod_pg/public.customers
+        target: analytics_ch/customers
+
+A connection's value may also be `{dsn: ...}` for room to grow (e.g. a
+future per-connection option) without breaking the plain-string form.
+Every table entry accepts the same options `diff` does (key, columns,
+exclude, where/where_source/where_target, algorithm, row_threshold,
+max_diff_rows, trim, case_insensitive, float_precision, assume_tz,
+column_map, fail_on) — `key`/`columns`/`exclude` accept either a YAML
+list or a comma-separated string, matching the CLI flag's own grammar.
+
+`${ENV_VAR}` is substituted everywhere in the raw file (spec §7:
+"Secrets: accept ${ENV_VAR} in config and DSNs") before YAML parsing even
+starts, so it works in a DSN, a `where` clause, or anywhere else — never
+printed or logged: PostgresConnector's own query logging only ever sees
+the substituted DSN inside a *DSN*, which _pgwire.py already never logs
+(passwords go through PGPASSWORD, not the SQL/-c argument list).
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass, field
+
+import yaml
+
+from rowproof.core.errors import RowProofError
+from rowproof.core.hashdiff import DEFAULT_MAX_DIFF_ROWS, DEFAULT_ROW_THRESHOLD
+
+_ENV_VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
+
+
+def _substitute_env(text: str) -> str:
+    def replace(m: re.Match) -> str:
+        name = m.group(1)
+        if name not in os.environ:
+            raise RowProofError(
+                f"config references ${{{name}}} but that environment variable is not set"
+            )
+        return os.environ[name]
+
+    return _ENV_VAR_RE.sub(replace, text)
+
+
+def _as_list(value) -> list[str] | None:
+    if value is None:
+        return None
+    if isinstance(value, str):
+        return [v.strip() for v in value.split(",") if v.strip()]
+    if isinstance(value, list):
+        return [str(v) for v in value]
+    raise RowProofError(f"expected a list or comma-separated string, got {value!r}")
+
+
+def _as_column_map(value) -> dict | None:
+    if value is None:
+        return None
+    if isinstance(value, dict):
+        return {str(k): str(v) for k, v in value.items()}
+    if isinstance(value, str):
+        result = {}
+        for pair in value.split(","):
+            pair = pair.strip()
+            if not pair:
+                continue
+            src, _, tgt = pair.partition(":")
+            result[src.strip()] = tgt.strip()
+        return result or None
+    raise RowProofError(f"column_map: expected a mapping or 'a:b,c:d' string, got {value!r}")
+
+
+@dataclass(frozen=True)
+class TableJob:
+    source: str
+    target: str
+    key: list[str] | None = None
+    columns: list[str] | None = None
+    exclude: list[str] | None = None
+    where: str | None = None
+    where_source: str | None = None
+    where_target: str | None = None
+    algorithm: str = "auto"
+    row_threshold: int = DEFAULT_ROW_THRESHOLD
+    max_diff_rows: int = DEFAULT_MAX_DIFF_ROWS
+    trim: bool = False
+    case_insensitive: bool = False
+    float_precision: int = 15
+    assume_tz: str = "UTC"
+    column_map: dict | None = None
+    fail_on: str = "any"
+
+
+@dataclass(frozen=True)
+class Config:
+    connections: dict[str, str] = field(default_factory=dict)
+    tables: list[TableJob] = field(default_factory=list)
+
+
+def _resolve_connection_value(name: str, value) -> str:
+    if isinstance(value, str):
+        return value
+    if isinstance(value, dict) and "dsn" in value:
+        return str(value["dsn"])
+    raise RowProofError(f"connections.{name}: expected a DSN string or a mapping with a 'dsn' key")
+
+
+def load_config(path: str) -> Config:
+    try:
+        with open(path, encoding="utf-8") as f:
+            raw_text = f.read()
+    except OSError as e:
+        raise RowProofError(f"could not read config file '{path}': {e}") from e
+
+    raw_text = _substitute_env(raw_text)
+    try:
+        data = yaml.safe_load(raw_text) or {}
+    except yaml.YAMLError as e:
+        raise RowProofError(f"'{path}' is not valid YAML: {e}") from e
+    if not isinstance(data, dict):
+        raise RowProofError(f"'{path}' must be a YAML mapping with 'connections'/'tables' keys")
+
+    connections = {
+        name: _resolve_connection_value(name, value)
+        for name, value in (data.get("connections") or {}).items()
+    }
+
+    tables = []
+    for i, t in enumerate(data.get("tables") or []):
+        if not isinstance(t, dict) or "source" not in t or "target" not in t:
+            raise RowProofError(f"tables[{i}]: both 'source' and 'target' are required")
+        tables.append(
+            TableJob(
+                source=t["source"],
+                target=t["target"],
+                key=_as_list(t.get("key")),
+                columns=_as_list(t.get("columns")),
+                exclude=_as_list(t.get("exclude")),
+                where=t.get("where"),
+                where_source=t.get("where_source"),
+                where_target=t.get("where_target"),
+                algorithm=t.get("algorithm", "auto"),
+                row_threshold=t.get("row_threshold", DEFAULT_ROW_THRESHOLD),
+                max_diff_rows=t.get("max_diff_rows", DEFAULT_MAX_DIFF_ROWS),
+                trim=t.get("trim", False),
+                case_insensitive=t.get("case_insensitive", False),
+                float_precision=t.get("float_precision", 15),
+                assume_tz=t.get("assume_tz", "UTC"),
+                column_map=_as_column_map(t.get("column_map")),
+                fail_on=t.get("fail_on", "any"),
+            )
+        )
+
+    return Config(connections=connections, tables=tables)

rowproof/connectors/_chwire.py

@@ -0,0 +1,160 @@
+"""ClickHouse wire access, via `clickhouse-connect` — the real spec §3
+driver.
+
+`ClickHouseConnector` in clickhouse.py calls only the functions below,
+never clickhouse_connect directly. This module used to be a hand-rolled
+stdlib `urllib` HTTP client (see git history / docs/DEV_ENVIRONMENT.md
+for why); clickhouse-connect replaces that with the real driver, which
+speaks the same HTTP interface but also handles response decoding,
+per-engine type mapping, and connection re-use itself instead of this
+module doing it by hand.
+
+Two query paths, matching two different needs (mirrors _pgwire.py's own
+split — see that module's docstring for the full performance rationale):
+
+* `open_client()` / `run_query_on()` / `close_client()` — a genuinely
+  persistent client, opened once by `connect()` and reused for every
+  query for the lifetime of a diff (spec §5: "One connection per side.
+  No connection pooling in v1."). Reconnecting per query measured at
+  ~20ms of pure overhead against a local ClickHouse — negligible for one
+  query, ruinous across the thousands a large diff issues.
+* `run_query()` / `check_connection()` — a one-shot query against a
+  fresh client, for stateless reachability probes and test-only admin
+  operations that have no diff-lifetime client to reuse.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from urllib.parse import unquote, urlsplit
+
+import clickhouse_connect
+from clickhouse_connect.driver.client import Client
+from clickhouse_connect.driver.exceptions import ClickHouseError
+from clickhouse_connect.driver.exceptions import OperationalError as ChOperationalError
+
+# clickhouse-connect logs a warning (not an exception) when a DDL
+# response's X-ClickHouse-Summary header isn't the JSON shape a SELECT
+# response has (e.g. CREATE DATABASE) — harmless (the statement still
+# succeeds), but noisy by default; quiet it to genuine errors only.
+logging.getLogger("clickhouse_connect").setLevel(logging.ERROR)
+
+# ClickHouse's AUTHENTICATION_FAILED error code (see
+# https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp).
+# Unlike Postgres, a bad password over ClickHouse's HTTP interface doesn't
+# fail at the transport layer (no OperationalError) — the server answers
+# with an ordinary-looking error response carrying this code, so it has to
+# be recognised by code, not by exception type, to be treated as a
+# connection failure rather than a query failure (spec §10: "Clean error
+# for: bad credentials").
+_AUTH_FAILED_CODE = 516
+
+
+class ConnectionFailedError(Exception):
+    pass
+
+
+class QueryFailedError(Exception):
+    def __init__(self, message: str, sql: str):
+        super().__init__(message)
+        self.sql = sql
+
+
+@dataclass(frozen=True)
+class ChDsn:
+    host: str
+    port: int
+    user: str
+    password: str | None
+    database: str
+
+
+def parse_ch_dsn(dsn: str) -> ChDsn:
+    """Parse `clickhouse://user:pw@host:port/database`. Default port 8123
+    (ClickHouse's plain-HTTP port — what clickhouse-connect itself talks;
+    8443 for HTTPS is out of scope for v1, same as Postgres's DSN handling
+    not covering `sslmode`)."""
+    parts = urlsplit(dsn)
+    if parts.scheme not in ("clickhouse", "ch"):
+        raise ValueError(f"not a clickhouse DSN: {dsn!r}")
+    database = parts.path.lstrip("/")
+    return ChDsn(
+        host=parts.hostname or "localhost",
+        port=parts.port or 8123,
+        user=unquote(parts.username) if parts.username else "default",
+        password=unquote(parts.password) if parts.password else None,
+        database=database or "default",
+    )
+
+
+def _reraise_as_rowproof_error(e: Exception, sql: str) -> None:
+    if isinstance(e, ChOperationalError):
+        # A real transport-level failure: refused/unreachable host,
+        # connection reset, DNS failure.
+        raise ConnectionFailedError(str(e)) from e
+    if isinstance(e, ClickHouseError) and getattr(e, "code", None) == _AUTH_FAILED_CODE:
+        raise ConnectionFailedError(str(e)) from e
+    if isinstance(e, ClickHouseError):
+        raise QueryFailedError(str(e), sql) from e
+    raise e
+
+
+def open_client(dsn: ChDsn, timeout: float | None = 10.0) -> Client:
+    """Open and return a single persistent client — the one this side of
+    a diff uses for every query until `close_client()`."""
+    try:
+        return clickhouse_connect.get_client(
+            host=dsn.host,
+            port=dsn.port,
+            username=dsn.user,
+            password=dsn.password or "",
+            database=dsn.database,
+            connect_timeout=max(1, int(timeout)) if timeout else 10,
+        )
+    except Exception as e:  # noqa: BLE001 - re-raised as a typed rowproof error below
+        _reraise_as_rowproof_error(e, "<connect>")
+        raise  # pragma: no cover - _reraise_as_rowproof_error always raises
+
+
+def close_client(client: Client) -> None:
+    client.close()
+
+
+def run_query_on(client: Client, sql: str, timeout: float | None = None) -> list[tuple]:
+    """Run `sql` on an already-open persistent client and return rows as
+    tuples. This is the hot path every per-segment query in a diff goes
+    through — no new client, no re-authentication. Never mutates `sql`
+    (no FORMAT clause appended, no query rewriting) — the exact text a
+    caller built is the exact text sent, matching spec §5's
+    `--verbose`-reproducibility promise.
+    """
+    try:
+        settings = {"max_execution_time": timeout} if timeout else None
+        result = client.query(sql, settings=settings)
+        return [tuple(row) for row in result.result_rows]
+    except Exception as e:  # noqa: BLE001
+        _reraise_as_rowproof_error(e, sql)
+        raise  # pragma: no cover
+
+
+def check_connection(dsn: ChDsn) -> None:
+    try:
+        run_query(dsn, "SELECT 1")
+    except QueryFailedError as e:
+        raise ConnectionFailedError(str(e)) from e
+
+
+def run_query(dsn: ChDsn, sql: str, timeout: float | None = 30.0) -> list[tuple]:
+    """Open a fresh client, run `sql`, return rows as tuples, close.
+
+    A deliberate one-shot path — for `check_connection`'s stateless
+    reachability probes and test-only admin operations, never for the
+    diff hot path (see module docstring; `ClickHouseConnector.query()`
+    uses `run_query_on()` on its one persistent client instead).
+    """
+    client = open_client(dsn, timeout)
+    try:
+        return run_query_on(client, sql, timeout)
+    finally:
+        client.close()

rowproof/connectors/_pgwire.py

@@ -0,0 +1,165 @@
+"""Postgres wire access, via `psycopg` (v3) — the real spec §3 driver.
+
+`PostgresConnector` in postgres.py calls only the functions below, never
+psycopg directly, so the rest of the codebase is insulated from driver
+specifics. This module used to shell out to the `psql` CLI binary (see
+git history / docs/DEV_ENVIRONMENT.md for why); psycopg replaces that
+with a real wire-protocol connection and hands back already-typed Python
+values (int, Decimal, datetime, bool, ...) instead of text that has to be
+guessed back into a type.
+
+Two query paths, matching two different needs:
+
+* `open_connection()` / `run_query_on()` / `close_connection()` — a
+  genuinely persistent connection, opened once by `connect()` and reused
+  for every query for the lifetime of a diff. This is what spec §5 means
+  by "One connection per side. No connection pooling in v1." and is
+  performance-critical: a 100M-row diff issues thousands of small
+  per-segment queries (spec §13's 100M-row/5-minute benchmark), and
+  reconnecting for each one measured at ~17ms of pure overhead against a
+  local Postgres — at a few thousand queries that alone would blow the
+  time budget several times over.
+* `run_query()` / `check_connection()` — a one-shot, self-contained
+  query against a fresh connection, for the handful of call sites that
+  genuinely want that: a stateless reachability probe (`check_connection`,
+  used both by `connect()` below and by test fixtures polling "is the
+  server back up yet") and test-only admin operations (creating/dropping
+  whole scratch databases — see `tests/integration/conftest.py`'s
+  `exec_sql`), which have no diff-lifetime connection to reuse anyway.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from urllib.parse import unquote, urlsplit
+
+import psycopg
+
+
+class ConnectionFailedError(Exception):
+    pass
+
+
+class QueryFailedError(Exception):
+    def __init__(self, message: str, sql: str):
+        super().__init__(message)
+        self.sql = sql
+
+
+@dataclass(frozen=True)
+class PgDsn:
+    host: str
+    port: int
+    user: str
+    password: str | None
+    database: str
+
+
+def parse_pg_dsn(dsn: str) -> PgDsn:
+    """Parse `postgres://user:pw@host:port/database`.
+
+    Only the connection portion is handled here — a trailing
+    `/schema.table` (spec §7's SOURCE/TARGET grammar) is the CLI layer's
+    job to split off before calling connect().
+    """
+    parts = urlsplit(dsn)
+    if parts.scheme not in ("postgres", "postgresql"):
+        raise ValueError(f"not a postgres DSN: {dsn!r}")
+    database = parts.path.lstrip("/")
+    return PgDsn(
+        host=parts.hostname or "localhost",
+        port=parts.port or 5432,
+        user=unquote(parts.username) if parts.username else "",
+        password=unquote(parts.password) if parts.password else None,
+        database=database,
+    )
+
+
+def _connect_kwargs(dsn: PgDsn, connect_timeout: float | None) -> dict:
+    kwargs = {
+        "host": dsn.host,
+        "port": dsn.port,
+        "user": dsn.user,
+        "dbname": dsn.database,
+        "autocommit": True,
+    }
+    if dsn.password is not None:
+        kwargs["password"] = dsn.password
+    if connect_timeout:
+        # libpq's connect_timeout is whole seconds; round up so a small
+        # fractional timeout doesn't collapse to 0 (which libpq treats as
+        # "no timeout" — the opposite of what a caller asking for a short
+        # timeout means).
+        kwargs["connect_timeout"] = max(1, int(connect_timeout + 0.999))
+    return kwargs
+
+
+def _classify(e: Exception, sql: str, timeout: float | None) -> None:
+    """Re-raise a psycopg exception as the rowproof-typed error the rest
+    of the codebase expects. Shared by both the persistent-connection and
+    one-shot query paths so the two stay classified identically."""
+    if isinstance(e, psycopg.errors.QueryCanceled):
+        raise QueryFailedError(f"query timed out after {timeout}s", sql) from e
+    if isinstance(e, psycopg.OperationalError):
+        # Every libpq-level failure (refused/reset connection, auth
+        # failure, unreachable host, dropped mid-query) surfaces as
+        # OperationalError in psycopg — see spec §10's error-path list.
+        raise ConnectionFailedError(str(e)) from e
+    if isinstance(e, psycopg.Error):
+        raise QueryFailedError(str(e), sql) from e
+    raise e
+
+
+def open_connection(dsn: PgDsn, timeout: float | None = 10.0) -> psycopg.Connection:
+    """Open and return a single persistent connection — the one this side
+    of a diff uses for every query until `close_connection()`."""
+    try:
+        return psycopg.connect(**_connect_kwargs(dsn, timeout))
+    except psycopg.Error as e:
+        _classify(e, "<connect>", timeout)
+        raise  # pragma: no cover - _classify always raises for psycopg.Error
+
+
+def close_connection(conn: psycopg.Connection) -> None:
+    conn.close()
+
+
+def run_query_on(conn: psycopg.Connection, sql: str, timeout: float | None = None) -> list[tuple]:
+    """Run `sql` on an already-open persistent connection and return rows
+    as tuples. This is the hot path every per-segment query in a diff
+    goes through — no new connection, no re-authentication."""
+    try:
+        if timeout:
+            # Server-side safety net so a hung query can't block forever.
+            conn.execute(f"SET statement_timeout = {int(timeout * 1000)}")
+        with conn.cursor() as cur:
+            cur.execute(sql)
+            if cur.description is None:
+                return []
+            return [tuple(row) for row in cur.fetchall()]
+    except psycopg.Error as e:
+        _classify(e, sql, timeout)
+        raise  # pragma: no cover
+
+
+def check_connection(dsn: PgDsn, timeout: float = 10.0) -> None:
+    try:
+        run_query(dsn, "SELECT 1", timeout=timeout)
+    except QueryFailedError as e:
+        raise ConnectionFailedError(str(e)) from e
+
+
+def run_query(dsn: PgDsn, sql: str, timeout: float | None = 60.0) -> list[tuple]:
+    """Open a fresh connection, run `sql`, return rows as tuples, close.
+
+    A deliberate one-shot path — for `check_connection`'s stateless
+    reachability probes and test-only admin operations, never for the
+    diff hot path (see module docstring; `PostgresConnector.query()` uses
+    `run_query_on()` on its one persistent connection instead).
+    """
+    try:
+        with psycopg.connect(**_connect_kwargs(dsn, timeout)) as conn:
+            return run_query_on(conn, sql, timeout)
+    except psycopg.Error as e:
+        _classify(e, sql, timeout)
+        raise  # pragma: no cover