parity-diff 0.1.0__py3-none-any.whl

parity/types.py

@@ -0,0 +1,125 @@
+"""Core value types shared across dialects and the diff engine.
+
+BUILD_SPEC sketched a ``TableRef`` and a ``Segment`` dataclass. Neither
+survived contact with the implementation: the engine takes a dialect, a
+table name and a key as separate arguments rather than bundling them, and a
+segment is a plain ``(lo, hi)`` tuple whose checksums live in the dict the
+dialect returns. They were carried unused for a while, which is worse than
+not having them - an unused public dataclass invites someone to build on it
+and then drift out of sync with what the code really does.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class LogicalType(str, Enum):
+    """Engine-independent type categories.
+
+    Every physical column type a dialect knows about is mapped onto one of
+    these, and each one has exactly one canonical text encoding (see
+    ``Dialect.normalize``). Two rows are equal iff their canonical encodings
+    are byte-identical, so this enum is where cross-engine correctness lives.
+    """
+
+    INTEGER = "integer"
+    DECIMAL = "decimal"
+    FLOAT = "float"
+    BOOLEAN = "boolean"
+    STRING = "string"
+    DATE = "date"
+    TIMESTAMP = "timestamp"
+    UNKNOWN = "unknown"
+
+
+@dataclass(frozen=True)
+class Column:
+    name: str
+    logical_type: LogicalType
+    raw_type: str = ""
+
+
+@dataclass(frozen=True)
+class KeyStats:
+    """What one side reports about its key column, from a single scan.
+
+    ``rows`` and ``distinct`` ride along with ``min``/``max`` because the
+    min/max query already has to visit the key column. Getting uniqueness for
+    free matters: a non-unique key silently collapses rows during comparison
+    and makes differences vanish, which is the most dangerous failure this
+    tool can have.
+    """
+
+    lo: int | None
+    hi: int | None
+    rows: int
+    distinct: int
+    #: Rows whose key is not NULL. ``None`` means the dialect did not measure
+    #: it, in which case no NULL keys are assumed.
+    non_null: int | None = None
+
+    @property
+    def empty(self) -> bool:
+        return self.rows == 0
+
+    @property
+    def null_keys(self) -> int:
+        return 0 if self.non_null is None else self.rows - self.non_null
+
+    @property
+    def has_null_keys(self) -> bool:
+        return self.null_keys > 0
+
+    @property
+    def has_duplicate_keys(self) -> bool:
+        # Compare like with like: `count(distinct k)` ignores NULLs, so
+        # measuring it against `count(*)` would report every NULL key as a
+        # duplicate and send the reader hunting for duplicates that do not
+        # exist. NULL keys are diagnosed separately.
+        comparable = self.rows if self.non_null is None else self.non_null
+        return comparable != self.distinct
+
+
+@dataclass
+class RowDiff:
+    key: int
+    kind: str  # "only_in_a" | "only_in_b" | "different"
+    columns: list[str] = field(default_factory=list)
+    values_a: dict[str, str] = field(default_factory=dict)
+    values_b: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class DiffStats:
+    queries: int = 0
+    segments_checked: int = 0
+    rows_downloaded: int = 0
+    rows_compared_a: int = 0
+    rows_compared_b: int = 0
+    seconds: float = 0.0
+
+
+@dataclass
+class DiffResult:
+    diffs: list[RowDiff]
+    stats: DiffStats
+    columns: list[Column]
+    warnings: list[str] = field(default_factory=list)
+    #: True when the walk stopped early (``max_diffs``), so ``diffs`` is a
+    #: partial answer. CLAUDE.md section 8: never let an approximate result
+    #: look exact. Callers must not read ``identical`` as "tables match" when
+    #: this is set.
+    truncated: bool = False
+    #: Decimal places at which DECIMAL/FLOAT columns were compared. Surfaced
+    #: in output so a reader knows the comparison was rounded, not exact.
+    float_scale: int = 6
+
+    @property
+    def identical(self) -> bool:
+        """No differences found *and* the whole key space was walked."""
+        return not self.diffs and not self.truncated
+
+    def by_kind(self, kind: str) -> list[RowDiff]:
+        return [d for d in self.diffs if d.kind == kind]

parity_diff-0.1.0.dist-info/METADATA

@@ -0,0 +1,268 @@
+Metadata-Version: 2.4
+Name: parity-diff
+Version: 0.1.0
+Summary: Prove two tables in two different database engines hold the same data - without moving the data out of either engine.
+Author: Alessio Sorio
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Aleixiou/parity
+Project-URL: Repository, https://github.com/Aleixiou/parity
+Project-URL: Issues, https://github.com/Aleixiou/parity/issues
+Keywords: data-diff,migration,postgres,duckdb,data-quality,cutover,warehouse,parity
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: duckdb
+Requires-Dist: duckdb>=1.0; extra == "duckdb"
+Provides-Extra: postgres
+Requires-Dist: psycopg[binary]>=3.1; extra == "postgres"
+Provides-Extra: all
+Requires-Dist: duckdb>=1.0; extra == "all"
+Requires-Dist: psycopg[binary]>=3.1; extra == "all"
+Dynamic: license-file
+
+# parity
+
+[![tests](https://github.com/Aleixiou/parity/actions/workflows/tests.yml/badge.svg)](https://github.com/Aleixiou/parity/actions/workflows/tests.yml)
+
+**Prove two tables in two different database engines hold the same data —
+without moving the data out of either engine.**
+
+Migrations don't fail on translation. They fail at cutover, because nobody can
+prove the new pipeline produces the same data as the old one — so the legacy
+system runs in parallel "just to be safe", forever, at double the cost.
+`parity` is the proof.
+
+```bash
+parity diff \
+  --a "postgres://user:pw@legacy-host/warehouse" --a-table public.orders \
+  --b "duckdb:///./new.duckdb"                   --b-table main.orders \
+  --key order_id
+```
+
+```
+✗ 5 differences in 10,000,000 rows
+  28 queries · 7,628 rows downloaded (0.04% of both tables) · 54.2s
+  1 only in A · 1 only in B · 3 different
+
+  only in A   key 999999999
+  only in B   key 4000
+  different   key 13             columns: note
+      note   A ''   B NULL
+  different   key 6010000        columns: amount
+      amount   A 700.010000   B 700.000000
+  different   key 8700000        columns: is_refunded
+      is_refunded   A NULL   B false
+
+  comparing floats and decimals at 6 decimal places
+```
+
+Exit code `1`. Put it in CI and the build fails until the data agrees.
+
+## How it works
+
+`parity` pushes hash aggregation **down into both engines**. It asks each side
+for one checksum per bucket of the key range, compares a handful of integers,
+recurses only into the buckets that disagree, and downloads rows only from
+ranges already proven to differ.
+
+On identical tables it downloads **zero rows** and issues **four queries**,
+whether the table has ten thousand rows or ten million.
+
+## Measured
+
+10,000,000 rows per side, PostgreSQL 18.4 ↔ DuckDB 1.5.5, median of five runs
+on one developer laptop (`demo/benchmark.py`):
+
+| Scenario | Queries | Rows downloaded | Wall time |
+|---|---|---|---|
+| identical tables | 4 | **0** (0.0000%) | 26.9s |
+| 5 planted differences | 28 | 7,628 (0.0381%) | 54.2s |
+
+The query count and the rows-downloaded figures are **exact and
+hardware-independent** — they are properties of the algorithm, and the test
+suite pins them. The wall times are one machine under sustained load; treat
+them as an order of magnitude, not a specification.
+
+The five planted differences — a changed decimal, a deleted row, an inserted
+row at key 999,999,999, a `NULL` turned into `''`, and a `FALSE` turned into
+`NULL` — are all found exactly, with no false positives.
+
+Cost is roughly **one full hash pass per side**. It is CPU-bound on MD5, not
+IO-bound on key lookup, which is why **an index on the key column makes no
+measurable difference** (measured at 10M: 38.8s without, 37.3s with).
+Raising
+`--bisection-factor` does not speed up the dominant first pass; it only reduces
+round trips on later levels.
+
+## Install
+
+```bash
+pip install "parity-diff[all]"        # both engines
+pip install "parity-diff[duckdb]"     # DuckDB only — no PostgreSQL driver pulled in
+pip install "parity-diff[postgres]"   # PostgreSQL only
+```
+
+> The PyPI distribution is `parity-diff` — plain `parity` is squatted by an
+> empty project. The command you run and the module you import are both
+> `parity`.
+
+Python 3.10+. The core has no dependencies; drivers are optional extras and are
+imported lazily, so a DuckDB-only user is never made to install `psycopg`.
+
+## Usage
+
+```
+parity diff --a CONN --a-table TABLE --b CONN --b-table TABLE --key COL
+            [--columns a,b,c] [--exclude x,y]
+            [--bisection-factor 32] [--threshold 10000] [--float-scale 6]
+            [--max-diffs 100] [--json] [--quiet]
+```
+
+**Exit codes** — this is what makes it a CI check:
+
+| Code | Meaning |
+|---|---|
+| `0` | identical |
+| `1` | differences found |
+| `2` | error (bad connection string, missing table, unusable key) |
+
+An error is never `1`. A CI job can always tell "the tables differ" from "the
+tool could not run".
+
+Connection strings:
+
+```
+postgres://user:password@host:port/database     (also postgresql://)
+duckdb:///relative/path.duckdb                  (three slashes = relative)
+duckdb:////var/lib/warehouse.duckdb             (four slashes = absolute)
+duckdb:///C:/data/warehouse.duckdb              (absolute, Windows)
+duckdb:///:memory:
+```
+
+Table names may be schema-qualified. Unqualified names default to `public` on
+PostgreSQL and `main` on DuckDB.
+
+`--json` emits the same content as a machine-readable object, including
+`identical`, `truncated`, per-difference values, and the full stats block.
+
+### In CI
+
+```yaml
+- name: prove the migration is complete
+  run: |
+    parity diff \
+      --a "$LEGACY_URL"     --a-table public.orders \
+      --b "$WAREHOUSE_URL"  --b-table analytics.orders \
+      --key order_id --quiet
+```
+
+## Limitations — read these before trusting a result
+
+A parity tool that reports a false match is worse than useless, so these are
+stated plainly rather than buried.
+
+- **Integer keys only.** The bisection arithmetic divides the key range. A
+  `varchar` or `uuid` key is rejected with a clear message, not guessed at.
+  Composite and hashed keys are a planned extension.
+- **Floats and decimals are compared at 6 decimal places** by default. Two
+  values differing only in the 7th place are reported as *equal*. This is a
+  deliberate cross-engine rounding contract — the two engines do not otherwise
+  agree on float text. Change it with `--float-scale`; it always applies to
+  both sides, and the scale in force is printed on every run.
+- **Keys must be unique.** A non-unique key is detected up front and rejected.
+  Silently collapsing duplicate rows would hide real differences.
+- **`Infinity` and `NaN`** in float columns are compared as those literal
+  tokens on both sides. A double whose magnitude reaches 1e32 is not supported
+  and fails loudly on DuckDB.
+- **Wide tables are fine.** PostgreSQL caps a function call at 100 arguments,
+  so the row concatenation is built as a nested tree; tested to 500 columns.
+- **Supported types:** integer, decimal, float, boolean, string, date,
+  timestamp. Anything else is compared as raw text and reported as a warning —
+  two engines may render the same JSON or array differently for reasons that
+  have nothing to do with the data.
+- **At most 10,000 differences are reported by default.** Each one costs about
+  715 bytes, so two tables that share nothing would need gigabytes rather than
+  producing an answer - and pointing the tool at the wrong table or the wrong
+  environment is exactly what it exists to catch. Past the limit the run is
+  flagged `truncated`, `identical` is never true, and the output says "at
+  least N". It does not mean the rest matched. `--max-diffs 0` lifts the limit.
+- **Timezone-aware timestamps are compared as instants, in UTC.** Both
+  sessions are pinned to UTC on connect, so the same instant matches whatever
+  the two servers' default timezones are. Without that pin, two sides in
+  different zones render every `timestamptz` differently and the tool reports
+  the entire table as changed.
+- Columns present on only one side are skipped with a warning, not treated as
+  differences.
+
+## Supported engines
+
+| Engine | Status |
+|---|---|
+| PostgreSQL | supported (tested against 16 and 18) |
+| DuckDB | supported (tested against 1.5) |
+| Snowflake, BigQuery | not yet — see `CONTRIBUTING.md`, a dialect is ~80 lines |
+
+## Scope
+
+In scope: proving two tables match, finding exactly which rows and columns
+don't, doing it cheaply on large tables, running in CI.
+
+Deliberately out of scope: data quality rules, freshness checks, anomaly
+detection, lineage, cataloguing, orchestration, transformation, a web UI, a
+server, schema migration. If a feature does not help someone answer *"can I
+safely switch off the old system?"*, it does not belong here.
+
+## Development
+
+```bash
+python -m venv .venv
+.venv/Scripts/Activate.ps1        # Windows;  source .venv/bin/activate elsewhere
+pip install -e ".[all]" pytest
+pytest
+```
+
+Tests that need PostgreSQL read `PARITY_TEST_PG` (default
+`postgres://parity:parity@127.0.0.1:5432/parity`) and **skip cleanly** when no
+server is reachable — the DuckDB and pure-Python suites still run.
+
+A disposable PostgreSQL:
+
+```bash
+docker run -d --name parity-pg -p 55432:5432 \
+  -e POSTGRES_USER=parity -e POSTGRES_PASSWORD=parity -e POSTGRES_DB=parity \
+  postgres:16-alpine
+export PARITY_TEST_PG="postgres://parity:parity@127.0.0.1:55432/parity"
+```
+
+Reproduce the benchmark:
+
+```bash
+python demo/generate.py --rows 10000000
+python demo/benchmark.py --expect-clean
+python demo/generate.py --rows 10000000 --plant
+python demo/benchmark.py --expect-planted
+```
+
+`CLAUDE.md` holds the verified cross-engine SQL and why each expression is the
+way it is. `BUILD_SPEC.md` is the build plan.
+
+## Changelog
+
+See `CHANGELOG.md`.
+
+## License
+
+MIT — see `LICENSE`.

parity_diff-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+parity/__init__.py,sha256=9gMr1dCb1XYXshAdj_lgv05BBoh4RON2tm8AlcxyzeE,902
+parity/cli.py,sha256=1A1xHfWD_sDjV7YCArxInV11CTG1W2nA-Gc3cU1aUdc,13454
+parity/engine.py,sha256=k6ryEZy7jP8eXQ735BAW4lkakmHbtVa4iWyuttHCzEk,18901
+parity/types.py,sha256=s1zTT0FS_Hlh3V7XrAbPHD4pCssxt3yXEw9iGPPVTTk,4111
+parity/dialects/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+parity/dialects/base.py,sha256=tD5GTAGcQJQARA6VpbdUF_tsj_mrwbCr1TlpLXv4AOM,20718
+parity/dialects/duckdb_dialect.py,sha256=Mt9Ucmkdgxagoy_t_yC_y1XOevC4ebDMO4nF0_JR13E,5828
+parity/dialects/postgres_dialect.py,sha256=dq0SHd_eK9AqvyH1YFi2-bV1LD0fvvLCsu3TfGOD1nY,6500
+parity_diff-0.1.0.dist-info/licenses/LICENSE,sha256=J48Z0nBL4u7D3Vrdkh4o4w1kLhPB_3Ke7pq7-e8D3gs,1070
+parity_diff-0.1.0.dist-info/METADATA,sha256=8FmvTeo_nCZQKoCl_adN3wImc3vFXZOjEw59eg0oK2w,10371
+parity_diff-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+parity_diff-0.1.0.dist-info/entry_points.txt,sha256=5e_vs-8G8VRgoc_VYD-IKHrlelGrk3vwIxMsAFgAdbg,43
+parity_diff-0.1.0.dist-info/top_level.txt,sha256=f4JdYzINvWc3BYfKSKF9JPS87QlPOIxMQRbG1esvjBY,7
+parity_diff-0.1.0.dist-info/RECORD,,

parity_diff-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

parity_diff-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+parity = parity.cli:main

parity_diff-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Alessio Sorio
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

parity_diff-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+parity