parity-diff 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {parity_diff-0.1.0 → parity_diff-0.2.0}/CONTRIBUTING.md +15 -1
- {parity_diff-0.1.0/src/parity_diff.egg-info → parity_diff-0.2.0}/PKG-INFO +89 -16
- {parity_diff-0.1.0 → parity_diff-0.2.0}/README.md +82 -12
- {parity_diff-0.1.0 → parity_diff-0.2.0}/pyproject.toml +8 -5
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity/__init__.py +6 -1
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity/cli.py +68 -4
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity/dialects/base.py +150 -38
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity/dialects/duckdb_dialect.py +36 -3
- parity_diff-0.2.0/src/parity/dialects/mysql_dialect.py +238 -0
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity/dialects/postgres_dialect.py +44 -3
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity/engine.py +115 -38
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity/types.py +43 -2
- {parity_diff-0.1.0 → parity_diff-0.2.0/src/parity_diff.egg-info}/PKG-INFO +89 -16
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity_diff.egg-info/SOURCES.txt +3 -1
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity_diff.egg-info/requires.txt +4 -0
- {parity_diff-0.1.0 → parity_diff-0.2.0}/tests/conftest.py +51 -0
- {parity_diff-0.1.0 → parity_diff-0.2.0}/tests/fakes.py +40 -2
- {parity_diff-0.1.0 → parity_diff-0.2.0}/tests/test_cli.py +44 -2
- {parity_diff-0.1.0 → parity_diff-0.2.0}/tests/test_encoding.py +214 -21
- {parity_diff-0.1.0 → parity_diff-0.2.0}/tests/test_engine.py +150 -7
- {parity_diff-0.1.0 → parity_diff-0.2.0}/tests/test_integration.py +14 -7
- parity_diff-0.2.0/tests/test_mysql.py +226 -0
- {parity_diff-0.1.0 → parity_diff-0.2.0}/LICENSE +0 -0
- {parity_diff-0.1.0 → parity_diff-0.2.0}/MANIFEST.in +0 -0
- {parity_diff-0.1.0 → parity_diff-0.2.0}/setup.cfg +0 -0
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity/dialects/__init__.py +0 -0
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity_diff.egg-info/dependency_links.txt +0 -0
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity_diff.egg-info/entry_points.txt +0 -0
- {parity_diff-0.1.0 → parity_diff-0.2.0}/src/parity_diff.egg-info/top_level.txt +0 -0
|
@@ -43,7 +43,7 @@ That comes to roughly 70–85 lines. Read
|
|
|
43
43
|
`src/parity/dialects/duckdb_dialect.py` first — it is the shortest complete
|
|
44
44
|
example.
|
|
45
45
|
|
|
46
|
-
### The
|
|
46
|
+
### The things that will bite you
|
|
47
47
|
|
|
48
48
|
Each of these cost real time to discover. They are documented at length in
|
|
49
49
|
`CLAUDE.md` §4; the short version:
|
|
@@ -83,6 +83,20 @@ Each of these cost real time to discover. They are documented at length in
|
|
|
83
83
|
`numeric`, `NUMERIC(38,0)` — and check `int_div` still truncates on that
|
|
84
84
|
type rather than producing a scaled or rounded result.
|
|
85
85
|
|
|
86
|
+
MySQL, added after v0.1.0, turned up two more that a warehouse dialect may hit:
|
|
87
|
+
|
|
88
|
+
7. **Not every engine has `chr`, and `char(n)` may be binary.** MySQL spells
|
|
89
|
+
the separator `char(31)`, not `chr(31)`, and a bare `char(31)` is a *binary*
|
|
90
|
+
string that coerces the whole `concat_ws` to bytes - which then comes back
|
|
91
|
+
from `fetch_range` as `bytes`, not `str`, and every row reads as different.
|
|
92
|
+
The separator and the NULL sentinel are dialect hooks (`separator_sql`,
|
|
93
|
+
`null_sentinel_sql`) for exactly this reason.
|
|
94
|
+
|
|
95
|
+
8. **Backslash in a string literal is not portable.** MySQL processes a
|
|
96
|
+
backslash as an escape inside `'...'`, toggled by `sql_mode`, so the `\N`
|
|
97
|
+
sentinel silently became `N`. Build such bytes from `CHAR`/hex, not a
|
|
98
|
+
literal, and verify by hashing rather than by reading the SQL.
|
|
99
|
+
|
|
86
100
|
### Proving it works
|
|
87
101
|
|
|
88
102
|
A dialect is not done until `tests/test_encoding.py` passes against it. That
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: parity-diff
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Prove two tables in two different database engines hold the same data - without moving the data out of either engine.
|
|
5
5
|
Author: Alessio Sorio
|
|
6
6
|
License-Expression: MIT
|
|
7
|
-
Project-URL: Homepage, https://github.com/Aleixiou/parity
|
|
8
|
-
Project-URL: Repository, https://github.com/Aleixiou/parity
|
|
9
|
-
Project-URL: Issues, https://github.com/Aleixiou/parity/issues
|
|
7
|
+
Project-URL: Homepage, https://github.com/Aleixiou/parity-diff
|
|
8
|
+
Project-URL: Repository, https://github.com/Aleixiou/parity-diff
|
|
9
|
+
Project-URL: Issues, https://github.com/Aleixiou/parity-diff/issues
|
|
10
10
|
Keywords: data-diff,migration,postgres,duckdb,data-quality,cutover,warehouse,parity
|
|
11
11
|
Classifier: Development Status :: 3 - Alpha
|
|
12
12
|
Classifier: Environment :: Console
|
|
@@ -28,14 +28,17 @@ Provides-Extra: duckdb
|
|
|
28
28
|
Requires-Dist: duckdb>=1.0; extra == "duckdb"
|
|
29
29
|
Provides-Extra: postgres
|
|
30
30
|
Requires-Dist: psycopg[binary]>=3.1; extra == "postgres"
|
|
31
|
+
Provides-Extra: mysql
|
|
32
|
+
Requires-Dist: mysql-connector-python>=8.0; extra == "mysql"
|
|
31
33
|
Provides-Extra: all
|
|
32
34
|
Requires-Dist: duckdb>=1.0; extra == "all"
|
|
33
35
|
Requires-Dist: psycopg[binary]>=3.1; extra == "all"
|
|
36
|
+
Requires-Dist: mysql-connector-python>=8.0; extra == "all"
|
|
34
37
|
Dynamic: license-file
|
|
35
38
|
|
|
36
39
|
# parity
|
|
37
40
|
|
|
38
|
-
[](https://github.com/Aleixiou/parity/actions/workflows/tests.yml)
|
|
41
|
+
[](https://github.com/Aleixiou/parity-diff/actions/workflows/tests.yml)
|
|
39
42
|
|
|
40
43
|
**Prove two tables in two different database engines hold the same data —
|
|
41
44
|
without moving the data out of either engine.**
|
|
@@ -54,7 +57,7 @@ parity diff \
|
|
|
54
57
|
|
|
55
58
|
```
|
|
56
59
|
✗ 5 differences in 10,000,000 rows
|
|
57
|
-
28 queries · 7,628 rows downloaded (0.04% of both tables) ·
|
|
60
|
+
28 queries · 7,628 rows downloaded (0.04% of both tables) · 47.9s
|
|
58
61
|
1 only in A · 1 only in B · 3 different
|
|
59
62
|
|
|
60
63
|
only in A key 999999999
|
|
@@ -88,8 +91,16 @@ on one developer laptop (`demo/benchmark.py`):
|
|
|
88
91
|
|
|
89
92
|
| Scenario | Queries | Rows downloaded | Wall time |
|
|
90
93
|
|---|---|---|---|
|
|
91
|
-
| identical tables | 4 | **0** (0.0000%) |
|
|
92
|
-
| 5 planted differences | 28 | 7,628 (0.0381%) |
|
|
94
|
+
| identical tables | **4** | **0** (0.0000%) | 25–31s |
|
|
95
|
+
| 5 planted differences | **28** | **7,628** (0.0381%) | 48–55s |
|
|
96
|
+
|
|
97
|
+
**The query counts and row counts are exact and hardware-independent** — they
|
|
98
|
+
are properties of the algorithm, and the test suite pins them. Reproduce them
|
|
99
|
+
and you should get the same integers.
|
|
100
|
+
|
|
101
|
+
The wall times are a range across six runs on one developer laptop, and they
|
|
102
|
+
vary by about 25% with whatever else that laptop is doing. Treat them as an
|
|
103
|
+
order of magnitude, not a specification.
|
|
93
104
|
|
|
94
105
|
The query count and the rows-downloaded figures are **exact and
|
|
95
106
|
hardware-independent** — they are properties of the algorithm, and the test
|
|
@@ -110,9 +121,10 @@ round trips on later levels.
|
|
|
110
121
|
## Install
|
|
111
122
|
|
|
112
123
|
```bash
|
|
113
|
-
pip install "parity-diff[all]" #
|
|
114
|
-
pip install "parity-diff[duckdb]" # DuckDB only — no
|
|
124
|
+
pip install "parity-diff[all]" # every engine
|
|
125
|
+
pip install "parity-diff[duckdb]" # DuckDB only — no other driver pulled in
|
|
115
126
|
pip install "parity-diff[postgres]" # PostgreSQL only
|
|
127
|
+
pip install "parity-diff[mysql]" # MySQL only
|
|
116
128
|
```
|
|
117
129
|
|
|
118
130
|
> The PyPI distribution is `parity-diff` — plain `parity` is squatted by an
|
|
@@ -125,7 +137,7 @@ imported lazily, so a DuckDB-only user is never made to install `psycopg`.
|
|
|
125
137
|
## Usage
|
|
126
138
|
|
|
127
139
|
```
|
|
128
|
-
parity diff --a CONN --a-table TABLE --b CONN --b-table TABLE --key COL
|
|
140
|
+
parity diff --a CONN --a-table TABLE --b CONN --b-table TABLE --key COL[,COL...]
|
|
129
141
|
[--columns a,b,c] [--exclude x,y]
|
|
130
142
|
[--bisection-factor 32] [--threshold 10000] [--float-scale 6]
|
|
131
143
|
[--max-diffs 100] [--json] [--quiet]
|
|
@@ -146,6 +158,7 @@ Connection strings:
|
|
|
146
158
|
|
|
147
159
|
```
|
|
148
160
|
postgres://user:password@host:port/database (also postgresql://)
|
|
161
|
+
mysql://user:password@host:port/database
|
|
149
162
|
duckdb:///relative/path.duckdb (three slashes = relative)
|
|
150
163
|
duckdb:////var/lib/warehouse.duckdb (four slashes = absolute)
|
|
151
164
|
duckdb:///C:/data/warehouse.duckdb (absolute, Windows)
|
|
@@ -169,14 +182,73 @@ PostgreSQL and `main` on DuckDB.
|
|
|
169
182
|
--key order_id --quiet
|
|
170
183
|
```
|
|
171
184
|
|
|
185
|
+
## Using it from Python
|
|
186
|
+
|
|
187
|
+
The CLI is a thin wrapper. `diff` returns the whole result, so you can act on
|
|
188
|
+
it rather than parse output. Importing `parity` pulls in no database driver —
|
|
189
|
+
they load when `get_dialect` needs one.
|
|
190
|
+
|
|
191
|
+
```python
|
|
192
|
+
from parity import diff, get_dialect
|
|
193
|
+
|
|
194
|
+
a = get_dialect("duckdb:///old.duckdb", side="A")
|
|
195
|
+
b = get_dialect("duckdb:///new.duckdb", side="B")
|
|
196
|
+
try:
|
|
197
|
+
result = diff(a, b, "main.orders", "main.orders", key="id")
|
|
198
|
+
finally:
|
|
199
|
+
a.close()
|
|
200
|
+
b.close()
|
|
201
|
+
|
|
202
|
+
if result.identical:
|
|
203
|
+
print("the tables match")
|
|
204
|
+
else:
|
|
205
|
+
for d in result.diffs:
|
|
206
|
+
print(d.kind, d.key, d.columns, d.values_a, d.values_b)
|
|
207
|
+
|
|
208
|
+
print(f"{result.stats.rows_downloaded} rows crossed the network")
|
|
209
|
+
print(f"truncated: {result.truncated}")
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
```
|
|
213
|
+
different 2 ['status'] {'status': 'ok'} {'status': 'CHANGED'}
|
|
214
|
+
2 rows crossed the network
|
|
215
|
+
truncated: False
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
`diff` takes the same options as the CLI: `columns`, `exclude`,
|
|
219
|
+
`bisection_factor`, `threshold`, `max_diffs`. Pass `max_diffs=None` to lift the
|
|
220
|
+
10,000 default, and `float_scale` to `get_dialect` — both sides must agree or
|
|
221
|
+
the comparison is refused before it runs.
|
|
222
|
+
|
|
223
|
+
### What you get back
|
|
224
|
+
|
|
225
|
+
`DiffResult` carries:
|
|
226
|
+
|
|
227
|
+
| Attribute | |
|
|
228
|
+
|---|---|
|
|
229
|
+
| `identical` | `True` only if nothing differed **and** the whole key space was walked. False whenever `truncated` is set. |
|
|
230
|
+
| `truncated` | The walk stopped early, so this is a partial answer. Never read a truncated result as "the rest matched". |
|
|
231
|
+
| `diffs` | `RowDiff` objects in key order, each with `key`, `kind` (`only_in_a`, `only_in_b`, `different`), the `columns` that moved, and `values_a` / `values_b` as raw canonical text. |
|
|
232
|
+
| `columns` | The columns actually compared, after `columns` and `exclude`. |
|
|
233
|
+
| `warnings` | Everything the comparison decided on your behalf: columns skipped, types that differ between sides, timezone-awareness mismatches. Worth surfacing. |
|
|
234
|
+
| `float_scale` | The rounding in force, so a caller can state it alongside the verdict. |
|
|
235
|
+
| `stats` | `queries`, `rows_downloaded`, `rows_compared_a`, `rows_compared_b`, `segments_checked`, `seconds`. |
|
|
236
|
+
|
|
237
|
+
`diff` raises `ValueError` for anything it refuses — a non-integer key, a
|
|
238
|
+
non-unique or NULL key, a missing table, mismatched float scales. The message
|
|
239
|
+
always names which side.
|
|
240
|
+
|
|
172
241
|
## Limitations — read these before trusting a result
|
|
173
242
|
|
|
174
243
|
A parity tool that reports a false match is worse than useless, so these are
|
|
175
244
|
stated plainly rather than buried.
|
|
176
245
|
|
|
177
|
-
- **
|
|
178
|
-
|
|
179
|
-
|
|
246
|
+
- **Any key type, but non-integer keys are bucketed by a hash.** A single
|
|
247
|
+
integer column is bisected directly. A `uuid`, a natural string key, or
|
|
248
|
+
several columns together are hashed to 60 bits so the key space can be
|
|
249
|
+
divided — and the run says so. Rows are still matched and reported by their
|
|
250
|
+
real key, so a hash collision can only put two rows in the same bucket; it
|
|
251
|
+
can never merge them.
|
|
180
252
|
- **Floats and decimals are compared at 6 decimal places** by default. Two
|
|
181
253
|
values differing only in the 7th place are reported as *equal*. This is a
|
|
182
254
|
deliberate cross-engine rounding contract — the two engines do not otherwise
|
|
@@ -213,7 +285,8 @@ stated plainly rather than buried.
|
|
|
213
285
|
|---|---|
|
|
214
286
|
| PostgreSQL | supported (tested against 16 and 18) |
|
|
215
287
|
| DuckDB | supported (tested against 1.5) |
|
|
216
|
-
|
|
|
288
|
+
| MySQL | supported (tested against 8.0) |
|
|
289
|
+
| Snowflake, BigQuery, Redshift | not yet — see `CONTRIBUTING.md`, a dialect is ~80 lines |
|
|
217
290
|
|
|
218
291
|
## Scope
|
|
219
292
|
|
|
@@ -257,7 +330,7 @@ python demo/benchmark.py --expect-planted
|
|
|
257
330
|
```
|
|
258
331
|
|
|
259
332
|
`CLAUDE.md` holds the verified cross-engine SQL and why each expression is the
|
|
260
|
-
way it is. `
|
|
333
|
+
way it is. `ROADMAP.md` is what is done and what comes next.
|
|
261
334
|
|
|
262
335
|
## Changelog
|
|
263
336
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# parity
|
|
2
2
|
|
|
3
|
-
[](https://github.com/Aleixiou/parity/actions/workflows/tests.yml)
|
|
3
|
+
[](https://github.com/Aleixiou/parity-diff/actions/workflows/tests.yml)
|
|
4
4
|
|
|
5
5
|
**Prove two tables in two different database engines hold the same data —
|
|
6
6
|
without moving the data out of either engine.**
|
|
@@ -19,7 +19,7 @@ parity diff \
|
|
|
19
19
|
|
|
20
20
|
```
|
|
21
21
|
✗ 5 differences in 10,000,000 rows
|
|
22
|
-
28 queries · 7,628 rows downloaded (0.04% of both tables) ·
|
|
22
|
+
28 queries · 7,628 rows downloaded (0.04% of both tables) · 47.9s
|
|
23
23
|
1 only in A · 1 only in B · 3 different
|
|
24
24
|
|
|
25
25
|
only in A key 999999999
|
|
@@ -53,8 +53,16 @@ on one developer laptop (`demo/benchmark.py`):
|
|
|
53
53
|
|
|
54
54
|
| Scenario | Queries | Rows downloaded | Wall time |
|
|
55
55
|
|---|---|---|---|
|
|
56
|
-
| identical tables | 4 | **0** (0.0000%) |
|
|
57
|
-
| 5 planted differences | 28 | 7,628 (0.0381%) |
|
|
56
|
+
| identical tables | **4** | **0** (0.0000%) | 25–31s |
|
|
57
|
+
| 5 planted differences | **28** | **7,628** (0.0381%) | 48–55s |
|
|
58
|
+
|
|
59
|
+
**The query counts and row counts are exact and hardware-independent** — they
|
|
60
|
+
are properties of the algorithm, and the test suite pins them. Reproduce them
|
|
61
|
+
and you should get the same integers.
|
|
62
|
+
|
|
63
|
+
The wall times are a range across six runs on one developer laptop, and they
|
|
64
|
+
vary by about 25% with whatever else that laptop is doing. Treat them as an
|
|
65
|
+
order of magnitude, not a specification.
|
|
58
66
|
|
|
59
67
|
The query count and the rows-downloaded figures are **exact and
|
|
60
68
|
hardware-independent** — they are properties of the algorithm, and the test
|
|
@@ -75,9 +83,10 @@ round trips on later levels.
|
|
|
75
83
|
## Install
|
|
76
84
|
|
|
77
85
|
```bash
|
|
78
|
-
pip install "parity-diff[all]" #
|
|
79
|
-
pip install "parity-diff[duckdb]" # DuckDB only — no
|
|
86
|
+
pip install "parity-diff[all]" # every engine
|
|
87
|
+
pip install "parity-diff[duckdb]" # DuckDB only — no other driver pulled in
|
|
80
88
|
pip install "parity-diff[postgres]" # PostgreSQL only
|
|
89
|
+
pip install "parity-diff[mysql]" # MySQL only
|
|
81
90
|
```
|
|
82
91
|
|
|
83
92
|
> The PyPI distribution is `parity-diff` — plain `parity` is squatted by an
|
|
@@ -90,7 +99,7 @@ imported lazily, so a DuckDB-only user is never made to install `psycopg`.
|
|
|
90
99
|
## Usage
|
|
91
100
|
|
|
92
101
|
```
|
|
93
|
-
parity diff --a CONN --a-table TABLE --b CONN --b-table TABLE --key COL
|
|
102
|
+
parity diff --a CONN --a-table TABLE --b CONN --b-table TABLE --key COL[,COL...]
|
|
94
103
|
[--columns a,b,c] [--exclude x,y]
|
|
95
104
|
[--bisection-factor 32] [--threshold 10000] [--float-scale 6]
|
|
96
105
|
[--max-diffs 100] [--json] [--quiet]
|
|
@@ -111,6 +120,7 @@ Connection strings:
|
|
|
111
120
|
|
|
112
121
|
```
|
|
113
122
|
postgres://user:password@host:port/database (also postgresql://)
|
|
123
|
+
mysql://user:password@host:port/database
|
|
114
124
|
duckdb:///relative/path.duckdb (three slashes = relative)
|
|
115
125
|
duckdb:////var/lib/warehouse.duckdb (four slashes = absolute)
|
|
116
126
|
duckdb:///C:/data/warehouse.duckdb (absolute, Windows)
|
|
@@ -134,14 +144,73 @@ PostgreSQL and `main` on DuckDB.
|
|
|
134
144
|
--key order_id --quiet
|
|
135
145
|
```
|
|
136
146
|
|
|
147
|
+
## Using it from Python
|
|
148
|
+
|
|
149
|
+
The CLI is a thin wrapper. `diff` returns the whole result, so you can act on
|
|
150
|
+
it rather than parse output. Importing `parity` pulls in no database driver —
|
|
151
|
+
they load when `get_dialect` needs one.
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
from parity import diff, get_dialect
|
|
155
|
+
|
|
156
|
+
a = get_dialect("duckdb:///old.duckdb", side="A")
|
|
157
|
+
b = get_dialect("duckdb:///new.duckdb", side="B")
|
|
158
|
+
try:
|
|
159
|
+
result = diff(a, b, "main.orders", "main.orders", key="id")
|
|
160
|
+
finally:
|
|
161
|
+
a.close()
|
|
162
|
+
b.close()
|
|
163
|
+
|
|
164
|
+
if result.identical:
|
|
165
|
+
print("the tables match")
|
|
166
|
+
else:
|
|
167
|
+
for d in result.diffs:
|
|
168
|
+
print(d.kind, d.key, d.columns, d.values_a, d.values_b)
|
|
169
|
+
|
|
170
|
+
print(f"{result.stats.rows_downloaded} rows crossed the network")
|
|
171
|
+
print(f"truncated: {result.truncated}")
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
```
|
|
175
|
+
different 2 ['status'] {'status': 'ok'} {'status': 'CHANGED'}
|
|
176
|
+
2 rows crossed the network
|
|
177
|
+
truncated: False
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
`diff` takes the same options as the CLI: `columns`, `exclude`,
|
|
181
|
+
`bisection_factor`, `threshold`, `max_diffs`. Pass `max_diffs=None` to lift the
|
|
182
|
+
10,000 default, and `float_scale` to `get_dialect` — both sides must agree or
|
|
183
|
+
the comparison is refused before it runs.
|
|
184
|
+
|
|
185
|
+
### What you get back
|
|
186
|
+
|
|
187
|
+
`DiffResult` carries:
|
|
188
|
+
|
|
189
|
+
| Attribute | |
|
|
190
|
+
|---|---|
|
|
191
|
+
| `identical` | `True` only if nothing differed **and** the whole key space was walked. False whenever `truncated` is set. |
|
|
192
|
+
| `truncated` | The walk stopped early, so this is a partial answer. Never read a truncated result as "the rest matched". |
|
|
193
|
+
| `diffs` | `RowDiff` objects in key order, each with `key`, `kind` (`only_in_a`, `only_in_b`, `different`), the `columns` that moved, and `values_a` / `values_b` as raw canonical text. |
|
|
194
|
+
| `columns` | The columns actually compared, after `columns` and `exclude`. |
|
|
195
|
+
| `warnings` | Everything the comparison decided on your behalf: columns skipped, types that differ between sides, timezone-awareness mismatches. Worth surfacing. |
|
|
196
|
+
| `float_scale` | The rounding in force, so a caller can state it alongside the verdict. |
|
|
197
|
+
| `stats` | `queries`, `rows_downloaded`, `rows_compared_a`, `rows_compared_b`, `segments_checked`, `seconds`. |
|
|
198
|
+
|
|
199
|
+
`diff` raises `ValueError` for anything it refuses — a non-integer key, a
|
|
200
|
+
non-unique or NULL key, a missing table, mismatched float scales. The message
|
|
201
|
+
always names which side.
|
|
202
|
+
|
|
137
203
|
## Limitations — read these before trusting a result
|
|
138
204
|
|
|
139
205
|
A parity tool that reports a false match is worse than useless, so these are
|
|
140
206
|
stated plainly rather than buried.
|
|
141
207
|
|
|
142
|
-
- **
|
|
143
|
-
|
|
144
|
-
|
|
208
|
+
- **Any key type, but non-integer keys are bucketed by a hash.** A single
|
|
209
|
+
integer column is bisected directly. A `uuid`, a natural string key, or
|
|
210
|
+
several columns together are hashed to 60 bits so the key space can be
|
|
211
|
+
divided — and the run says so. Rows are still matched and reported by their
|
|
212
|
+
real key, so a hash collision can only put two rows in the same bucket; it
|
|
213
|
+
can never merge them.
|
|
145
214
|
- **Floats and decimals are compared at 6 decimal places** by default. Two
|
|
146
215
|
values differing only in the 7th place are reported as *equal*. This is a
|
|
147
216
|
deliberate cross-engine rounding contract — the two engines do not otherwise
|
|
@@ -178,7 +247,8 @@ stated plainly rather than buried.
|
|
|
178
247
|
|---|---|
|
|
179
248
|
| PostgreSQL | supported (tested against 16 and 18) |
|
|
180
249
|
| DuckDB | supported (tested against 1.5) |
|
|
181
|
-
|
|
|
250
|
+
| MySQL | supported (tested against 8.0) |
|
|
251
|
+
| Snowflake, BigQuery, Redshift | not yet — see `CONTRIBUTING.md`, a dialect is ~80 lines |
|
|
182
252
|
|
|
183
253
|
## Scope
|
|
184
254
|
|
|
@@ -222,7 +292,7 @@ python demo/benchmark.py --expect-planted
|
|
|
222
292
|
```
|
|
223
293
|
|
|
224
294
|
`CLAUDE.md` holds the verified cross-engine SQL and why each expression is the
|
|
225
|
-
way it is. `
|
|
295
|
+
way it is. `ROADMAP.md` is what is done and what comes next.
|
|
226
296
|
|
|
227
297
|
## Changelog
|
|
228
298
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# PyPI by an empty project. The import name, the CLI command and the repo are
|
|
4
4
|
# all still `parity`: `pip install parity-diff` gives you `parity ...`.
|
|
5
5
|
name = "parity-diff"
|
|
6
|
-
version = "0.
|
|
6
|
+
version = "0.2.0"
|
|
7
7
|
description = "Prove two tables in two different database engines hold the same data - without moving the data out of either engine."
|
|
8
8
|
readme = "README.md"
|
|
9
9
|
license = "MIT"
|
|
@@ -36,12 +36,13 @@ dependencies = []
|
|
|
36
36
|
[project.optional-dependencies]
|
|
37
37
|
duckdb = ["duckdb>=1.0"]
|
|
38
38
|
postgres = ["psycopg[binary]>=3.1"]
|
|
39
|
-
|
|
39
|
+
mysql = ["mysql-connector-python>=8.0"]
|
|
40
|
+
all = ["duckdb>=1.0", "psycopg[binary]>=3.1", "mysql-connector-python>=8.0"]
|
|
40
41
|
|
|
41
42
|
[project.urls]
|
|
42
|
-
Homepage = "https://github.com/Aleixiou/parity"
|
|
43
|
-
Repository = "https://github.com/Aleixiou/parity"
|
|
44
|
-
Issues = "https://github.com/Aleixiou/parity/issues"
|
|
43
|
+
Homepage = "https://github.com/Aleixiou/parity-diff"
|
|
44
|
+
Repository = "https://github.com/Aleixiou/parity-diff"
|
|
45
|
+
Issues = "https://github.com/Aleixiou/parity-diff/issues"
|
|
45
46
|
|
|
46
47
|
[project.scripts]
|
|
47
48
|
parity = "parity.cli:main"
|
|
@@ -58,6 +59,7 @@ testpaths = ["tests"]
|
|
|
58
59
|
addopts = "-q --strict-markers"
|
|
59
60
|
markers = [
|
|
60
61
|
"postgres: requires a reachable PostgreSQL server",
|
|
62
|
+
"mysql: requires a reachable MySQL server",
|
|
61
63
|
"duckdb: requires the duckdb driver",
|
|
62
64
|
]
|
|
63
65
|
|
|
@@ -99,6 +101,7 @@ ignore = [
|
|
|
99
101
|
# have their own injection tests in tests/test_encoding.py.
|
|
100
102
|
"src/parity/dialects/base.py" = ["S608"]
|
|
101
103
|
"src/parity/dialects/postgres_dialect.py" = ["S608"]
|
|
104
|
+
"src/parity/dialects/mysql_dialect.py" = ["S608"]
|
|
102
105
|
# proof.py is kept close to the original author's script so it stays readable
|
|
103
106
|
# next to the findings it produced; its terse one-line style is deliberate.
|
|
104
107
|
"demo/proof.py" = ["E401", "E402", "E701", "E702", "B007", "S311", "S608"]
|
|
@@ -13,12 +13,17 @@ from __future__ import annotations
|
|
|
13
13
|
|
|
14
14
|
from typing import Any
|
|
15
15
|
|
|
16
|
-
__version__ = "0.
|
|
16
|
+
__version__ = "0.2.0"
|
|
17
17
|
|
|
18
18
|
__all__ = ["__version__", "diff", "get_dialect"]
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
def __getattr__(name: str) -> Any:
|
|
22
|
+
"""Resolve `parity.diff` and `parity.get_dialect` on first use.
|
|
23
|
+
|
|
24
|
+
Deferring the import is what keeps `import parity` free of database
|
|
25
|
+
drivers, so a DuckDB-only user is never made to install psycopg.
|
|
26
|
+
"""
|
|
22
27
|
# Lazy re-export: keeps `import parity` free of driver imports.
|
|
23
28
|
if name == "get_dialect":
|
|
24
29
|
from parity.dialects.base import get_dialect
|
|
@@ -37,6 +37,11 @@ KIND_LABELS = {
|
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
def build_parser() -> argparse.ArgumentParser:
|
|
40
|
+
"""Build the whole command line surface.
|
|
41
|
+
|
|
42
|
+
Split out from `main` so `--help` can be rendered, and the flags asserted,
|
|
43
|
+
without running anything or importing a database driver.
|
|
44
|
+
"""
|
|
40
45
|
parser = argparse.ArgumentParser(
|
|
41
46
|
prog="parity",
|
|
42
47
|
description=(
|
|
@@ -61,7 +66,13 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
61
66
|
d.add_argument("--a-table", required=True, metavar="TABLE", help="side A table")
|
|
62
67
|
d.add_argument("--b", required=True, metavar="CONN", help="side B connection string")
|
|
63
68
|
d.add_argument("--b-table", required=True, metavar="TABLE", help="side B table")
|
|
64
|
-
d.add_argument(
|
|
69
|
+
d.add_argument(
|
|
70
|
+
"--key", required=True, metavar="COL[,COL...]",
|
|
71
|
+
help="the column(s) that identify a row. One integer column is used "
|
|
72
|
+
"directly; a uuid, a text key or several columns together are "
|
|
73
|
+
"hashed so the key space can be bisected, and rows are still "
|
|
74
|
+
"reported by their real key.",
|
|
75
|
+
)
|
|
65
76
|
d.add_argument(
|
|
66
77
|
"--columns", metavar="a,b,c",
|
|
67
78
|
help="compare only these columns (default: every column both sides share)",
|
|
@@ -98,6 +109,11 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
98
109
|
|
|
99
110
|
|
|
100
111
|
def _split(value: str | None) -> list[str]:
|
|
112
|
+
"""Turn a comma-separated flag value into a list, ignoring blanks.
|
|
113
|
+
|
|
114
|
+
So `--exclude "a, b,"` gives ["a", "b"] rather than an empty column name
|
|
115
|
+
that would later fail to match anything.
|
|
116
|
+
"""
|
|
101
117
|
if not value:
|
|
102
118
|
return []
|
|
103
119
|
return [part.strip() for part in value.split(",") if part.strip()]
|
|
@@ -109,6 +125,7 @@ def _split(value: str | None) -> list[str]:
|
|
|
109
125
|
|
|
110
126
|
|
|
111
127
|
def _plural(n: int, word: str) -> str:
|
|
128
|
+
"""Format a count with its noun, pluralised and thousands-separated."""
|
|
112
129
|
return f"{n:,} {word}{'' if n == 1 else 's'}"
|
|
113
130
|
|
|
114
131
|
|
|
@@ -138,6 +155,19 @@ def _symbols(out: TextIO) -> dict[str, str]:
|
|
|
138
155
|
return {"bad": "✗", "ok": "✓", "dot": "·", "partial": "!"}
|
|
139
156
|
|
|
140
157
|
|
|
158
|
+
def _display_key(key: int | str) -> str:
|
|
159
|
+
"""Render a row key for a human.
|
|
160
|
+
|
|
161
|
+
A composite key's canonical text is joined by ASCII Unit Separator, which
|
|
162
|
+
is exactly right for hashing and unreadable on a terminal. Show the parts
|
|
163
|
+
separated visibly instead. JSON keeps the raw text, so a machine still sees
|
|
164
|
+
what was actually compared.
|
|
165
|
+
"""
|
|
166
|
+
if isinstance(key, str) and "" in key:
|
|
167
|
+
return " | ".join(_display(part) for part in key.split(""))
|
|
168
|
+
return _display(key) if isinstance(key, str) else str(key)
|
|
169
|
+
|
|
170
|
+
|
|
141
171
|
def _display(value: str) -> str:
|
|
142
172
|
"""Make canonical text readable without misrepresenting it.
|
|
143
173
|
|
|
@@ -154,6 +184,12 @@ def _display(value: str) -> str:
|
|
|
154
184
|
|
|
155
185
|
|
|
156
186
|
def render_human(result: DiffResult, out: TextIO) -> None:
|
|
187
|
+
"""Write the report a person reads: verdict first, then the evidence.
|
|
188
|
+
|
|
189
|
+
The rows-downloaded percentage is always printed - it is the proof that
|
|
190
|
+
the tool pushed the work into the engines, and a figure that suddenly
|
|
191
|
+
reads 100% is how someone finds out their key column is wrong.
|
|
192
|
+
"""
|
|
157
193
|
sym = _symbols(out)
|
|
158
194
|
stats = result.stats
|
|
159
195
|
total = max(stats.rows_compared_a, stats.rows_compared_b)
|
|
@@ -220,12 +256,19 @@ def render_human(result: DiffResult, out: TextIO) -> None:
|
|
|
220
256
|
|
|
221
257
|
|
|
222
258
|
def _render_diff(d: RowDiff, out: TextIO) -> None:
|
|
259
|
+
"""Write one difference.
|
|
260
|
+
|
|
261
|
+
A changed row shows both values side by side, which is far easier to scan
|
|
262
|
+
for the character that moved - until the values are too long to share a
|
|
263
|
+
line, at which point they stack.
|
|
264
|
+
"""
|
|
223
265
|
label = KIND_LABELS[d.kind]
|
|
224
266
|
if d.kind != "different":
|
|
225
|
-
print(f" {label:<11} key {d.key}", file=out)
|
|
267
|
+
print(f" {label:<11} key {_display_key(d.key)}", file=out)
|
|
226
268
|
return
|
|
227
269
|
|
|
228
|
-
|
|
270
|
+
shown_key = _display_key(d.key)
|
|
271
|
+
print(f" {label:<11} key {shown_key:<14} columns: {', '.join(d.columns)}", file=out)
|
|
229
272
|
name_w = max((len(c) for c in d.columns), default=0)
|
|
230
273
|
a_vals = {c: _display(d.values_a.get(c, "")) for c in d.columns}
|
|
231
274
|
b_vals = {c: _display(d.values_b.get(c, "")) for c in d.columns}
|
|
@@ -246,6 +289,13 @@ def _render_diff(d: RowDiff, out: TextIO) -> None:
|
|
|
246
289
|
|
|
247
290
|
|
|
248
291
|
def to_dict(result: DiffResult) -> dict[str, Any]:
|
|
292
|
+
"""Shape the result for `--json`.
|
|
293
|
+
|
|
294
|
+
Carries the raw canonical text rather than the human-friendly rendering,
|
|
295
|
+
so a machine sees exactly what was compared. `identical` is false whenever
|
|
296
|
+
the walk was cut short, so a consumer reading only that field cannot be
|
|
297
|
+
misled by a partial run.
|
|
298
|
+
"""
|
|
249
299
|
stats = result.stats
|
|
250
300
|
moveable = stats.rows_compared_a + stats.rows_compared_b
|
|
251
301
|
return {
|
|
@@ -289,6 +339,11 @@ def to_dict(result: DiffResult) -> dict[str, Any]:
|
|
|
289
339
|
|
|
290
340
|
|
|
291
341
|
def _run_diff(args: argparse.Namespace, out: TextIO) -> int:
|
|
342
|
+
"""Open both sides, run the comparison, render it, and return the exit code.
|
|
343
|
+
|
|
344
|
+
Both connections are closed even when the diff raises, and each is opened
|
|
345
|
+
separately so a failure can name which side it was.
|
|
346
|
+
"""
|
|
292
347
|
# Imported here, not at module scope, so `parity --help` works with no
|
|
293
348
|
# database driver installed at all.
|
|
294
349
|
from parity.dialects.base import get_dialect
|
|
@@ -303,7 +358,9 @@ def _run_diff(args: argparse.Namespace, out: TextIO) -> int:
|
|
|
303
358
|
a, b,
|
|
304
359
|
a_table=args.a_table,
|
|
305
360
|
b_table=args.b_table,
|
|
306
|
-
|
|
361
|
+
# Comma-separated for composite keys; a single name is just a
|
|
362
|
+
# one-element list.
|
|
363
|
+
key=_split(args.key) or args.key,
|
|
307
364
|
columns=_split(args.columns) or None,
|
|
308
365
|
exclude=_split(args.exclude),
|
|
309
366
|
bisection_factor=args.bisection_factor,
|
|
@@ -336,6 +393,13 @@ def main(
|
|
|
336
393
|
out: TextIO | None = None,
|
|
337
394
|
err: TextIO | None = None,
|
|
338
395
|
) -> int:
|
|
396
|
+
"""Entry point. Returns the exit code rather than calling sys.exit.
|
|
397
|
+
|
|
398
|
+
`out` and `err` are injectable so the tests can drive the real CLI and
|
|
399
|
+
read what it wrote. Anything that is not a clean verdict returns 2, never
|
|
400
|
+
1 - a CI job has to be able to tell "the tables differ" from "the tool
|
|
401
|
+
broke".
|
|
402
|
+
"""
|
|
339
403
|
out = out if out is not None else sys.stdout
|
|
340
404
|
err = err if err is not None else sys.stderr
|
|
341
405
|
|