riftdiff 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- riftdiff-0.1.0/LICENSE +21 -0
- riftdiff-0.1.0/PKG-INFO +301 -0
- riftdiff-0.1.0/README.md +268 -0
- riftdiff-0.1.0/pyproject.toml +55 -0
- riftdiff-0.1.0/setup.cfg +4 -0
- riftdiff-0.1.0/src/riftdiff/__init__.py +7 -0
- riftdiff-0.1.0/src/riftdiff/__main__.py +5 -0
- riftdiff-0.1.0/src/riftdiff/cli.py +182 -0
- riftdiff-0.1.0/src/riftdiff/engine.py +321 -0
- riftdiff-0.1.0/src/riftdiff.egg-info/PKG-INFO +301 -0
- riftdiff-0.1.0/src/riftdiff.egg-info/SOURCES.txt +15 -0
- riftdiff-0.1.0/src/riftdiff.egg-info/dependency_links.txt +1 -0
- riftdiff-0.1.0/src/riftdiff.egg-info/entry_points.txt +2 -0
- riftdiff-0.1.0/src/riftdiff.egg-info/requires.txt +5 -0
- riftdiff-0.1.0/src/riftdiff.egg-info/top_level.txt +1 -0
- riftdiff-0.1.0/tests/test_cli.py +60 -0
- riftdiff-0.1.0/tests/test_engine.py +189 -0
riftdiff-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Chetan Chirag
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
riftdiff-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: riftdiff
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Local-first data diff, powered by DuckDB. See exactly what changed before you merge.
|
|
5
|
+
Author: Chetan Chirag
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ChetanChirag24/rift
|
|
8
|
+
Project-URL: Source, https://github.com/ChetanChirag24/rift
|
|
9
|
+
Project-URL: Issues, https://github.com/ChetanChirag24/rift/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/ChetanChirag24/rift/releases
|
|
11
|
+
Keywords: diff,data,csv,parquet,duckdb,data-quality,git
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Database
|
|
24
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: duckdb>=0.10.0
|
|
29
|
+
Requires-Dist: rich>=13.0.0
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
<p align="center">
|
|
35
|
+
<img src="https://raw.githubusercontent.com/ChetanChirag24/rift/719315b1c043077b447e6c6ff89f3b1db84befa5/docs/assets/hero.svg" alt="rift: two stacked data layers, before and after, with a scanner finding added, removed and changed rows" width="100%">
|
|
36
|
+
</p>
|
|
37
|
+
|
|
38
|
+
<p align="center">
|
|
39
|
+
<a href="https://github.com/ChetanChirag24/rift/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/ChetanChirag24/rift/actions/workflows/ci.yml/badge.svg"></a>
|
|
40
|
+
<img alt="Python 3.10+" src="https://img.shields.io/badge/python-3.10%2B-3776AB?logo=python&logoColor=white">
|
|
41
|
+
<img alt="Powered by DuckDB" src="https://img.shields.io/badge/powered%20by-DuckDB-FFF000?logo=duckdb&logoColor=black">
|
|
42
|
+
<img alt="Formats" src="https://img.shields.io/badge/formats-CSV%20%C2%B7%20Parquet%20%C2%B7%20JSON-22d3ee">
|
|
43
|
+
<img alt="Git aware" src="https://img.shields.io/badge/git-aware-F05032?logo=git&logoColor=white">
|
|
44
|
+
<img alt="MIT License" src="https://img.shields.io/badge/license-MIT-a78bfa">
|
|
45
|
+
</p>
|
|
46
|
+
|
|
47
|
+
<p align="center"><b>Think <code>git diff</code>, but for tables.</b><br>
|
|
48
|
+
Point rift at two versions of a dataset and it tells you which rows were <b>added</b>, which were <b>removed</b>, and exactly which values <b>changed</b>, all on your own machine.</p>
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## Why rift?
|
|
53
|
+
|
|
54
|
+
Code changes get reviewed line by line. **Data changes usually don't.** A CSV export is regenerated, a SQL model is refactored, a price list is updated, and the only check is "the row count looks about right."
|
|
55
|
+
|
|
56
|
+
rift shows you what actually changed:
|
|
57
|
+
|
|
58
|
+
- π **Row-level answers.** Not just "3 rows differ": you see *which* rows and *which* columns, with before β after values.
|
|
59
|
+
- π **Local-first.** Your data never leaves your laptop. No server, no account, no upload.
|
|
60
|
+
- β‘ **Fast.** Both files are loaded into an in-memory [DuckDB](https://duckdb.org) database and compared with SQL, so millions of rows are fine.
|
|
61
|
+
- πΏ **Git-aware.** Compare a file with any earlier commit: `rift HEAD~1:data.csv data.csv -k id`.
|
|
62
|
+
- π€ **CI-ready.** Exits like `diff`: `0` = identical, `1` = different, `2` = error. Add `--json` for machine-readable output.
|
|
63
|
+
|
|
64
|
+
## See it in action
|
|
65
|
+
|
|
66
|
+
<p align="center">
|
|
67
|
+
<img src="https://raw.githubusercontent.com/ChetanChirag24/rift/719315b1c043077b447e6c6ff89f3b1db84befa5/docs/assets/terminal.svg" alt="Terminal: rift compares data/customers.csv with its previous git commit and prints added, removed and changed rows" width="100%">
|
|
68
|
+
</p>
|
|
69
|
+
|
|
70
|
+
## How it works
|
|
71
|
+
|
|
72
|
+
<p align="center">
|
|
73
|
+
<img src="https://raw.githubusercontent.com/ChetanChirag24/rift/719315b1c043077b447e6c6ff89f3b1db84befa5/docs/assets/how-it-works.svg" alt="Diagram: two files (one read from git) flow into a DuckDB cube which joins them on the key, then out to a rich table, JSON and an exit code" width="100%">
|
|
74
|
+
</p>
|
|
75
|
+
|
|
76
|
+
1. **Load.** rift reads each side into DuckDB. Either side can be a CSV, Parquet or JSON file, or a version of a file from git history (`REV:path`).
|
|
77
|
+
2. **Check the key.** The key column(s) you pass must exist on both sides and be unique and non-empty, so every row has exactly one partner.
|
|
78
|
+
3. **Compare.** rift joins the two tables on the key:
|
|
79
|
+
- keys only in *after* β **added**
|
|
80
|
+
- keys only in *before* β **removed**
|
|
81
|
+
- keys on both sides where any column `IS DISTINCT FROM` its partner β **changed** (so `NULL` vs `NULL` correctly counts as equal)
|
|
82
|
+
4. **Report.** You get a colour table in the terminal, JSON with `--json`, and an exit code your scripts can act on. Added, removed and retyped columns are reported too.
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## Real-world examples
|
|
87
|
+
|
|
88
|
+
### π§Ύ "Did last night's CRM export silently drop customers?"
|
|
89
|
+
|
|
90
|
+
Your finance dashboard is fed by a nightly export. Today's revenue number looks odd.
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
rift exports/2026-11-15/customers.csv exports/2026-11-16/customers.csv -k customer_id
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
```text
|
|
97
|
+
Removed -1 β Initech vanished from the export
|
|
98
|
+
Changed ~2 β Globex: status active β churned, revenue 8500 β 0
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Two minutes later you know whether it's a real churn or a broken export.
|
|
102
|
+
|
|
103
|
+
### π οΈ "I refactored a SQL model. Is the output exactly the same?"
|
|
104
|
+
|
|
105
|
+
You rewrote a 400-line query to make it faster. The output should not change at all.
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
rift old_model.parquet new_model.parquet -k order_id --ignore _loaded_at
|
|
109
|
+
echo $? # 0 β identical, ship it. 1 β look at the diff first.
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
`--ignore` skips columns that are *supposed* to differ, like load timestamps.
|
|
113
|
+
|
|
114
|
+
### π "Which prices am I about to change in the store?"
|
|
115
|
+
|
|
116
|
+
Merchandising sent a new price sheet. Before you upload it, check exactly what it changes:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
rift prices_live.csv prices_new.csv -k sku --summary
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
```text
|
|
123
|
+
Changed values by column
|
|
124
|
+
column rows changed
|
|
125
|
+
price 412
|
|
126
|
+
sale_price 37
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
412 price changes when you expected 40? Better to catch that now than after customers do.
|
|
130
|
+
|
|
131
|
+
### π€ "Why did my model's accuracy drop between training runs?"
|
|
132
|
+
|
|
133
|
+
Diff the feature tables from both runs, keyed on a composite key, and pipe the JSON into `jq`:
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
rift features_run41.parquet features_run42.parquet -k user_id,event_date --json \
|
|
137
|
+
| jq '.column_change_counts'
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
```json
|
|
141
|
+
{ "avg_session_minutes": 120394, "country": 3 }
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
One feature changed for 120k users. That's your bug.
|
|
145
|
+
|
|
146
|
+
### π Multi-region tables with composite keys
|
|
147
|
+
|
|
148
|
+
The same `id` can exist in several regions? Combine columns into one key:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
rift before.csv after.csv -k region,id
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
---
|
|
155
|
+
|
|
156
|
+
## Works with Git
|
|
157
|
+
|
|
158
|
+
Data files live in git repos too, and rift treats git history as a first-class source.
|
|
159
|
+
|
|
160
|
+
### Compare with any commit, branch or tag
|
|
161
|
+
|
|
162
|
+
Use git's own `REV:path` syntax (paths are relative to the repo root, or start with `./` for the current directory):
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
rift HEAD~1:data/customers.csv data/customers.csv -k customer_id # what did my last commit change?
|
|
166
|
+
rift main:data/customers.csv data/customers.csv -k customer_id # what does my branch change vs main?
|
|
167
|
+
rift v1.0:exports/users.parquet v2.0:exports/users.parquet -k id # between two releases
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### Use rift as your `git difftool`
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
git config --global difftool.rift.cmd 'rift "$LOCAL" "$REMOTE" --key id'
|
|
174
|
+
git difftool -y -t rift HEAD~1 -- data/customers.csv
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
### Catch data changes in pull requests
|
|
178
|
+
|
|
179
|
+
```mermaid
|
|
180
|
+
flowchart LR
|
|
181
|
+
A[π©βπ» Pull request<br/>changes data/] --> B[βοΈ CI runs<br/>rift main:data.csv data.csv]
|
|
182
|
+
B -->|exit 0| C[β
Data unchanged<br/>merge freely]
|
|
183
|
+
B -->|exit 1| D[β οΈ Data changed<br/>diff printed in the log]
|
|
184
|
+
B -->|exit 2| E[β Bad key or file<br/>fix the input]
|
|
185
|
+
D --> F[π Reviewer checks<br/>the row-level diff]
|
|
186
|
+
F --> C
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
A minimal GitHub Actions job:
|
|
190
|
+
|
|
191
|
+
```yaml
|
|
192
|
+
name: data-diff
|
|
193
|
+
on: pull_request
|
|
194
|
+
|
|
195
|
+
jobs:
|
|
196
|
+
rift:
|
|
197
|
+
runs-on: ubuntu-latest
|
|
198
|
+
steps:
|
|
199
|
+
- uses: actions/checkout@v7
|
|
200
|
+
with:
|
|
201
|
+
fetch-depth: 0 # rift needs the main branch history
|
|
202
|
+
- uses: actions/setup-python@v7
|
|
203
|
+
with:
|
|
204
|
+
python-version: "3.12"
|
|
205
|
+
- run: pip install riftdiff
|
|
206
|
+
- name: Show what this PR changes in the data
|
|
207
|
+
run: rift origin/main:data/customers.csv data/customers.csv -k customer_id
|
|
208
|
+
continue-on-error: true # report the diff without failing the build
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
Remove `continue-on-error` if any data change should block the merge until someone signs off.
|
|
212
|
+
|
|
213
|
+
---
|
|
214
|
+
|
|
215
|
+
## Install
|
|
216
|
+
|
|
217
|
+
```bash
|
|
218
|
+
pip install riftdiff
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
Or from source:
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
git clone https://github.com/ChetanChirag24/rift.git && cd rift
|
|
225
|
+
python3 -m venv .venv && source .venv/bin/activate
|
|
226
|
+
pip install -e '.[dev]'
|
|
227
|
+
pytest
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
> **macOS + iCloud Desktop/Documents:** iCloud can mark files inside `.venv` as hidden, and Python then skips the `.pth` file that makes the editable install work (`ModuleNotFoundError: No module named 'riftdiff'`). Create the venv as `.venv.nosync` (iCloud ignores `*.nosync`) and symlink it: `python3 -m venv .venv.nosync && ln -s .venv.nosync .venv`.
|
|
231
|
+
|
|
232
|
+
## Usage
|
|
233
|
+
|
|
234
|
+
```
|
|
235
|
+
rift BEFORE AFTER --key COLUMN [options]
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
`BEFORE` and `AFTER` are file paths or git revisions (`HEAD~1:path/to/file.csv`).
|
|
239
|
+
|
|
240
|
+
| Option | What it does |
|
|
241
|
+
| --- | --- |
|
|
242
|
+
| `-k, --key COLUMN` | Column that identifies a row. Repeat it or comma-separate it for a composite key: `-k region,id`. |
|
|
243
|
+
| `-i, --ignore COLUMN` | Leave a column out of the comparison, e.g. `-i updated_at`. Repeatable / comma-separated. |
|
|
244
|
+
| `-n, --limit N` | Show at most N rows per section (default 10; `0` shows all). Counts are always exact. |
|
|
245
|
+
| `--summary` | Print only the counts, schema changes and per-column totals. |
|
|
246
|
+
| `--json` | Print the full diff as JSON for scripts and CI. |
|
|
247
|
+
| `--version` | Print the version. |
|
|
248
|
+
|
|
249
|
+
**Supported formats:** CSV/TSV (`.csv`, `.tsv`, `.txt`), Parquet (`.parquet`, `.pq`) and JSON (`.json`, `.jsonl`, `.ndjson`), optionally compressed with `.gz` or `.zst`. The two sides may be different formats.
|
|
250
|
+
|
|
251
|
+
### What rift reports
|
|
252
|
+
|
|
253
|
+
| | Meaning |
|
|
254
|
+
| --- | --- |
|
|
255
|
+
| π’ **Added** | Keys that exist only in `AFTER` |
|
|
256
|
+
| π΄ **Removed** | Keys that exist only in `BEFORE` |
|
|
257
|
+
| π‘ **Changed** | Keys on both sides where at least one value differs, with before β after for each changed column |
|
|
258
|
+
| π **By column** | How many rows changed in each column |
|
|
259
|
+
| 𧬠**Schema** | Columns added, removed, or whose type changed. Retyped columns are compared by their text form, so `10` and `"10"` count as equal. |
|
|
260
|
+
|
|
261
|
+
rift refuses keys that are missing, empty or not unique, because a diff on those would be misleading.
|
|
262
|
+
|
|
263
|
+
### Exit status
|
|
264
|
+
|
|
265
|
+
| Code | Meaning |
|
|
266
|
+
| --- | --- |
|
|
267
|
+
| `0` | No differences |
|
|
268
|
+
| `1` | Differences found |
|
|
269
|
+
| `2` | Error (missing file, bad key, unreadable dataβ¦) |
|
|
270
|
+
|
|
271
|
+
## Python API
|
|
272
|
+
|
|
273
|
+
```python
|
|
274
|
+
from riftdiff import diff
|
|
275
|
+
|
|
276
|
+
result = diff("HEAD~1:data/orders.csv", "data/orders.parquet",
|
|
277
|
+
key_column=["region", "order_id"],
|
|
278
|
+
ignore_columns=["updated_at"], limit=100)
|
|
279
|
+
|
|
280
|
+
result.summary() # {'rows_before': ..., 'rows_added': ..., ...}
|
|
281
|
+
result.added # list of row dicts
|
|
282
|
+
result.removed # list of row dicts
|
|
283
|
+
for change in result.changed:
|
|
284
|
+
print(change.key, change.changes) # {'region': 'EU', 'order_id': 7} {'total': (120, 135)}
|
|
285
|
+
result.has_differences # bool
|
|
286
|
+
result.to_dict() # JSON-ready dict
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
Problems with the input raise `riftdiff.RiftError` with a readable message.
|
|
290
|
+
|
|
291
|
+
## Try it now
|
|
292
|
+
|
|
293
|
+
The repo includes a small example:
|
|
294
|
+
|
|
295
|
+
```bash
|
|
296
|
+
rift examples/customers_before.csv examples/customers_after.csv -k customer_id
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
## License
|
|
300
|
+
|
|
301
|
+
[MIT](https://github.com/ChetanChirag24/rift/blob/719315b1c043077b447e6c6ff89f3b1db84befa5/LICENSE)
|
riftdiff-0.1.0/README.md
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/ChetanChirag24/rift/719315b1c043077b447e6c6ff89f3b1db84befa5/docs/assets/hero.svg" alt="rift: two stacked data layers, before and after, with a scanner finding added, removed and changed rows" width="100%">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center">
|
|
6
|
+
<a href="https://github.com/ChetanChirag24/rift/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/ChetanChirag24/rift/actions/workflows/ci.yml/badge.svg"></a>
|
|
7
|
+
<img alt="Python 3.10+" src="https://img.shields.io/badge/python-3.10%2B-3776AB?logo=python&logoColor=white">
|
|
8
|
+
<img alt="Powered by DuckDB" src="https://img.shields.io/badge/powered%20by-DuckDB-FFF000?logo=duckdb&logoColor=black">
|
|
9
|
+
<img alt="Formats" src="https://img.shields.io/badge/formats-CSV%20%C2%B7%20Parquet%20%C2%B7%20JSON-22d3ee">
|
|
10
|
+
<img alt="Git aware" src="https://img.shields.io/badge/git-aware-F05032?logo=git&logoColor=white">
|
|
11
|
+
<img alt="MIT License" src="https://img.shields.io/badge/license-MIT-a78bfa">
|
|
12
|
+
</p>
|
|
13
|
+
|
|
14
|
+
<p align="center"><b>Think <code>git diff</code>, but for tables.</b><br>
|
|
15
|
+
Point rift at two versions of a dataset and it tells you which rows were <b>added</b>, which were <b>removed</b>, and exactly which values <b>changed</b>, all on your own machine.</p>
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## Why rift?
|
|
20
|
+
|
|
21
|
+
Code changes get reviewed line by line. **Data changes usually don't.** A CSV export is regenerated, a SQL model is refactored, a price list is updated, and the only check is "the row count looks about right."
|
|
22
|
+
|
|
23
|
+
rift shows you what actually changed:
|
|
24
|
+
|
|
25
|
+
- π **Row-level answers.** Not just "3 rows differ": you see *which* rows and *which* columns, with before β after values.
|
|
26
|
+
- π **Local-first.** Your data never leaves your laptop. No server, no account, no upload.
|
|
27
|
+
- β‘ **Fast.** Both files are loaded into an in-memory [DuckDB](https://duckdb.org) database and compared with SQL, so millions of rows are fine.
|
|
28
|
+
- πΏ **Git-aware.** Compare a file with any earlier commit: `rift HEAD~1:data.csv data.csv -k id`.
|
|
29
|
+
- π€ **CI-ready.** Exits like `diff`: `0` = identical, `1` = different, `2` = error. Add `--json` for machine-readable output.
|
|
30
|
+
|
|
31
|
+
## See it in action
|
|
32
|
+
|
|
33
|
+
<p align="center">
|
|
34
|
+
<img src="https://raw.githubusercontent.com/ChetanChirag24/rift/719315b1c043077b447e6c6ff89f3b1db84befa5/docs/assets/terminal.svg" alt="Terminal: rift compares data/customers.csv with its previous git commit and prints added, removed and changed rows" width="100%">
|
|
35
|
+
</p>
|
|
36
|
+
|
|
37
|
+
## How it works
|
|
38
|
+
|
|
39
|
+
<p align="center">
|
|
40
|
+
<img src="https://raw.githubusercontent.com/ChetanChirag24/rift/719315b1c043077b447e6c6ff89f3b1db84befa5/docs/assets/how-it-works.svg" alt="Diagram: two files (one read from git) flow into a DuckDB cube which joins them on the key, then out to a rich table, JSON and an exit code" width="100%">
|
|
41
|
+
</p>
|
|
42
|
+
|
|
43
|
+
1. **Load.** rift reads each side into DuckDB. Either side can be a CSV, Parquet or JSON file, or a version of a file from git history (`REV:path`).
|
|
44
|
+
2. **Check the key.** The key column(s) you pass must exist on both sides and be unique and non-empty, so every row has exactly one partner.
|
|
45
|
+
3. **Compare.** rift joins the two tables on the key:
|
|
46
|
+
- keys only in *after* β **added**
|
|
47
|
+
- keys only in *before* β **removed**
|
|
48
|
+
- keys on both sides where any column `IS DISTINCT FROM` its partner β **changed** (so `NULL` vs `NULL` correctly counts as equal)
|
|
49
|
+
4. **Report.** You get a colour table in the terminal, JSON with `--json`, and an exit code your scripts can act on. Added, removed and retyped columns are reported too.
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Real-world examples
|
|
54
|
+
|
|
55
|
+
### π§Ύ "Did last night's CRM export silently drop customers?"
|
|
56
|
+
|
|
57
|
+
Your finance dashboard is fed by a nightly export. Today's revenue number looks odd.
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
rift exports/2026-11-15/customers.csv exports/2026-11-16/customers.csv -k customer_id
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
```text
|
|
64
|
+
Removed -1 β Initech vanished from the export
|
|
65
|
+
Changed ~2 β Globex: status active β churned, revenue 8500 β 0
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Two minutes later you know whether it's a real churn or a broken export.
|
|
69
|
+
|
|
70
|
+
### π οΈ "I refactored a SQL model. Is the output exactly the same?"
|
|
71
|
+
|
|
72
|
+
You rewrote a 400-line query to make it faster. The output should not change at all.
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
rift old_model.parquet new_model.parquet -k order_id --ignore _loaded_at
|
|
76
|
+
echo $? # 0 β identical, ship it. 1 β look at the diff first.
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
`--ignore` skips columns that are *supposed* to differ, like load timestamps.
|
|
80
|
+
|
|
81
|
+
### π "Which prices am I about to change in the store?"
|
|
82
|
+
|
|
83
|
+
Merchandising sent a new price sheet. Before you upload it, check exactly what it changes:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
rift prices_live.csv prices_new.csv -k sku --summary
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
```text
|
|
90
|
+
Changed values by column
|
|
91
|
+
column rows changed
|
|
92
|
+
price 412
|
|
93
|
+
sale_price 37
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
412 price changes when you expected 40? Better to catch that now than after customers do.
|
|
97
|
+
|
|
98
|
+
### π€ "Why did my model's accuracy drop between training runs?"
|
|
99
|
+
|
|
100
|
+
Diff the feature tables from both runs, keyed on a composite key, and pipe the JSON into `jq`:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
rift features_run41.parquet features_run42.parquet -k user_id,event_date --json \
|
|
104
|
+
| jq '.column_change_counts'
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
```json
|
|
108
|
+
{ "avg_session_minutes": 120394, "country": 3 }
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
One feature changed for 120k users. That's your bug.
|
|
112
|
+
|
|
113
|
+
### π Multi-region tables with composite keys
|
|
114
|
+
|
|
115
|
+
The same `id` can exist in several regions? Combine columns into one key:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
rift before.csv after.csv -k region,id
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
---
|
|
122
|
+
|
|
123
|
+
## Works with Git
|
|
124
|
+
|
|
125
|
+
Data files live in git repos too, and rift treats git history as a first-class source.
|
|
126
|
+
|
|
127
|
+
### Compare with any commit, branch or tag
|
|
128
|
+
|
|
129
|
+
Use git's own `REV:path` syntax (paths are relative to the repo root, or start with `./` for the current directory):
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
rift HEAD~1:data/customers.csv data/customers.csv -k customer_id # what did my last commit change?
|
|
133
|
+
rift main:data/customers.csv data/customers.csv -k customer_id # what does my branch change vs main?
|
|
134
|
+
rift v1.0:exports/users.parquet v2.0:exports/users.parquet -k id # between two releases
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
### Use rift as your `git difftool`
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
git config --global difftool.rift.cmd 'rift "$LOCAL" "$REMOTE" --key id'
|
|
141
|
+
git difftool -y -t rift HEAD~1 -- data/customers.csv
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
### Catch data changes in pull requests
|
|
145
|
+
|
|
146
|
+
```mermaid
|
|
147
|
+
flowchart LR
|
|
148
|
+
A[π©βπ» Pull request<br/>changes data/] --> B[βοΈ CI runs<br/>rift main:data.csv data.csv]
|
|
149
|
+
B -->|exit 0| C[β
Data unchanged<br/>merge freely]
|
|
150
|
+
B -->|exit 1| D[β οΈ Data changed<br/>diff printed in the log]
|
|
151
|
+
B -->|exit 2| E[β Bad key or file<br/>fix the input]
|
|
152
|
+
D --> F[π Reviewer checks<br/>the row-level diff]
|
|
153
|
+
F --> C
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
A minimal GitHub Actions job:
|
|
157
|
+
|
|
158
|
+
```yaml
|
|
159
|
+
name: data-diff
|
|
160
|
+
on: pull_request
|
|
161
|
+
|
|
162
|
+
jobs:
|
|
163
|
+
rift:
|
|
164
|
+
runs-on: ubuntu-latest
|
|
165
|
+
steps:
|
|
166
|
+
- uses: actions/checkout@v7
|
|
167
|
+
with:
|
|
168
|
+
fetch-depth: 0 # rift needs the main branch history
|
|
169
|
+
- uses: actions/setup-python@v7
|
|
170
|
+
with:
|
|
171
|
+
python-version: "3.12"
|
|
172
|
+
- run: pip install riftdiff
|
|
173
|
+
- name: Show what this PR changes in the data
|
|
174
|
+
run: rift origin/main:data/customers.csv data/customers.csv -k customer_id
|
|
175
|
+
continue-on-error: true # report the diff without failing the build
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Remove `continue-on-error` if any data change should block the merge until someone signs off.
|
|
179
|
+
|
|
180
|
+
---
|
|
181
|
+
|
|
182
|
+
## Install
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
pip install riftdiff
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
Or from source:
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
git clone https://github.com/ChetanChirag24/rift.git && cd rift
|
|
192
|
+
python3 -m venv .venv && source .venv/bin/activate
|
|
193
|
+
pip install -e '.[dev]'
|
|
194
|
+
pytest
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
> **macOS + iCloud Desktop/Documents:** iCloud can mark files inside `.venv` as hidden, and Python then skips the `.pth` file that makes the editable install work (`ModuleNotFoundError: No module named 'riftdiff'`). Create the venv as `.venv.nosync` (iCloud ignores `*.nosync`) and symlink it: `python3 -m venv .venv.nosync && ln -s .venv.nosync .venv`.
|
|
198
|
+
|
|
199
|
+
## Usage
|
|
200
|
+
|
|
201
|
+
```
|
|
202
|
+
rift BEFORE AFTER --key COLUMN [options]
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
`BEFORE` and `AFTER` are file paths or git revisions (`HEAD~1:path/to/file.csv`).
|
|
206
|
+
|
|
207
|
+
| Option | What it does |
|
|
208
|
+
| --- | --- |
|
|
209
|
+
| `-k, --key COLUMN` | Column that identifies a row. Repeat it or comma-separate it for a composite key: `-k region,id`. |
|
|
210
|
+
| `-i, --ignore COLUMN` | Leave a column out of the comparison, e.g. `-i updated_at`. Repeatable / comma-separated. |
|
|
211
|
+
| `-n, --limit N` | Show at most N rows per section (default 10; `0` shows all). Counts are always exact. |
|
|
212
|
+
| `--summary` | Print only the counts, schema changes and per-column totals. |
|
|
213
|
+
| `--json` | Print the full diff as JSON for scripts and CI. |
|
|
214
|
+
| `--version` | Print the version. |
|
|
215
|
+
|
|
216
|
+
**Supported formats:** CSV/TSV (`.csv`, `.tsv`, `.txt`), Parquet (`.parquet`, `.pq`) and JSON (`.json`, `.jsonl`, `.ndjson`), optionally compressed with `.gz` or `.zst`. The two sides may be different formats.
|
|
217
|
+
|
|
218
|
+
### What rift reports
|
|
219
|
+
|
|
220
|
+
| | Meaning |
|
|
221
|
+
| --- | --- |
|
|
222
|
+
| π’ **Added** | Keys that exist only in `AFTER` |
|
|
223
|
+
| π΄ **Removed** | Keys that exist only in `BEFORE` |
|
|
224
|
+
| π‘ **Changed** | Keys on both sides where at least one value differs, with before β after for each changed column |
|
|
225
|
+
| π **By column** | How many rows changed in each column |
|
|
226
|
+
| 𧬠**Schema** | Columns added, removed, or whose type changed. Retyped columns are compared by their text form, so `10` and `"10"` count as equal. |
|
|
227
|
+
|
|
228
|
+
rift refuses keys that are missing, empty or not unique, because a diff on those would be misleading.
|
|
229
|
+
|
|
230
|
+
### Exit status
|
|
231
|
+
|
|
232
|
+
| Code | Meaning |
|
|
233
|
+
| --- | --- |
|
|
234
|
+
| `0` | No differences |
|
|
235
|
+
| `1` | Differences found |
|
|
236
|
+
| `2` | Error (missing file, bad key, unreadable dataβ¦) |
|
|
237
|
+
|
|
238
|
+
## Python API
|
|
239
|
+
|
|
240
|
+
```python
|
|
241
|
+
from riftdiff import diff
|
|
242
|
+
|
|
243
|
+
result = diff("HEAD~1:data/orders.csv", "data/orders.parquet",
|
|
244
|
+
key_column=["region", "order_id"],
|
|
245
|
+
ignore_columns=["updated_at"], limit=100)
|
|
246
|
+
|
|
247
|
+
result.summary() # {'rows_before': ..., 'rows_added': ..., ...}
|
|
248
|
+
result.added # list of row dicts
|
|
249
|
+
result.removed # list of row dicts
|
|
250
|
+
for change in result.changed:
|
|
251
|
+
print(change.key, change.changes) # {'region': 'EU', 'order_id': 7} {'total': (120, 135)}
|
|
252
|
+
result.has_differences # bool
|
|
253
|
+
result.to_dict() # JSON-ready dict
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
Problems with the input raise `riftdiff.RiftError` with a readable message.
|
|
257
|
+
|
|
258
|
+
## Try it now
|
|
259
|
+
|
|
260
|
+
The repo includes a small example:
|
|
261
|
+
|
|
262
|
+
```bash
|
|
263
|
+
rift examples/customers_before.csv examples/customers_after.csv -k customer_id
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
## License
|
|
267
|
+
|
|
268
|
+
[MIT](https://github.com/ChetanChirag24/rift/blob/719315b1c043077b447e6c6ff89f3b1db84befa5/LICENSE)
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "riftdiff"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Local-first data diff, powered by DuckDB. See exactly what changed before you merge."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Chetan Chirag" }]
|
|
14
|
+
keywords = ["diff", "data", "csv", "parquet", "duckdb", "data-quality", "git"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Programming Language :: Python :: 3.14",
|
|
27
|
+
"Topic :: Database",
|
|
28
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
29
|
+
]
|
|
30
|
+
dependencies = [
|
|
31
|
+
"duckdb>=0.10.0",
|
|
32
|
+
"rich>=13.0.0",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
[project.urls]
|
|
36
|
+
Homepage = "https://github.com/ChetanChirag24/rift"
|
|
37
|
+
Source = "https://github.com/ChetanChirag24/rift"
|
|
38
|
+
Issues = "https://github.com/ChetanChirag24/rift/issues"
|
|
39
|
+
Changelog = "https://github.com/ChetanChirag24/rift/releases"
|
|
40
|
+
|
|
41
|
+
[project.optional-dependencies]
|
|
42
|
+
dev = ["pytest>=7"]
|
|
43
|
+
|
|
44
|
+
[project.scripts]
|
|
45
|
+
rift = "riftdiff.cli:main"
|
|
46
|
+
|
|
47
|
+
[tool.setuptools.dynamic]
|
|
48
|
+
version = { attr = "riftdiff.__version__" }
|
|
49
|
+
|
|
50
|
+
[tool.setuptools.packages.find]
|
|
51
|
+
where = ["src"]
|
|
52
|
+
|
|
53
|
+
[tool.pytest.ini_options]
|
|
54
|
+
testpaths = ["tests"]
|
|
55
|
+
pythonpath = ["src"]
|
riftdiff-0.1.0/setup.cfg
ADDED