riftdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
riftdiff/__init__.py ADDED
@@ -0,0 +1,7 @@
1
+ """rift: local-first data diff, powered by DuckDB."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+ from .engine import DiffResult, RiftError, RowChange, diff # noqa: E402
6
+
7
+ __all__ = ["diff", "DiffResult", "RowChange", "RiftError", "__version__"]
riftdiff/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
riftdiff/cli.py ADDED
@@ -0,0 +1,182 @@
1
+ """Command-line interface: ``rift BEFORE AFTER --key COLUMN``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import sys
8
+ from typing import Any, List, Optional, Sequence
9
+
10
+ from rich.console import Console
11
+ from rich.table import Table
12
+ from rich.text import Text
13
+
14
+ from . import __version__
15
+ from .engine import DiffResult, RiftError, diff
16
+
17
+ EXIT_SAME = 0
18
+ EXIT_DIFFERENT = 1
19
+ EXIT_ERROR = 2
20
+
21
+
22
+ def _split_columns(values: Optional[List[str]]) -> List[str]:
23
+ """Accept both ``-k a -k b`` and ``-k a,b``."""
24
+ out: List[str] = []
25
+ for v in values or []:
26
+ out.extend(part.strip() for part in v.split(",") if part.strip())
27
+ return out
28
+
29
+
30
+ def build_parser() -> argparse.ArgumentParser:
31
+ parser = argparse.ArgumentParser(
32
+ prog="rift",
33
+ description="Local-first data diff, powered by DuckDB. "
34
+ "See exactly what changed before you merge.",
35
+ epilog="Exit status: 0 if the files match, 1 if they differ, 2 on error.",
36
+ )
37
+ parser.add_argument(
38
+ "before",
39
+ help="the original file (CSV, Parquet or JSON), or a git revision like HEAD~1:data/users.csv",
40
+ )
41
+ parser.add_argument("after", help="the new file (or git revision) to compare against it")
42
+ parser.add_argument(
43
+ "-k", "--key", action="append", required=True, metavar="COLUMN",
44
+ help="column(s) that identify a row; repeat or comma-separate for a composite key",
45
+ )
46
+ parser.add_argument(
47
+ "-i", "--ignore", action="append", metavar="COLUMN",
48
+ help="column(s) to leave out of the comparison (e.g. updated_at)",
49
+ )
50
+ parser.add_argument(
51
+ "-n", "--limit", type=int, default=10, metavar="N",
52
+ help="show at most N rows per section (default: 10, 0 for all)",
53
+ )
54
+ parser.add_argument(
55
+ "--summary", action="store_true", help="print only the summary, no row details",
56
+ )
57
+ parser.add_argument(
58
+ "--json", action="store_true", help="print the full diff as JSON (for scripts and CI)",
59
+ )
60
+ parser.add_argument("--version", action="version", version=f"rift {__version__}")
61
+ return parser
62
+
63
+
64
+ def _fmt(value: Any) -> str:
65
+ return "βˆ…" if value is None else str(value)
66
+
67
+
68
+ def _rows_table(title: str, rows: List[dict], total: int, style: str) -> Table:
69
+ columns = list(rows[0].keys())
70
+ table = Table(title=title, title_style=f"bold {style}", title_justify="left")
71
+ for c in columns:
72
+ table.add_column(c, style=style)
73
+ for row in rows:
74
+ table.add_row(*(_fmt(row[c]) for c in columns))
75
+ if total > len(rows):
76
+ table.caption = f"… and {total - len(rows)} more"
77
+ return table
78
+
79
+
80
+ def render(result: DiffResult, console: Console, summary_only: bool = False) -> None:
81
+ console.print(
82
+ Text.assemble(
83
+ ("rift ", "bold magenta"), (result.before_source, "cyan"), " β†’ ",
84
+ (result.after_source, "cyan"), (" key: ", "dim"), (", ".join(result.key_columns), "bold"),
85
+ )
86
+ )
87
+ console.print()
88
+
89
+ summary = Table(show_header=False, box=None, padding=(0, 2))
90
+ summary.add_column(justify="left")
91
+ summary.add_column(justify="right")
92
+ summary.add_row("Rows before", f"{result.rows_before:,}")
93
+ summary.add_row("Rows after", f"{result.rows_after:,}")
94
+ summary.add_row(Text("Added", style="green"), Text(f"+{result.rows_added:,}", style="green"))
95
+ summary.add_row(Text("Removed", style="red"), Text(f"-{result.rows_removed:,}", style="red"))
96
+ summary.add_row(Text("Changed", style="yellow"), Text(f"~{result.rows_changed:,}", style="yellow"))
97
+ summary.add_row(Text("Unchanged", style="dim"), Text(f"{result.rows_unchanged:,}", style="dim"))
98
+ console.print(summary)
99
+ console.print()
100
+
101
+ if result.has_schema_changes:
102
+ console.print("[bold]Schema changes[/bold]")
103
+ for c in result.columns_added:
104
+ console.print(f" [green]+ column {c}[/green]")
105
+ for c in result.columns_removed:
106
+ console.print(f" [red]- column {c}[/red]")
107
+ for c, (old, new) in result.type_changes.items():
108
+ console.print(f" [yellow]~ column {c}: {old} β†’ {new}[/yellow]")
109
+ console.print()
110
+
111
+ if result.column_change_counts:
112
+ cols = Table(title="Changed values by column", title_style="bold", title_justify="left")
113
+ cols.add_column("column")
114
+ cols.add_column("rows changed", justify="right", style="yellow")
115
+ for c, n in sorted(result.column_change_counts.items(), key=lambda kv: -kv[1]):
116
+ cols.add_row(c, f"{n:,}")
117
+ console.print(cols)
118
+ console.print()
119
+
120
+ if summary_only:
121
+ return
122
+
123
+ if result.added:
124
+ console.print(_rows_table("Added rows", result.added, result.rows_added, "green"))
125
+ console.print()
126
+ if result.removed:
127
+ console.print(_rows_table("Removed rows", result.removed, result.rows_removed, "red"))
128
+ console.print()
129
+ if result.changed:
130
+ table = Table(title="Changed rows", title_style="bold yellow", title_justify="left")
131
+ for k in result.key_columns:
132
+ table.add_column(k, style="bold")
133
+ table.add_column("column")
134
+ table.add_column("before", style="red")
135
+ table.add_column("after", style="green")
136
+ for rc in result.changed:
137
+ first = True
138
+ for c, (old, new) in rc.changes.items():
139
+ key_cells = [_fmt(v) for v in rc.key.values()] if first else [""] * len(rc.key)
140
+ table.add_row(*key_cells, c, _fmt(old), _fmt(new))
141
+ first = False
142
+ table.add_section()
143
+ if result.rows_changed > len(result.changed):
144
+ table.caption = f"… and {result.rows_changed - len(result.changed)} more"
145
+ console.print(table)
146
+ console.print()
147
+
148
+ if not result.has_differences:
149
+ console.print("[bold green]βœ“ No differences.[/bold green]")
150
+
151
+
152
+ def main(argv: Optional[Sequence[str]] = None) -> int:
153
+ args = build_parser().parse_args(argv)
154
+ keys = _split_columns(args.key)
155
+ ignore = _split_columns(args.ignore)
156
+ if args.limit < 0:
157
+ print("rift: error: --limit must be 0 or more", file=sys.stderr)
158
+ return EXIT_ERROR
159
+ limit = None if (args.limit == 0 or args.json) else args.limit
160
+ if args.summary and not args.json:
161
+ limit = 0
162
+
163
+ try:
164
+ result = diff(args.before, args.after, keys, ignore_columns=ignore, limit=limit)
165
+ except RiftError as exc:
166
+ Console(stderr=True).print(f"[bold red]rift: error:[/bold red] {exc}", highlight=False)
167
+ return EXIT_ERROR
168
+
169
+ if args.json:
170
+ payload = result.to_dict()
171
+ if args.summary:
172
+ for section in ("added", "removed", "changed"):
173
+ payload.pop(section)
174
+ print(json.dumps(payload, indent=2, default=str))
175
+ else:
176
+ render(result, Console(), summary_only=args.summary)
177
+
178
+ return EXIT_DIFFERENT if result.has_differences else EXIT_SAME
179
+
180
+
181
+ if __name__ == "__main__": # pragma: no cover
182
+ sys.exit(main())
riftdiff/engine.py ADDED
@@ -0,0 +1,321 @@
1
+ """Core diff engine: load two datasets into DuckDB and compare them by key."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import subprocess
6
+ import tempfile
7
+ from dataclasses import dataclass, field
8
+ from pathlib import Path
9
+ from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
10
+
11
+ import duckdb
12
+
13
+ CSV_SUFFIXES = {".csv", ".tsv", ".txt"}
14
+ PARQUET_SUFFIXES = {".parquet", ".pq"}
15
+ JSON_SUFFIXES = {".json", ".jsonl", ".ndjson"}
16
+ COMPRESSION_SUFFIXES = {".gz", ".zst"}
17
+
18
+
19
+ class RiftError(Exception):
20
+ """Raised for user-facing problems: missing files, bad keys, duplicate keys."""
21
+
22
+
23
+ @dataclass
24
+ class RowChange:
25
+ """One row that exists on both sides but whose values differ."""
26
+
27
+ key: Dict[str, Any]
28
+ changes: Dict[str, Tuple[Any, Any]] # column -> (before, after)
29
+
30
+
31
+ @dataclass
32
+ class DiffResult:
33
+ before_source: str
34
+ after_source: str
35
+ key_columns: List[str]
36
+ rows_before: int
37
+ rows_after: int
38
+ rows_added: int
39
+ rows_removed: int
40
+ rows_changed: int
41
+ columns_added: List[str] = field(default_factory=list)
42
+ columns_removed: List[str] = field(default_factory=list)
43
+ type_changes: Dict[str, Tuple[str, str]] = field(default_factory=dict)
44
+ compared_columns: List[str] = field(default_factory=list)
45
+ column_change_counts: Dict[str, int] = field(default_factory=dict)
46
+ added: List[Dict[str, Any]] = field(default_factory=list)
47
+ removed: List[Dict[str, Any]] = field(default_factory=list)
48
+ changed: List[RowChange] = field(default_factory=list)
49
+
50
+ @property
51
+ def rows_unchanged(self) -> int:
52
+ return self.rows_before - self.rows_removed - self.rows_changed
53
+
54
+ @property
55
+ def has_schema_changes(self) -> bool:
56
+ return bool(self.columns_added or self.columns_removed or self.type_changes)
57
+
58
+ @property
59
+ def has_differences(self) -> bool:
60
+ return bool(
61
+ self.rows_added or self.rows_removed or self.rows_changed or self.has_schema_changes
62
+ )
63
+
64
+ def summary(self) -> Dict[str, int]:
65
+ return {
66
+ "rows_before": self.rows_before,
67
+ "rows_after": self.rows_after,
68
+ "rows_added": self.rows_added,
69
+ "rows_removed": self.rows_removed,
70
+ "rows_changed": self.rows_changed,
71
+ "rows_unchanged": self.rows_unchanged,
72
+ }
73
+
74
+ def to_dict(self) -> Dict[str, Any]:
75
+ return {
76
+ "before": self.before_source,
77
+ "after": self.after_source,
78
+ "key": self.key_columns,
79
+ "summary": self.summary(),
80
+ "schema": {
81
+ "columns_added": self.columns_added,
82
+ "columns_removed": self.columns_removed,
83
+ "type_changes": {
84
+ c: {"before": b, "after": a} for c, (b, a) in self.type_changes.items()
85
+ },
86
+ },
87
+ "column_change_counts": self.column_change_counts,
88
+ "added": self.added,
89
+ "removed": self.removed,
90
+ "changed": [
91
+ {
92
+ "key": rc.key,
93
+ "changes": {c: {"before": b, "after": a} for c, (b, a) in rc.changes.items()},
94
+ }
95
+ for rc in self.changed
96
+ ],
97
+ }
98
+
99
+
100
+ def quote(identifier: str) -> str:
101
+ """Quote a SQL identifier so any column name is safe to interpolate."""
102
+ return '"' + identifier.replace('"', '""') + '"'
103
+
104
+
105
+ def _detect_format(path: Path) -> str:
106
+ suffixes = [s.lower() for s in path.suffixes]
107
+ while suffixes and suffixes[-1] in COMPRESSION_SUFFIXES:
108
+ suffixes.pop()
109
+ ext = suffixes[-1] if suffixes else ""
110
+ if ext in CSV_SUFFIXES:
111
+ return "csv"
112
+ if ext in PARQUET_SUFFIXES:
113
+ return "parquet"
114
+ if ext in JSON_SUFFIXES:
115
+ return "json"
116
+ raise RiftError(
117
+ f"Can't tell the file format of '{path}'. "
118
+ "Supported: .csv, .tsv, .txt, .parquet, .json, .jsonl, .ndjson (optionally .gz/.zst)."
119
+ )
120
+
121
+
122
+ def _git_revision_file(source: str, workdir: Path) -> Optional[Path]:
123
+ """Resolve git's ``REV:path`` syntax (e.g. ``HEAD~1:data/users.csv``) to a temp file.
124
+
125
+ Returns None if ``source`` doesn't look like a git object reference.
126
+ """
127
+ rev, sep, path = source.partition(":")
128
+ if not sep or not rev or not path or len(rev) == 1: # len 1 => Windows drive letter
129
+ return None
130
+ try:
131
+ blob = subprocess.run(
132
+ ["git", "show", source], capture_output=True, check=False, timeout=60
133
+ )
134
+ except (OSError, subprocess.SubprocessError) as exc:
135
+ raise RiftError(f"Could not run git to read '{source}': {exc}") from exc
136
+ if blob.returncode != 0:
137
+ message = blob.stderr.decode(errors="replace").strip()
138
+ raise RiftError(f"git could not read '{source}': {message}")
139
+ target = workdir / f"{len(list(workdir.iterdir()))}_{Path(path).name}"
140
+ target.write_bytes(blob.stdout)
141
+ return target
142
+
143
+
144
+ def _load(
145
+ con: duckdb.DuckDBPyConnection, source: Union[str, Path], table: str, workdir: Path
146
+ ) -> None:
147
+ path = Path(source)
148
+ if not path.exists():
149
+ path = _git_revision_file(str(source), workdir) or path
150
+ if not path.exists():
151
+ raise RiftError(f"File not found: {source}")
152
+ fmt = _detect_format(path)
153
+ try:
154
+ if fmt == "csv":
155
+ rel = con.read_csv(str(path))
156
+ elif fmt == "parquet":
157
+ rel = con.read_parquet(str(path))
158
+ else:
159
+ rel = con.read_json(str(path))
160
+ rel.create(table)
161
+ except duckdb.Error as exc:
162
+ raise RiftError(f"Could not read '{path}': {exc}") from exc
163
+
164
+
165
+ def _schema(con: duckdb.DuckDBPyConnection, table: str) -> Dict[str, str]:
166
+ return {row[0]: row[1] for row in con.execute(f"DESCRIBE {table}").fetchall()}
167
+
168
+
169
+ def _rows_as_dicts(cursor: duckdb.DuckDBPyConnection) -> List[Dict[str, Any]]:
170
+ names = [d[0] for d in cursor.description]
171
+ return [dict(zip(names, row)) for row in cursor.fetchall()]
172
+
173
+
174
+ def _check_keys(con: duckdb.DuckDBPyConnection, table: str, label: str, keys: List[str]) -> None:
175
+ key_list = ", ".join(quote(k) for k in keys)
176
+
177
+ null_cond = " OR ".join(f"{quote(k)} IS NULL" for k in keys)
178
+ nulls = con.execute(f"SELECT count(*) FROM {table} WHERE {null_cond}").fetchone()[0]
179
+ if nulls:
180
+ raise RiftError(f"{nulls} row(s) in the {label} file have an empty key ({', '.join(keys)}).")
181
+
182
+ dup = con.execute(
183
+ f"SELECT {key_list}, count(*) AS n FROM {table} "
184
+ f"GROUP BY ALL HAVING count(*) > 1 ORDER BY n DESC LIMIT 1"
185
+ ).fetchone()
186
+ if dup:
187
+ example = ", ".join(f"{k}={v!r}" for k, v in zip(keys, dup[:-1]))
188
+ raise RiftError(
189
+ f"Key ({', '.join(keys)}) is not unique in the {label} file: "
190
+ f"{example} appears {dup[-1]} times. Pick a key that identifies each row, "
191
+ "or combine columns with a composite key."
192
+ )
193
+
194
+
195
+ def diff(
196
+ before_source: Union[str, Path],
197
+ after_source: Union[str, Path],
198
+ key_column: Union[str, Sequence[str]],
199
+ ignore_columns: Sequence[str] = (),
200
+ limit: Optional[int] = None,
201
+ ) -> DiffResult:
202
+ """Compare two datasets row by row, matched on ``key_column``.
203
+
204
+ Each source is a file path or a git object reference like
205
+ ``HEAD~1:data/users.csv`` (read with ``git show``).
206
+
207
+ ``key_column`` may be one column name or a list (composite key).
208
+ ``limit`` caps how many added/removed/changed rows are returned in detail;
209
+ counts are always exact.
210
+ """
211
+ keys = [key_column] if isinstance(key_column, str) else list(key_column)
212
+ if not keys:
213
+ raise RiftError("At least one key column is required.")
214
+
215
+ con = duckdb.connect()
216
+ with tempfile.TemporaryDirectory(prefix="rift-") as tmp:
217
+ _load(con, before_source, "before", Path(tmp))
218
+ _load(con, after_source, "after", Path(tmp))
219
+
220
+ schema_before = _schema(con, "before")
221
+ schema_after = _schema(con, "after")
222
+
223
+ for label, schema in (("before", schema_before), ("after", schema_after)):
224
+ missing = [k for k in keys if k not in schema]
225
+ if missing:
226
+ raise RiftError(
227
+ f"Key column(s) {', '.join(missing)} not found in the {label} file. "
228
+ f"Available columns: {', '.join(schema)}"
229
+ )
230
+
231
+ unknown_ignored = [c for c in ignore_columns if c not in schema_before and c not in schema_after]
232
+ if unknown_ignored:
233
+ raise RiftError(f"Ignored column(s) not found in either file: {', '.join(unknown_ignored)}")
234
+
235
+ _check_keys(con, "before", "before", keys)
236
+ _check_keys(con, "after", "after", keys)
237
+
238
+ ignored = set(ignore_columns) | set(keys)
239
+ columns_removed = [c for c in schema_before if c not in schema_after and c not in ignored]
240
+ columns_added = [c for c in schema_after if c not in schema_before and c not in ignored]
241
+ compared = [c for c in schema_before if c in schema_after and c not in ignored]
242
+ type_changes = {
243
+ c: (schema_before[c], schema_after[c])
244
+ for c in compared
245
+ if schema_before[c] != schema_after[c]
246
+ }
247
+
248
+ join_cond = " AND ".join(f"a.{quote(k)} = b.{quote(k)}" for k in keys)
249
+ order_by = ", ".join(quote(k) for k in keys)
250
+ limit_sql = f" LIMIT {int(limit)}" if limit is not None else ""
251
+
252
+ rows_before = con.execute("SELECT count(*) FROM before").fetchone()[0]
253
+ rows_after = con.execute("SELECT count(*) FROM after").fetchone()[0]
254
+
255
+ added_sql = f"FROM after a WHERE NOT EXISTS (SELECT 1 FROM before b WHERE {join_cond})"
256
+ removed_sql = f"FROM before b WHERE NOT EXISTS (SELECT 1 FROM after a WHERE {join_cond})"
257
+ rows_added = con.execute(f"SELECT count(*) {added_sql}").fetchone()[0]
258
+ rows_removed = con.execute(f"SELECT count(*) {removed_sql}").fetchone()[0]
259
+ added = _rows_as_dicts(con.execute(f"SELECT a.* {added_sql} ORDER BY {order_by}{limit_sql}"))
260
+ removed = _rows_as_dicts(con.execute(f"SELECT b.* {removed_sql} ORDER BY {order_by}{limit_sql}"))
261
+
262
+ def differs(c: str) -> str:
263
+ if c in type_changes:
264
+ # Different types: compare their text form so e.g. 1 vs '1' is equal.
265
+ return f"CAST(b.{quote(c)} AS VARCHAR) IS DISTINCT FROM CAST(a.{quote(c)} AS VARCHAR)"
266
+ return f"b.{quote(c)} IS DISTINCT FROM a.{quote(c)}"
267
+
268
+ column_change_counts: Dict[str, int] = {}
269
+ changed: List[RowChange] = []
270
+ rows_changed = 0
271
+
272
+ if compared:
273
+ any_diff = " OR ".join(f"({differs(c)})" for c in compared)
274
+ joined = f"FROM before b JOIN after a ON {join_cond}"
275
+
276
+ counts = con.execute(
277
+ "SELECT count(*) FILTER (WHERE " + any_diff + "), "
278
+ + ", ".join(f"count(*) FILTER (WHERE {differs(c)})" for c in compared)
279
+ + f" {joined}"
280
+ ).fetchone()
281
+ rows_changed = counts[0]
282
+ column_change_counts = {c: n for c, n in zip(compared, counts[1:]) if n}
283
+
284
+ select = [f"b.{quote(k)}" for k in keys]
285
+ for c in compared:
286
+ select += [f"b.{quote(c)}", f"a.{quote(c)}", f"({differs(c)})"]
287
+ b_order = ", ".join(f"b.{quote(k)}" for k in keys)
288
+ detail = con.execute(
289
+ f"SELECT {', '.join(select)} {joined} WHERE {any_diff} ORDER BY {b_order}{limit_sql}"
290
+ ).fetchall()
291
+
292
+ n = len(keys)
293
+ for row in detail:
294
+ key = dict(zip(keys, row[:n]))
295
+ changes = {}
296
+ for i, c in enumerate(compared):
297
+ old, new, is_diff = row[n + 3 * i : n + 3 * i + 3]
298
+ if is_diff:
299
+ changes[c] = (old, new)
300
+ changed.append(RowChange(key=key, changes=changes))
301
+
302
+ con.close()
303
+
304
+ return DiffResult(
305
+ before_source=str(before_source),
306
+ after_source=str(after_source),
307
+ key_columns=keys,
308
+ rows_before=rows_before,
309
+ rows_after=rows_after,
310
+ rows_added=rows_added,
311
+ rows_removed=rows_removed,
312
+ rows_changed=rows_changed,
313
+ columns_added=columns_added,
314
+ columns_removed=columns_removed,
315
+ type_changes=type_changes,
316
+ compared_columns=compared,
317
+ column_change_counts=column_change_counts,
318
+ added=added,
319
+ removed=removed,
320
+ changed=changed,
321
+ )
@@ -0,0 +1,301 @@
1
+ Metadata-Version: 2.4
2
+ Name: riftdiff
3
+ Version: 0.1.0
4
+ Summary: Local-first data diff, powered by DuckDB. See exactly what changed before you merge.
5
+ Author: Chetan Chirag
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/ChetanChirag24/rift
8
+ Project-URL: Source, https://github.com/ChetanChirag24/rift
9
+ Project-URL: Issues, https://github.com/ChetanChirag24/rift/issues
10
+ Project-URL: Changelog, https://github.com/ChetanChirag24/rift/releases
11
+ Keywords: diff,data,csv,parquet,duckdb,data-quality,git
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Environment :: Console
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3 :: Only
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Programming Language :: Python :: 3.14
23
+ Classifier: Topic :: Database
24
+ Classifier: Topic :: Software Development :: Quality Assurance
25
+ Requires-Python: >=3.10
26
+ Description-Content-Type: text/markdown
27
+ License-File: LICENSE
28
+ Requires-Dist: duckdb>=0.10.0
29
+ Requires-Dist: rich>=13.0.0
30
+ Provides-Extra: dev
31
+ Requires-Dist: pytest>=7; extra == "dev"
32
+ Dynamic: license-file
33
+
34
+ <p align="center">
35
+ <img src="https://raw.githubusercontent.com/ChetanChirag24/rift/719315b1c043077b447e6c6ff89f3b1db84befa5/docs/assets/hero.svg" alt="rift: two stacked data layers, before and after, with a scanner finding added, removed and changed rows" width="100%">
36
+ </p>
37
+
38
+ <p align="center">
39
+ <a href="https://github.com/ChetanChirag24/rift/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/ChetanChirag24/rift/actions/workflows/ci.yml/badge.svg"></a>
40
+ <img alt="Python 3.10+" src="https://img.shields.io/badge/python-3.10%2B-3776AB?logo=python&logoColor=white">
41
+ <img alt="Powered by DuckDB" src="https://img.shields.io/badge/powered%20by-DuckDB-FFF000?logo=duckdb&logoColor=black">
42
+ <img alt="Formats" src="https://img.shields.io/badge/formats-CSV%20%C2%B7%20Parquet%20%C2%B7%20JSON-22d3ee">
43
+ <img alt="Git aware" src="https://img.shields.io/badge/git-aware-F05032?logo=git&logoColor=white">
44
+ <img alt="MIT License" src="https://img.shields.io/badge/license-MIT-a78bfa">
45
+ </p>
46
+
47
+ <p align="center"><b>Think <code>git diff</code>, but for tables.</b><br>
48
+ Point rift at two versions of a dataset and it tells you which rows were <b>added</b>, which were <b>removed</b>, and exactly which values <b>changed</b>, all on your own machine.</p>
49
+
50
+ ---
51
+
52
+ ## Why rift?
53
+
54
+ Code changes get reviewed line by line. **Data changes usually don't.** A CSV export is regenerated, a SQL model is refactored, a price list is updated, and the only check is "the row count looks about right."
55
+
56
+ rift shows you what actually changed:
57
+
58
+ - πŸ” **Row-level answers.** Not just "3 rows differ": you see *which* rows and *which* columns, with before β†’ after values.
59
+ - 🏠 **Local-first.** Your data never leaves your laptop. No server, no account, no upload.
60
+ - ⚑ **Fast.** Both files are loaded into an in-memory [DuckDB](https://duckdb.org) database and compared with SQL, so millions of rows are fine.
61
+ - 🌿 **Git-aware.** Compare a file with any earlier commit: `rift HEAD~1:data.csv data.csv -k id`.
62
+ - πŸ€– **CI-ready.** Exits like `diff`: `0` = identical, `1` = different, `2` = error. Add `--json` for machine-readable output.
63
+
64
+ ## See it in action
65
+
66
+ <p align="center">
67
+ <img src="https://raw.githubusercontent.com/ChetanChirag24/rift/719315b1c043077b447e6c6ff89f3b1db84befa5/docs/assets/terminal.svg" alt="Terminal: rift compares data/customers.csv with its previous git commit and prints added, removed and changed rows" width="100%">
68
+ </p>
69
+
70
+ ## How it works
71
+
72
+ <p align="center">
73
+ <img src="https://raw.githubusercontent.com/ChetanChirag24/rift/719315b1c043077b447e6c6ff89f3b1db84befa5/docs/assets/how-it-works.svg" alt="Diagram: two files (one read from git) flow into a DuckDB cube which joins them on the key, then out to a rich table, JSON and an exit code" width="100%">
74
+ </p>
75
+
76
+ 1. **Load.** rift reads each side into DuckDB. Either side can be a CSV, Parquet or JSON file, or a version of a file from git history (`REV:path`).
77
+ 2. **Check the key.** The key column(s) you pass must exist on both sides and be unique and non-empty, so every row has exactly one partner.
78
+ 3. **Compare.** rift joins the two tables on the key:
79
+ - keys only in *after* β†’ **added**
80
+ - keys only in *before* β†’ **removed**
81
+ - keys on both sides where any column `IS DISTINCT FROM` its partner β†’ **changed** (so `NULL` vs `NULL` correctly counts as equal)
82
+ 4. **Report.** You get a colour table in the terminal, JSON with `--json`, and an exit code your scripts can act on. Added, removed and retyped columns are reported too.
83
+
84
+ ---
85
+
86
+ ## Real-world examples
87
+
88
+ ### 🧾 "Did last night's CRM export silently drop customers?"
89
+
90
+ Your finance dashboard is fed by a nightly export. Today's revenue number looks odd.
91
+
92
+ ```bash
93
+ rift exports/2026-11-15/customers.csv exports/2026-11-16/customers.csv -k customer_id
94
+ ```
95
+
96
+ ```text
97
+ Removed -1 ← Initech vanished from the export
98
+ Changed ~2 ← Globex: status active β†’ churned, revenue 8500 β†’ 0
99
+ ```
100
+
101
+ Two minutes later you know whether it's a real churn or a broken export.
102
+
103
+ ### πŸ› οΈ "I refactored a SQL model. Is the output exactly the same?"
104
+
105
+ You rewrote a 400-line query to make it faster. The output should not change at all.
106
+
107
+ ```bash
108
+ rift old_model.parquet new_model.parquet -k order_id --ignore _loaded_at
109
+ echo $? # 0 β†’ identical, ship it. 1 β†’ look at the diff first.
110
+ ```
111
+
112
+ `--ignore` skips columns that are *supposed* to differ, like load timestamps.
113
+
114
+ ### πŸ›’ "Which prices am I about to change in the store?"
115
+
116
+ Merchandising sent a new price sheet. Before you upload it, check exactly what it changes:
117
+
118
+ ```bash
119
+ rift prices_live.csv prices_new.csv -k sku --summary
120
+ ```
121
+
122
+ ```text
123
+ Changed values by column
124
+ column rows changed
125
+ price 412
126
+ sale_price 37
127
+ ```
128
+
129
+ 412 price changes when you expected 40? Better to catch that now than after customers do.
130
+
131
+ ### πŸ€– "Why did my model's accuracy drop between training runs?"
132
+
133
+ Diff the feature tables from both runs, keyed on a composite key, and pipe the JSON into `jq`:
134
+
135
+ ```bash
136
+ rift features_run41.parquet features_run42.parquet -k user_id,event_date --json \
137
+ | jq '.column_change_counts'
138
+ ```
139
+
140
+ ```json
141
+ { "avg_session_minutes": 120394, "country": 3 }
142
+ ```
143
+
144
+ One feature changed for 120k users. That's your bug.
145
+
146
+ ### 🌍 Multi-region tables with composite keys
147
+
148
+ The same `id` can exist in several regions? Combine columns into one key:
149
+
150
+ ```bash
151
+ rift before.csv after.csv -k region,id
152
+ ```
153
+
154
+ ---
155
+
156
+ ## Works with Git
157
+
158
+ Data files live in git repos too, and rift treats git history as a first-class source.
159
+
160
+ ### Compare with any commit, branch or tag
161
+
162
+ Use git's own `REV:path` syntax (paths are relative to the repo root, or start with `./` for the current directory):
163
+
164
+ ```bash
165
+ rift HEAD~1:data/customers.csv data/customers.csv -k customer_id # what did my last commit change?
166
+ rift main:data/customers.csv data/customers.csv -k customer_id # what does my branch change vs main?
167
+ rift v1.0:exports/users.parquet v2.0:exports/users.parquet -k id # between two releases
168
+ ```
169
+
170
+ ### Use rift as your `git difftool`
171
+
172
+ ```bash
173
+ git config --global difftool.rift.cmd 'rift "$LOCAL" "$REMOTE" --key id'
174
+ git difftool -y -t rift HEAD~1 -- data/customers.csv
175
+ ```
176
+
177
+ ### Catch data changes in pull requests
178
+
179
+ ```mermaid
180
+ flowchart LR
181
+ A[πŸ‘©β€πŸ’» Pull request<br/>changes data/] --> B[βš™οΈ CI runs<br/>rift main:data.csv data.csv]
182
+ B -->|exit 0| C[βœ… Data unchanged<br/>merge freely]
183
+ B -->|exit 1| D[⚠️ Data changed<br/>diff printed in the log]
184
+ B -->|exit 2| E[❌ Bad key or file<br/>fix the input]
185
+ D --> F[πŸ‘€ Reviewer checks<br/>the row-level diff]
186
+ F --> C
187
+ ```
188
+
189
+ A minimal GitHub Actions job:
190
+
191
+ ```yaml
192
+ name: data-diff
193
+ on: pull_request
194
+
195
+ jobs:
196
+ rift:
197
+ runs-on: ubuntu-latest
198
+ steps:
199
+ - uses: actions/checkout@v7
200
+ with:
201
+ fetch-depth: 0 # rift needs the main branch history
202
+ - uses: actions/setup-python@v7
203
+ with:
204
+ python-version: "3.12"
205
+ - run: pip install riftdiff
206
+ - name: Show what this PR changes in the data
207
+ run: rift origin/main:data/customers.csv data/customers.csv -k customer_id
208
+ continue-on-error: true # report the diff without failing the build
209
+ ```
210
+
211
+ Remove `continue-on-error` if any data change should block the merge until someone signs off.
212
+
213
+ ---
214
+
215
+ ## Install
216
+
217
+ ```bash
218
+ pip install riftdiff
219
+ ```
220
+
221
+ Or from source:
222
+
223
+ ```bash
224
+ git clone https://github.com/ChetanChirag24/rift.git && cd rift
225
+ python3 -m venv .venv && source .venv/bin/activate
226
+ pip install -e '.[dev]'
227
+ pytest
228
+ ```
229
+
230
+ > **macOS + iCloud Desktop/Documents:** iCloud can mark files inside `.venv` as hidden, and Python then skips the `.pth` file that makes the editable install work (`ModuleNotFoundError: No module named 'riftdiff'`). Create the venv as `.venv.nosync` (iCloud ignores `*.nosync`) and symlink it: `python3 -m venv .venv.nosync && ln -s .venv.nosync .venv`.
231
+
232
+ ## Usage
233
+
234
+ ```
235
+ rift BEFORE AFTER --key COLUMN [options]
236
+ ```
237
+
238
+ `BEFORE` and `AFTER` are file paths or git revisions (`HEAD~1:path/to/file.csv`).
239
+
240
+ | Option | What it does |
241
+ | --- | --- |
242
+ | `-k, --key COLUMN` | Column that identifies a row. Repeat it or comma-separate it for a composite key: `-k region,id`. |
243
+ | `-i, --ignore COLUMN` | Leave a column out of the comparison, e.g. `-i updated_at`. Repeatable / comma-separated. |
244
+ | `-n, --limit N` | Show at most N rows per section (default 10; `0` shows all). Counts are always exact. |
245
+ | `--summary` | Print only the counts, schema changes and per-column totals. |
246
+ | `--json` | Print the full diff as JSON for scripts and CI. |
247
+ | `--version` | Print the version. |
248
+
249
+ **Supported formats:** CSV/TSV (`.csv`, `.tsv`, `.txt`), Parquet (`.parquet`, `.pq`) and JSON (`.json`, `.jsonl`, `.ndjson`), optionally compressed with `.gz` or `.zst`. The two sides may be different formats.
250
+
251
+ ### What rift reports
252
+
253
+ | | Meaning |
254
+ | --- | --- |
255
+ | 🟒 **Added** | Keys that exist only in `AFTER` |
256
+ | πŸ”΄ **Removed** | Keys that exist only in `BEFORE` |
257
+ | 🟑 **Changed** | Keys on both sides where at least one value differs, with before β†’ after for each changed column |
258
+ | πŸ“Š **By column** | How many rows changed in each column |
259
+ | 🧬 **Schema** | Columns added, removed, or whose type changed. Retyped columns are compared by their text form, so `10` and `"10"` count as equal. |
260
+
261
+ rift refuses keys that are missing, empty or not unique, because a diff on those would be misleading.
262
+
263
+ ### Exit status
264
+
265
+ | Code | Meaning |
266
+ | --- | --- |
267
+ | `0` | No differences |
268
+ | `1` | Differences found |
269
+ | `2` | Error (missing file, bad key, unreadable data…) |
270
+
271
+ ## Python API
272
+
273
+ ```python
274
+ from riftdiff import diff
275
+
276
+ result = diff("HEAD~1:data/orders.csv", "data/orders.parquet",
277
+ key_column=["region", "order_id"],
278
+ ignore_columns=["updated_at"], limit=100)
279
+
280
+ result.summary() # {'rows_before': ..., 'rows_added': ..., ...}
281
+ result.added # list of row dicts
282
+ result.removed # list of row dicts
283
+ for change in result.changed:
284
+ print(change.key, change.changes) # {'region': 'EU', 'order_id': 7} {'total': (120, 135)}
285
+ result.has_differences # bool
286
+ result.to_dict() # JSON-ready dict
287
+ ```
288
+
289
+ Problems with the input raise `riftdiff.RiftError` with a readable message.
290
+
291
+ ## Try it now
292
+
293
+ The repo includes a small example:
294
+
295
+ ```bash
296
+ rift examples/customers_before.csv examples/customers_after.csv -k customer_id
297
+ ```
298
+
299
+ ## License
300
+
301
+ [MIT](https://github.com/ChetanChirag24/rift/blob/719315b1c043077b447e6c6ff89f3b1db84befa5/LICENSE)
@@ -0,0 +1,10 @@
1
+ riftdiff/__init__.py,sha256=rrAVrDng8yUlS071oHKDotSm_oOiSLPGPrLexK3gr7c,226
2
+ riftdiff/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
3
+ riftdiff/cli.py,sha256=wCwqn4r3wX3-BPpcaz_wuyc7Rj_d7PQg9Pi9H_7DTxU,6954
4
+ riftdiff/engine.py,sha256=g4uhe96UUz_oyX5VnpjW24IIY-2jgAZQfrQVI-lSr3I,12065
5
+ riftdiff-0.1.0.dist-info/licenses/LICENSE,sha256=7wmnWrkWVOhvB47TpsonOsbkSGic9GMy9veJpud9-yA,1070
6
+ riftdiff-0.1.0.dist-info/METADATA,sha256=7P_ebTjmzSpCgsnSft2-XT5aeeVE5FRXF_d2lmWZql4,11864
7
+ riftdiff-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
8
+ riftdiff-0.1.0.dist-info/entry_points.txt,sha256=lz_b_-AW2Xh3yzsLxYZ4crT7ewt3C4AVYuSQlBcsW-0,43
9
+ riftdiff-0.1.0.dist-info/top_level.txt,sha256=bdXbPKpQCPekCNZRWTRC413Ix8v7UFMTmjPeAY29DG4,9
10
+ riftdiff-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ rift = riftdiff.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Chetan Chirag
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ riftdiff