csv-stream-diff 0.1.0__py3-none-any.whl

config.example.yaml

@@ -0,0 +1,59 @@
+files:
+  left: ./data/left.csv
+  right: ./data/right.csv
+
+csv:
+  left:
+    encoding: utf-8-sig
+    delimiter: ","
+    quotechar: '"'
+    escapechar:
+    newline: ""
+  right:
+    encoding: utf-8-sig
+    delimiter: ","
+    quotechar: '"'
+    escapechar:
+    newline: ""
+
+keys:
+  left:
+    - customer_id
+    - transaction_date
+  right:
+    - cust_id
+    - txn_dt
+
+compare:
+  left:
+    - amount
+    - status
+    - description
+  right:
+    - transaction_amount
+    - txn_status
+    - desc
+
+comparison:
+  case_insensitive: true
+  trim_whitespace: true
+  treat_null_as_equal: false
+
+sampling:
+  size: 0
+  seed: 12345
+
+performance:
+  chunk_size: 100000
+  workers:
+  bucket_count:
+  report_every_rows: 50000
+  temp_directory:
+  keep_temp_files: false
+  show_progress: true
+
+output:
+  directory: ./output
+  prefix: comparison_
+  include_full_rows: true
+  summary_format: both

csv_stream_diff-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jordi Corbilla
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

csv_stream_diff-0.1.0.dist-info/METADATA

@@ -0,0 +1,161 @@
+Metadata-Version: 2.1
+Name: csv-stream-diff
+Version: 0.1.0
+Summary: Stream and compare very large CSV files with multiprocessing.
+License: MIT
+Keywords: csv,diff,streaming,multiprocessing,comparison
+Author: Jordi
+Requires-Python: >=3.10,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: File Formats
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Utilities
+Requires-Dist: PyYAML (>=6.0)
+Requires-Dist: rich (>=13.7)
+Description-Content-Type: text/markdown
+
+# csv-stream-diff
+
+`csv-stream-diff` compares very large CSV files with streaming I/O, hashed bucket partitioning, and multiprocessing. It is designed for datasets that are too large to load fully into memory.
+
+## Features
+
+- Compare CSVs by configurable key columns, even when left and right headers differ
+- Stream files in chunks with configurable `chunk_size`
+- Partition by stable hashed key to keep worker memory bounded
+- Use all CPUs by default, or set a worker count explicitly
+- Write machine-usable output artifacts for left-only, right-only, cell differences, duplicate keys, and run summary
+- Support exact random sampling for validation runs with `sampling.size > 0`
+- Warn on duplicate keys and continue using the first occurrence per key
+- Include a fixture generator and both `pytest` and `behave` tests
+
+## Installation
+
+```bash
+pip install csv-stream-diff
+```
+
+For local development:
+
+```bash
+poetry install
+```
+
+## CLI
+
+```bash
+csv-stream-diff --config config.yaml
+```
+
+Optional overrides:
+
+```bash
+csv-stream-diff \
+  --config config.yaml \
+  --left-file ./left.csv \
+  --right-file ./right.csv \
+  --chunk-size 100000 \
+  --sample-size 100000 \
+  --sample-seed 20260321 \
+  --workers 8 \
+  --output-dir ./output \
+  --output-prefix run_
+```
+
+The YAML config is the default source of truth. CLI flags override it for a single run.
+
+## Configuration
+
+See [config.example.yaml](/c:/repo/csv-stream-diff/config.example.yaml) for a full example.
+
+Main sections:
+
+- `files.left`, `files.right`: input CSV paths
+- `csv.left`, `csv.right`: dialect and encoding settings
+- `keys.left`, `keys.right`: key columns used to match rows
+- `compare.left`, `compare.right`: value columns to compare
+- `comparison`: normalization options
+- `sampling`: `size: 0` means full comparison; any positive value means exact random sample by left-side unique key with a fixed seed
+- `performance`: chunking, worker count, bucket count, temp directory, progress reporting
+- `output`: output directory, filename prefix, whether to include serialized full rows, and whether to write a text summary
+
+## Output Files
+
+The tool writes these artifacts to `output.directory`:
+
+- `<prefix>only_in_left.csv`
+- `<prefix>only_in_right.csv`
+- `<prefix>differences.csv`
+- `<prefix>duplicate_keys.csv`
+- `<prefix>summary.json`
+- `<prefix>summary.txt` when `output.summary_format` is `text` or `both`
+
+`differences.csv` contains one row per differing cell with both the left and right column names and values.
+
+## Sampling
+
+- `sampling.size: 0` runs the full comparison.
+- `sampling.size > 0` selects an exact random sample of left-side unique keys using reservoir sampling.
+- Sampling is reproducible when `sampling.seed` stays the same.
+- Duplicate keys do not expand the sampling population because only the first occurrence per key is considered.
+
+## Duplicate Keys
+
+Duplicate keys do not stop the run. They are written to `duplicate_keys.csv`, counted in the summary, and the main comparison uses the first occurrence of each key on each side.
+
+## Generator
+
+The generator creates two baseline-identical CSVs, applies controlled mutations, writes a matching config, and saves an expected manifest:
+
+```bash
+python generator/generate_fixtures.py --output-dir ./generated --rows 10000 --seed 42
+```
+
+Generated artifacts:
+
+- `left.csv`
+- `right.csv`
+- `config.generated.yaml`
+- `expected.json`
+
+## Tests
+
+Run unit tests:
+
+```bash
+poetry run pytest
+```
+
+Run BDD acceptance tests:
+
+```bash
+poetry run behave tests/features
+```
+
+Run a package build:
+
+```bash
+poetry build
+```
+
+## PyPI Packaging
+
+Build source and wheel distributions:
+
+```bash
+poetry build
+```
+
+Upload after verifying artifacts:
+
+```bash
+poetry publish
+```
+

csv_stream_diff-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+config.example.yaml,sha256=8ENWrYFHA25qT0TdGNSiSgvzoQjMgV9Q-RRPcVaBRm0,866
+csvstreamdiff/__init__.py,sha256=Umar2SMg-gpnxi4lw26YR_RjNSp9-1ciWrh-TcoEQlo,81
+csvstreamdiff/cli.py,sha256=0gvfltCfZnHYLLRyaHgd1sRe77kuWO5syheZvdpGsCc,1914
+csvstreamdiff/comparer.py,sha256=LO3sEAWlhrpLhxBE2h2_bLOOuzcIQ0rcSxAsfz0KV8c,20122
+csvstreamdiff/hashing.py,sha256=rt4nfmtDxBjmiVNz9rOgXdH007-qLhEY00p6sqr1mY8,1858
+csvstreamdiff/multiprocessing.py,sha256=aHwYpVu8DJrpVnUh0YtD-XZsB9q8GApE2LvZvxXhra4,24671
+csvstreamdiff/streaming.py,sha256=HZVSEZW5Pa8Ye0SS77IGuq19e_aZ7tcQcccNWlTIAy0,3698
+csv_stream_diff-0.1.0.dist-info/entry_points.txt,sha256=31ZuS-GCnz2sFvaAq2y7_uUfjl-0nNIPgmwFtsd2TSc,58
+csv_stream_diff-0.1.0.dist-info/LICENSE,sha256=ywpYokjmlAtrA7JZDmTrosLLGHOkUsPAiC5XTK0VGEU,1092
+csv_stream_diff-0.1.0.dist-info/METADATA,sha256=oUe4aGT2alpmp13NplztNwtrvfU1bTfNW6lit2Q9CDA,4553
+csv_stream_diff-0.1.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+csv_stream_diff-0.1.0.dist-info/RECORD,,

csv_stream_diff-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.9.1
+Root-Is-Purelib: true
+Tag: py3-none-any

csv_stream_diff-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+csv-stream-diff=csvstreamdiff.cli:main
+

csvstreamdiff/__init__.py

@@ -0,0 +1,5 @@
+"""csv-stream-diff package."""
+
+__all__ = ["__version__"]
+
+__version__ = "0.1.0"

csvstreamdiff/cli.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import argparse
+import json
+import multiprocessing as mp
+import sys
+
+from .comparer import compare_from_path
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Compare very large CSV files using streaming and multiprocessing.")
+    parser.add_argument("--config", "-c", required=True, help="Path to the YAML configuration file.")
+    parser.add_argument("--left-file", help="Override files.left from the config.")
+    parser.add_argument("--right-file", help="Override files.right from the config.")
+    parser.add_argument("--chunk-size", type=int, help="Override performance.chunk_size.")
+    parser.add_argument("--size", "--sample-size", dest="sample_size", type=int, help="Override sampling.size.")
+    parser.add_argument("--sample-seed", type=int, help="Override sampling.seed.")
+    parser.add_argument("--workers", type=int, help="Override performance.workers.")
+    parser.add_argument("--output-dir", help="Override output.directory.")
+    parser.add_argument("--output-prefix", help="Override output.prefix.")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    overrides = {
+        "left_file": args.left_file,
+        "right_file": args.right_file,
+        "chunk_size": args.chunk_size,
+        "sample_size": args.sample_size,
+        "sample_seed": args.sample_seed,
+        "workers": args.workers,
+        "output_dir": args.output_dir,
+        "output_prefix": args.output_prefix,
+    }
+
+    try:
+        summary = compare_from_path(args.config, overrides)
+    except Exception as exc:
+        print(f"csv-stream-diff failed: {exc}", file=sys.stderr)
+        return 1
+
+    print(json.dumps(summary, indent=2, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    mp.freeze_support()
+    raise SystemExit(main())