lightweight-table-diff 0.1.0__tar.gz

lightweight_table_diff-0.1.0/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: lightweight-table-diff
+Version: 0.1.0
+Summary: Cell-level table diffing for Polars
+Requires-Python: >=3.11.1
+Description-Content-Type: text/markdown
+Requires-Dist: polars>=1.38
+Requires-Dist: polars-checkpoint>=0.1.2
+Provides-Extra: spss
+Requires-Dist: pyreadstat; extra == "spss"
+Provides-Extra: hive
+Requires-Dist: boto3; extra == "hive"

lightweight_table_diff-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["setuptools>=77.0.3"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "lightweight-table-diff"
+version = "0.1.0"
+description = "Cell-level table diffing for Polars"
+readme = "README.md"
+requires-python = ">=3.11.1"
+dependencies = [
+  "polars>=1.38",
+  "polars-checkpoint>=0.1.2",
+]
+
+[project.optional-dependencies]
+spss = [
+  "pyreadstat",
+]
+hive = [
+  "boto3",
+]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]

lightweight_table_diff-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

lightweight_table_diff-0.1.0/src/lightweight_table_diff/__init__.py

@@ -0,0 +1,5 @@
+"""table_diff — cell-level table diffing for Polars."""
+from .core import batch_diff_tbls, diff_tbls
+from .runner import run_comparison, run_config
+
+__all__ = ["diff_tbls", "batch_diff_tbls", "run_comparison", "run_config"]

lightweight_table_diff-0.1.0/src/lightweight_table_diff/__main__.py

@@ -0,0 +1,16 @@
+"""python -m table_diff config.yml"""
+import logging
+import sys
+
+from .runner import run_config
+
+logging.basicConfig(
+    format="%(asctime)s  %(levelname)-8s  %(message)s",
+    level=logging.INFO,
+)
+
+if len(sys.argv) < 2:
+    print("Usage: python -m table_diff <config.yml>", file=sys.stderr)
+    sys.exit(1)
+
+run_config(sys.argv[1])

lightweight_table_diff-0.1.0/src/lightweight_table_diff/adapters/__init__.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from typing import Any, Callable
+
+import polars as pl
+
+AdapterFn = Callable[..., pl.LazyFrame]
+
+_registry: dict[str, AdapterFn] = {}
+
+
+def register(name: str, adapter: AdapterFn) -> None:
+    _registry[name] = adapter
+
+
+def load(source_def: dict, **context: Any) -> pl.LazyFrame:
+    adapter_type = source_def.get("type", "parquet")
+    if adapter_type not in _registry:
+        raise ValueError(
+            f"Unknown adapter {adapter_type!r}. Registered: {sorted(_registry)}"
+        )
+    return _registry[adapter_type](source_def, **context)
+
+
+from .csv import load_csv      # noqa: E402
+from .parquet import load_parquet  # noqa: E402
+from .sav import load_sav      # noqa: E402
+
+register("parquet", load_parquet)
+register("csv", load_csv)
+register("sav", load_sav)

lightweight_table_diff-0.1.0/src/lightweight_table_diff/adapters/csv.py

@@ -0,0 +1,14 @@
+from pathlib import Path
+
+import polars as pl
+
+
+def load_csv(source_def: dict) -> pl.LazyFrame:
+    path = Path(source_def["path"])
+    glob = source_def.get("glob", "*.csv")
+    scan_path = str(path / glob) if path.is_dir() else str(path)
+    return pl.scan_csv(
+        scan_path,
+        infer_schema_length=source_def.get("infer_schema_length", 10_000),
+        ignore_errors=source_def.get("ignore_errors", True),
+    )

lightweight_table_diff-0.1.0/src/lightweight_table_diff/adapters/hive_s3.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from urllib.parse import urlparse
+
+import polars as pl
+
+logger = logging.getLogger(__name__)
+
+
+def load_hive(source_def: dict, **context) -> pl.LazyFrame:
+    import boto3
+
+    spark = context.get("spark")
+    if spark is None:
+        raise RuntimeError("Hive adapter requires spark= to be passed")
+
+    ssl_cert = context.get("ssl_cert")
+    table_name = source_def["table"]
+    cache_dir = Path(source_def.get("cache_dir", f"/tmp/hive_{table_name}"))
+    cache_dir.mkdir(parents=True, exist_ok=True)
+
+    rows = spark.sql(f"DESCRIBE FORMATTED {table_name}").collect()
+    location = next(
+        (r[1].strip() for r in rows if r[0] and "Location" in r[0]), None
+    )
+    if not location:
+        raise RuntimeError(f"Could not resolve S3 location for '{table_name}'")
+
+    parsed = urlparse(str(location).replace("s3a://", "s3://"))
+    bucket = parsed.netloc
+    prefix = parsed.path.lstrip("/").rstrip("/") + "/"
+
+    client = boto3.client("s3")
+    try:
+        import raz_client
+
+        if ssl_cert:
+            raz_client.configure_ranger_raz(client, ssl_file=ssl_cert)
+    except ImportError:
+        pass
+
+    logger.info("Downloading %s → %s", location, cache_dir)
+    n_files = 0
+    for page in client.get_paginator("list_objects_v2").paginate(
+        Bucket=bucket, Prefix=prefix
+    ):
+        for obj in page.get("Contents", []):
+            if not obj["Key"].endswith(".parquet"):
+                continue
+            n_files += 1
+            relative = (
+                obj["Key"][len(prefix) :].lstrip("/")
+                if obj["Key"].startswith(prefix)
+                else Path(obj["Key"]).name
+            )
+            dest = cache_dir / relative
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            client.download_file(bucket, obj["Key"], str(dest))
+
+    if not n_files:
+        raise FileNotFoundError(f"No parquet files found at {location}")
+
+    return pl.scan_parquet(
+        str(cache_dir / "**/*.parquet"), hive_partitioning=True
+    )

lightweight_table_diff-0.1.0/src/lightweight_table_diff/adapters/parquet.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+
+import polars as pl
+
+
+def load_parquet(source_def: dict) -> pl.LazyFrame:
+    path = Path(source_def["path"])
+    glob = source_def.get("glob", "*.parquet")
+    scan_path = str(path / glob) if path.is_dir() else str(path)
+    return pl.scan_parquet(
+        scan_path,
+        hive_partitioning=source_def.get("hive_partitioning", False),
+    )

lightweight_table_diff-0.1.0/src/lightweight_table_diff/adapters/sav.py

@@ -0,0 +1,23 @@
+"""
+SPSS .sav adapter.  Loads via pyreadstat into an in-memory Polars frame.
+Normalisation (null alignment, trailing-zero stripping, etc.)
+is left to the normaliser layer.
+"""
+import polars as pl
+
+
+def load_sav(source_def: dict) -> pl.LazyFrame:
+    try:
+        import pyreadstat
+    except ImportError:
+        raise ImportError(
+            "pyreadstat is required for .sav files: pip install pyreadstat"
+        ) from None
+
+    pdf, _meta = pyreadstat.read_sav(
+        str(source_def["path"]),
+        apply_value_formats=source_def.get("apply_value_formats", False),
+        formats_as_category=False,
+        user_missing=True,
+    )
+    return pl.from_pandas(pdf).lazy()

lightweight_table_diff-0.1.0/src/lightweight_table_diff/config.py

@@ -0,0 +1,40 @@
+"""YAML config loading and deep-merge expansion."""
+
+from __future__ import annotations
+
+import copy
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def deep_merge(base: dict, override: dict) -> dict:
+    result = copy.deepcopy(base)
+    _merge_in_place(result, override)
+    return result
+
+
+def _merge_in_place(target: dict, source: dict) -> None:
+    for k, v in source.items():
+        if k in target and isinstance(v, dict) and isinstance(target[k], dict):
+            _merge_in_place(target[k], v)
+        else:
+            target[k] = copy.deepcopy(v)
+
+
+def expand_comparisons(raw: dict[str, Any]) -> list[dict[str, Any]]:
+    """Split shared defaults + 'comparisons' list into per-job dicts."""
+    base = {k: v for k, v in raw.items() if k != "comparisons"}
+    items = raw.get("comparisons", [{}])
+    if not isinstance(items, list):
+        raise TypeError(
+            f"'comparisons' must be a list, got {type(items).__name__}"
+        )
+    return [deep_merge(base, item) for item in items]
+
+
+def load_config(path: str | Path) -> list[dict[str, Any]]:
+    with open(path, encoding="utf-8") as f:
+        raw = yaml.safe_load(f)
+    return expand_comparisons(raw)

lightweight_table_diff-0.1.0/src/lightweight_table_diff/core.py

@@ -0,0 +1,85 @@
+"""
+Cell-level table differ for Polars.
+
+Produces long-form (keys…, col_name, before_val, after_val) for every
+cell that changed between two LazyFrames.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import polars as pl
+from polars_checkpoint import CheckpointSession, checkpoint
+
+logger = logging.getLogger(__name__)
+
+
+def diff_tbls(
+    before: pl.LazyFrame,
+    after: pl.LazyFrame,
+    id_cols: list[str],
+    compare_cols: list[str] | None = None,
+    join_type: str = "full",
+) -> pl.LazyFrame:
+    if compare_cols is None:
+        compare_cols = list(before.drop(*id_cols).collect_schema().keys())
+
+    bef_cols = [pl.col(c).alias(f"b__{c}") for c in compare_cols]
+    aft_cols = [pl.col(c).alias(f"a__{c}") for c in compare_cols]
+    before = before.select(*id_cols, *bef_cols)
+    after = after.select(*id_cols, *aft_cols)
+
+    joined = before.join(after, on=id_cols, how=join_type, coalesce=True)
+
+    diff_structs = [
+        pl.when(~pl.col(f"b__{c}").eq_missing(pl.col(f"a__{c}")))
+        .then(
+            pl.struct(
+                pl.col(f"b__{c}").alias("before_val"),
+                pl.col(f"a__{c}").alias("after_val"),
+            )
+        )
+        .otherwise(None)
+        .alias(c)
+        for c in compare_cols
+    ]
+
+    return (
+        joined.select(*id_cols, *diff_structs)
+        .unpivot(
+            on=compare_cols,
+            index=id_cols,
+            variable_name="col_name",
+            value_name="diff",
+        )
+        .drop_nulls("diff")
+        .select(
+            *id_cols,
+            "col_name",
+            pl.col("diff").struct.field("before_val"),
+            pl.col("diff").struct.field("after_val"),
+        )
+    )
+
+
+def batch_diff_tbls(
+    before: pl.LazyFrame,
+    after: pl.LazyFrame,
+    id_cols: list[str],
+    compare_cols: list[str] | None = None,
+    batch_size: int = 50,
+    join_type: str = "full",
+) -> pl.LazyFrame:
+    if compare_cols is None:
+        compare_cols = list(before.drop(*id_cols).collect_schema().keys())
+
+    parts = []
+    n = len(compare_cols)
+    for i in range(0, n, batch_size):
+        batch = compare_cols[i : i + batch_size]
+        logger.info("  batch %d-%d of %d columns", i + 1, min(i + len(batch), n), n)
+        diff = diff_tbls(before, after, id_cols, batch, join_type=join_type)
+        parts.append(checkpoint(diff))
+
+    return pl.concat(parts)

lightweight_table_diff-0.1.0/src/lightweight_table_diff/dimensions.py

@@ -0,0 +1,96 @@
+"""Column/row checking and key-uniqueness validation."""
+
+from __future__ import annotations
+
+import polars as pl
+
+
+def get_cols_to_compare(
+    before: pl.LazyFrame,
+    after: pl.LazyFrame,
+    keys: list[str],
+    include_cols: list[str] | None = None,
+    exclude_cols: list[str] | None = None,
+) -> list[str]:
+    """Work out which non-key columns to compare, from schema intersection."""
+    before_names = set(before.collect_schema().names())
+    after_names = set(after.collect_schema().names())
+    key_set = set(keys)
+
+    missing = key_set - (before_names & after_names)
+    if missing:
+        raise ValueError(
+            f"Join key(s) missing from one or both sides: {missing}"
+        )
+
+    if include_cols:
+        cols = [
+            c
+            for c in include_cols
+            if c in before_names and c in after_names and c not in key_set
+        ]
+    else:
+        cols = sorted((before_names & after_names) - key_set)
+
+    if exclude_cols:
+        exclude_set = set(exclude_cols)
+        cols = [c for c in cols if c not in exclude_set]
+
+    if not cols:
+        raise ValueError("No columns to compare after applying filters")
+
+    return cols
+
+
+def validate_key_uniqueness(
+    before: pl.LazyFrame,
+    after: pl.LazyFrame,
+    keys: list[str],
+    sample_limit: int = 20,
+) -> None:
+    """Raise if either frame has duplicate key combinations."""
+    problems: list[str] = []
+    for label, lf in [("before", before), ("after", after)]:
+        dupes = (
+            lf.select(*keys)
+            .group_by(keys)
+            .len()
+            .filter(pl.col("len") > 1)
+            .limit(sample_limit)
+            .collect()
+        )
+        if dupes.height:
+            problems.append(
+                f"  {label}: {dupes.height} duplicate key group(s)\n{dupes}"
+            )
+
+    if problems:
+        raise ValueError(
+            "Duplicate keys (would cause row explosion):\n"
+            + "\n".join(problems)
+        )
+
+
+def column_indels(
+    before: pl.LazyFrame,
+    after: pl.LazyFrame,
+    keys: list[str],
+) -> tuple[list[str], list[str]]:
+    """Return (removed_cols, added_cols) relative to before → after."""
+    key_set = set(keys)
+    before_cols = set(before.collect_schema().names()) - key_set
+    after_cols = set(after.collect_schema().names()) - key_set
+    return sorted(before_cols - after_cols), sorted(after_cols - before_cols)
+
+
+def row_indels(
+    before: pl.LazyFrame,
+    after: pl.LazyFrame,
+    id_cols: list[str],
+) -> tuple[pl.LazyFrame, pl.LazyFrame]:
+    """Return (removed_rows, added_rows) as key-only LazyFrames."""
+    before_keys = before.select(*id_cols)
+    after_keys = after.select(*id_cols)
+    removed = before_keys.join(after_keys, on=id_cols, how="anti")
+    added = after_keys.join(before_keys, on=id_cols, how="anti")
+    return removed, added

lightweight_table_diff-0.1.0/src/lightweight_table_diff/normalisers.py

@@ -0,0 +1,42 @@
+"""
+Normalisation transforms applied *before* diffing to suppress irrelevant
+format differences between disparate sources.
+
+Each normaliser has signature:  (lf, keys, cols) -> lf
+"""
+
+from __future__ import annotations
+
+from typing import Callable
+
+import polars as pl
+
+NormaliserFn = Callable[[pl.LazyFrame, list[str], list[str]], pl.LazyFrame]
+
+NULLISH = ["", "nan", "none", "<na>"]
+
+
+def _norm_expr(expr: pl.Expr) -> pl.Expr:
+    s = expr.cast(pl.String).str.strip_chars()
+    s_lower = s.str.to_lowercase()
+    s = (
+        pl.when(s.is_null() | s_lower.is_in(NULLISH))
+        .then(pl.lit(None, dtype=pl.String))
+        .otherwise(s)
+    )
+    return s.str.replace(r"^(-?\d+)\.0+$", "${1}")
+
+
+def normalise_float_strings(
+    lf: pl.LazyFrame, keys: list[str], cols: list[str]
+) -> pl.LazyFrame:
+    """Cast everything to string, unify nulls, strip whitespace/trailing .0."""
+    return lf.select(
+        *[_norm_expr(pl.col(k)).alias(k) for k in keys],
+        *[_norm_expr(pl.col(c)).alias(c) for c in cols],
+    )
+
+
+REGISTRY: dict[str, NormaliserFn] = {
+    "float_strings": normalise_float_strings,
+}

lightweight_table_diff-0.1.0/src/lightweight_table_diff/runner.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import copy
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+
+import polars as pl
+
+from . import adapters
+from .dimensions import (
+    validate_key_uniqueness,
+    get_cols_to_compare,
+    column_indels,
+    row_indels,
+)
+from .config import load_config
+from .core import batch_diff_tbls
+from .normalisers import REGISTRY as NORMALISER_REGISTRY
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ComparisonResult:
+    name: str
+    diff: pl.LazyFrame
+    n_diffs: int
+    removed_rows: pl.LazyFrame
+    added_rows: pl.LazyFrame
+    removed_cols: list[str]
+    added_cols: list[str]
+
+
+def run_comparison(job: dict, **context) -> ComparisonResult:
+    job = copy.deepcopy(job)
+    name = job.get("name", "unnamed")
+    keys = job["join_keys"]
+    logger.info("Running comparison: %s", name)
+
+    before = adapters.load(job["before"], **context)
+    after = adapters.load(job["after"], **context)
+
+    removed_cols, added_cols = column_indels(before, after, keys)
+    if removed_cols:
+        logger.info("  %d column(s) removed: %s", len(removed_cols), removed_cols)
+    if added_cols:
+        logger.info("  %d column(s) added: %s", len(added_cols), added_cols)
+
+    cols = get_cols_to_compare(
+        before,
+        after,
+        keys,
+        include_cols=job.get("compare_cols"),
+        exclude_cols=job.get("exclude_cols"),
+    )
+    logger.info("  %d column(s) to compare", len(cols))
+    validate_key_uniqueness(before, after, keys)
+
+    removed, added = row_indels(before, after, keys)
+
+    if norm_name := job.get("normalisation"):
+        if norm_name not in NORMALISER_REGISTRY:
+            raise ValueError(f"Unknown normaliser: {norm_name!r}")
+        norm_fn = NORMALISER_REGISTRY[norm_name]
+        before = norm_fn(before, keys, cols)
+        after = norm_fn(after, keys, cols)
+
+    diff = batch_diff_tbls(
+        before,
+        after,
+        keys,
+        cols,
+        batch_size=job.get("batch_size", 50),
+        join_type=job.get("join_type", "full"),
+    )
+
+    n = diff.select(pl.len()).collect().item()
+    logger.info("  %d difference(s) found", n)
+
+    return ComparisonResult(
+        name=name,
+        diff=diff,
+        n_diffs=n,
+        removed_rows=removed,
+        added_rows=added,
+        removed_cols=removed_cols,
+        added_cols=added_cols,
+    )
+
+
+def write_results(result: ComparisonResult, output_dir: str | Path) -> None:
+    out = Path(output_dir)
+    out.mkdir(parents=True, exist_ok=True)
+
+    # -- column indels (just a small text/csv file) ----------------------------
+    if result.removed_cols or result.added_cols:
+        col_indels_path = out / f"{result.name}_column_indels.csv"
+        max_len = max(len(result.removed_cols), len(result.added_cols))
+        pl.DataFrame(
+            {
+                "removed_columns": result.removed_cols + [""] * (max_len - len(result.removed_cols)),
+                "added_columns": result.added_cols + [""] * (max_len - len(result.added_cols)),
+            }
+        ).write_csv(col_indels_path)
+        logger.info("  Wrote %s", col_indels_path)
+
+    # -- row indels ------------------------------------------------------------
+    for label, lf in [
+        ("removed", result.removed_rows),
+        ("added", result.added_rows),
+    ]:
+        n = lf.select(pl.len()).collect().item()
+        if n > 0:
+            path = out / f"{result.name}_{label}_rows.csv"
+            lf.sink_csv(path)
+            logger.info("  %d %s row(s) → %s", n, label, path)
+        else:
+            logger.info("  No %s rows", label)
+
+    # -- cell-level diff -------------------------------------------------------
+    if result.n_diffs == 0:
+        logger.info("  %s: no cell differences", result.name)
+        return
+
+    detail_path = out / f"{result.name}_detailed.csv"
+    result.diff.sink_csv(detail_path)
+    logger.info("  Wrote %s", detail_path)
+
+    summary_path = out / f"{result.name}_summary.csv"
+    (
+        result.diff.group_by("col_name", "before_val", "after_val")
+        .agg(pl.len().alias("count"))
+        .sort("count", descending=True)
+        .sink_csv(summary_path)
+    )
+    logger.info("  Wrote %s", summary_path)
+
+
+def run_config(config_path: str | Path, **context) -> list[ComparisonResult]:
+    jobs = load_config(config_path)
+    results: list[ComparisonResult] = []
+    for job in jobs:
+        result = run_comparison(job, **context)
+        write_results(result, job.get("output_dir", "./diff_output"))
+        results.append(result)
+    return results

lightweight_table_diff-0.1.0/src/lightweight_table_diff.egg-info/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: lightweight-table-diff
+Version: 0.1.0
+Summary: Cell-level table diffing for Polars
+Requires-Python: >=3.11.1
+Description-Content-Type: text/markdown
+Requires-Dist: polars>=1.38
+Requires-Dist: polars-checkpoint>=0.1.2
+Provides-Extra: spss
+Requires-Dist: pyreadstat; extra == "spss"
+Provides-Extra: hive
+Requires-Dist: boto3; extra == "hive"

lightweight_table_diff-0.1.0/src/lightweight_table_diff.egg-info/SOURCES.txt

@@ -0,0 +1,19 @@
+README.md
+pyproject.toml
+src/lightweight_table_diff/__init__.py
+src/lightweight_table_diff/__main__.py
+src/lightweight_table_diff/config.py
+src/lightweight_table_diff/core.py
+src/lightweight_table_diff/dimensions.py
+src/lightweight_table_diff/normalisers.py
+src/lightweight_table_diff/runner.py
+src/lightweight_table_diff.egg-info/PKG-INFO
+src/lightweight_table_diff.egg-info/SOURCES.txt
+src/lightweight_table_diff.egg-info/dependency_links.txt
+src/lightweight_table_diff.egg-info/requires.txt
+src/lightweight_table_diff.egg-info/top_level.txt
+src/lightweight_table_diff/adapters/__init__.py
+src/lightweight_table_diff/adapters/csv.py
+src/lightweight_table_diff/adapters/hive_s3.py
+src/lightweight_table_diff/adapters/parquet.py
+src/lightweight_table_diff/adapters/sav.py

lightweight_table_diff-0.1.0/src/lightweight_table_diff.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

lightweight_table_diff-0.1.0/src/lightweight_table_diff.egg-info/requires.txt

@@ -0,0 +1,8 @@
+polars>=1.38
+polars-checkpoint>=0.1.2
+
+[hive]
+boto3
+
+[spss]
+pyreadstat

lightweight_table_diff-0.1.0/src/lightweight_table_diff.egg-info/top_level.txt

@@ -0,0 +1 @@
+lightweight_table_diff