sheetdiff 0.1.0__py3-none-any.whl

sheetdiff/__init__.py

@@ -0,0 +1,8 @@
+from .core import diff_sheets, format_unified, diff_rows_full, diff_multi_grid, compare_workbooks
+from . import config
+from .storage import resolve as resolve_source, resolve_many as resolve_sources, StorageError
+
+__all__ = [
+    "diff_sheets", "format_unified", "diff_rows_full", "diff_multi_grid", "compare_workbooks",
+    "config", "resolve_source", "resolve_sources", "StorageError",
+]

sheetdiff/cli.py

@@ -0,0 +1,42 @@
+import argparse
+import sys
+import tempfile
+
+from . import config, storage
+from .core import diff_sheets, format_unified, diff_workbook, format_workbook
+
+def main():
+    p = argparse.ArgumentParser(prog="sheetdiff")
+    p.add_argument("left", help="local path or remote URI (s3://, minio://, r2://, garage://, az://)")
+    p.add_argument("right", help="local path or remote URI (s3://, minio://, r2://, garage://, az://)")
+    p.add_argument("--key", help="column name to use as row key")
+    p.add_argument("--sheet", help="sheet name for Excel files")
+    p.add_argument("--all-sheets", action="store_true", help="compare all sheets in workbooks")
+    p.add_argument(
+        "--max-remote-mb", type=int, default=None,
+        help=f"override the remote download size cap (default: {config.MAX_REMOTE_MB} MB, "
+             "also settable via SHEETDIFF_MAX_REMOTE_MB)",
+    )
+    args = p.parse_args()
+
+    max_bytes = args.max_remote_mb * 1024 * 1024 if args.max_remote_mb else None
+
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp:
+        try:
+            left_path = storage.resolve(args.left, tmp, max_bytes=max_bytes)
+            right_path = storage.resolve(args.right, tmp, max_bytes=max_bytes)
+        except storage.StorageError as e:
+            print(f"error: {e}", file=sys.stderr)
+            raise SystemExit(1)
+
+        if args.all_sheets:
+            from .core import compare_workbooks
+            wb = compare_workbooks(left_path, right_path, key=args.key)
+            print(format_workbook(wb))
+        else:
+            changes = diff_sheets(left_path, right_path, key=args.key, sheet_name=args.sheet)
+            print(format_unified(changes))
+
+
+if __name__ == "__main__":
+    main()

sheetdiff/config.py

@@ -0,0 +1,127 @@
+"""Central, environment-driven configuration for sheetdiff.
+
+Every knob here can be overridden with an environment variable so a
+deployment can tune limits and remote-storage credentials without touching
+code. Nothing here reads a hardcoded secret — credentials only ever come
+from the environment (or a `.env` loaded by the process before start-up),
+never from user-submitted form fields, so a web UI cannot be used to smuggle
+credentials for a backend the operator didn't configure.
+"""
+import os
+from dataclasses import dataclass, field
+from typing import Dict, Optional
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = os.environ.get(name)
+    if raw is None or raw.strip() == "":
+        return default
+    try:
+        value = int(raw)
+    except ValueError:
+        raise ValueError(f"{name}={raw!r} is not a valid integer") from None
+    if value <= 0:
+        raise ValueError(f"{name}={raw!r} must be a positive integer")
+    return value
+
+
+def _env_list(name: str, default: str) -> list:
+    raw = os.environ.get(name, default)
+    return [x.strip().lower() for x in raw.split(",") if x.strip()]
+
+
+# --- Upload / download size limits -----------------------------------------
+# All expressed in MB for readability at the env-var layer; converted to
+# bytes for use. Change per-deployment via env vars, e.g.:
+#   SHEETDIFF_MAX_UPLOAD_MB=250
+#   SHEETDIFF_MAX_REMOTE_MB=500
+#   SHEETDIFF_MAX_FILES=6
+MAX_UPLOAD_MB = _env_int("SHEETDIFF_MAX_UPLOAD_MB", 100)
+MAX_REMOTE_MB = _env_int("SHEETDIFF_MAX_REMOTE_MB", MAX_UPLOAD_MB)
+MAX_UPLOAD_BYTES = MAX_UPLOAD_MB * 1024 * 1024
+MAX_REMOTE_BYTES = MAX_REMOTE_MB * 1024 * 1024
+
+# Hard ceiling on how many files can be compared in one request (web UI and
+# CLI --all-sheets multi-file mode), to bound memory/CPU on shared servers.
+MAX_FILES = _env_int("SHEETDIFF_MAX_FILES", 8)
+
+# Network timeouts for remote storage reads (seconds).
+REMOTE_CONNECT_TIMEOUT = _env_int("SHEETDIFF_REMOTE_CONNECT_TIMEOUT", 10)
+REMOTE_READ_TIMEOUT = _env_int("SHEETDIFF_REMOTE_READ_TIMEOUT", 60)
+
+# Which remote URI schemes are enabled at all. Empty by default in the sense
+# that only schemes listed here are ever dispatched to fsspec; anything else
+# (including http/https, to avoid turning the diff endpoint into an SSRF
+# proxy) is rejected outright.
+ALLOWED_REMOTE_SCHEMES = set(
+    _env_list("SHEETDIFF_ALLOWED_SCHEMES", "s3,minio,r2,garage,az,azure,gs")
+)
+
+
+@dataclass
+class BackendConfig:
+    protocol: str  # fsspec protocol, e.g. "s3" or "az"
+    storage_options: Dict[str, str] = field(default_factory=dict)
+
+
+def _s3_backend(scheme: str) -> BackendConfig:
+    prefix = f"SHEETDIFF_STORAGE_{scheme.upper()}_"
+    opts: Dict[str, str] = {}
+    endpoint = os.environ.get(prefix + "ENDPOINT_URL")
+    if endpoint:
+        opts["client_kwargs"] = {"endpoint_url": endpoint}
+    key = os.environ.get(prefix + "KEY") or os.environ.get("AWS_ACCESS_KEY_ID")
+    secret = os.environ.get(prefix + "SECRET") or os.environ.get("AWS_SECRET_ACCESS_KEY")
+    region = os.environ.get(prefix + "REGION") or os.environ.get("AWS_DEFAULT_REGION")
+    if key:
+        opts["key"] = key
+    if secret:
+        opts["secret"] = secret
+    if region:
+        opts.setdefault("client_kwargs", {})["region_name"] = region
+    return BackendConfig(protocol="s3", storage_options=opts)
+
+
+def _azure_backend() -> BackendConfig:
+    prefix = "SHEETDIFF_STORAGE_AZ_"
+    opts: Dict[str, str] = {}
+    conn_str = os.environ.get(prefix + "CONNECTION_STRING")
+    if conn_str:
+        opts["connection_string"] = conn_str
+    else:
+        account = os.environ.get(prefix + "ACCOUNT_NAME")
+        account_key = os.environ.get(prefix + "ACCOUNT_KEY")
+        sas_token = os.environ.get(prefix + "SAS_TOKEN")
+        if account:
+            opts["account_name"] = account
+        if account_key:
+            opts["account_key"] = account_key
+        if sas_token:
+            opts["sas_token"] = sas_token
+    return BackendConfig(protocol="az", storage_options=opts)
+
+
+def _gcs_backend() -> BackendConfig:
+    opts: Dict[str, str] = {}
+    creds = os.environ.get("SHEETDIFF_STORAGE_GS_TOKEN")
+    if creds:
+        opts["token"] = creds
+    return BackendConfig(protocol="gcs", storage_options=opts)
+
+
+def backend_for_scheme(scheme: str) -> Optional[BackendConfig]:
+    """Map a URI scheme (e.g. 'minio', 'r2', 'az') to an fsspec backend.
+
+    Returns None if the scheme isn't recognized/enabled, so callers can
+    reject it instead of silently falling through to some default.
+    """
+    scheme = scheme.lower()
+    if scheme not in ALLOWED_REMOTE_SCHEMES:
+        return None
+    if scheme in ("s3", "minio", "r2", "garage"):
+        return _s3_backend(scheme)
+    if scheme in ("az", "azure"):
+        return _azure_backend()
+    if scheme == "gs":
+        return _gcs_backend()
+    return None

sheetdiff/core.py

@@ -0,0 +1,356 @@
+from typing import List, Tuple, Optional, Any, Dict
+import os
+import tempfile
+import pandas as pd
+import json
+
+# Optional Rust backend (xl_diff) if built and installed
+_RUST_AVAILABLE = False
+_rust = None
+try:
+    import xl_diff as _rust  # rust extension built via maturin/pyo3
+    _RUST_AVAILABLE = True
+except Exception:
+    _RUST_AVAILABLE = False
+
+def _load(path: str, sheet_name: Optional[str] = None) -> pd.DataFrame:
+    if str(path).lower().endswith(('.xls', '.xlsx', '.xlsm')):
+        return pd.read_excel(path, sheet_name=sheet_name if sheet_name is not None else 0, dtype=object)
+    return pd.read_csv(path, dtype=object)
+
+def _col_letter(n: int) -> str:
+    """0 -> A, 25 -> Z, 26 -> AA ... (Excel-style column naming)."""
+    s = ""
+    n += 1
+    while n:
+        n, rem = divmod(n - 1, 26)
+        s = chr(65 + rem) + s
+    return s
+
+def _trim_grid(df: pd.DataFrame) -> pd.DataFrame:
+    """Drop trailing all-empty columns/rows.
+
+    Excel files often advertise a "used range" far larger than the actual data
+    (leftover formatting on empty cells), which otherwise shows up as an
+    unbounded number of phantom empty columns/rows in the diff.
+    """
+    if df.empty:
+        return df
+    mask = df.notna() & df.astype(str).apply(lambda s: s.str.strip() != "")
+    col_has = mask.any(axis=0)
+    row_has = mask.any(axis=1)
+    if not col_has.any() or not row_has.any():
+        return df.iloc[0:0, 0:0]
+    last_col = col_has[col_has].index.max()
+    last_row = row_has[row_has].index.max()
+    return df.loc[:last_row, :last_col]
+
+def _load_grid(path: str, sheet_name: Optional[str] = None, header: bool = False) -> pd.DataFrame:
+    """Load a sheet as a raw cell grid (no assumed header row) unless header=True.
+
+    Every spreadsheet row becomes a data row, and columns are labelled A, B, C...
+    matching real Excel column letters, so line numbers line up with actual rows.
+    Trailing empty columns/rows (a common Excel "used range" artifact) are trimmed.
+    """
+    is_excel = str(path).lower().endswith(('.xls', '.xlsx', '.xlsm'))
+    read_header = 0 if header else None
+    if is_excel:
+        df = pd.read_excel(path, sheet_name=sheet_name if sheet_name is not None else 0, dtype=object, header=read_header)
+    else:
+        df = pd.read_csv(path, dtype=object, header=read_header)
+    df = _trim_grid(df)
+    if not header:
+        df.columns = [_col_letter(i) for i in range(len(df.columns))]
+    return df
+
+def _norm(df: pd.DataFrame) -> pd.DataFrame:
+    return df.fillna("").astype(str)
+
+Change = Tuple[str, Any, Optional[str], Optional[str], Optional[str]]
+
+def diff_sheets(left: str, right: str, key: Optional[str] = None, sheet_name: Optional[str] = None) -> List[Change]:
+    """Return list of changes: (type, row_key_or_index, column, old, new).
+    Types: 'added_row','removed_row','modified_cell'.
+    """
+    L = _norm(_load(left, sheet_name))
+    R = _norm(_load(right, sheet_name))
+
+    changes: List[Change] = []
+    if key:
+        L2 = L.set_index(key)
+        R2 = R.set_index(key)
+        all_keys = list(sorted(set(L2.index).union(R2.index)))
+        for k in all_keys:
+            inL = k in L2.index
+            inR = k in R2.index
+            if not inL:
+                changes.append(("added_row", k, None, None, None))
+                continue
+            if not inR:
+                changes.append(("removed_row", k, None, None, None))
+                continue
+            lrow = L2.loc[k]
+            rrow = R2.loc[k]
+            cols = sorted(set(lrow.index).union(rrow.index))
+            for c in cols:
+                a = str(lrow.get(c, ""))
+                b = str(rrow.get(c, ""))
+                if a != b:
+                    changes.append(("modified_cell", k, c, a, b))
+    else:
+        maxr = max(len(L), len(R))
+        for i in range(maxr):
+            if i >= len(L):
+                changes.append(("added_row", i, None, None, None)); continue
+            if i >= len(R):
+                changes.append(("removed_row", i, None, None, None)); continue
+            lrow = L.iloc[i]
+            rrow = R.iloc[i]
+            cols = sorted(set(lrow.index).union(rrow.index))
+            for c in cols:
+                a = str(lrow.get(c, ""))
+                b = str(rrow.get(c, ""))
+                if a != b:
+                    changes.append(("modified_cell", i, c, a, b))
+    return changes
+
+def _is_excel(path: str) -> bool:
+    return str(path).lower().endswith(('.xls', '.xlsx', '.xlsm'))
+
+def diff_workbook(left: str, right: str, key: Optional[str] = None) -> Dict[str, List[Change]]:
+    """Compare workbooks across all sheets. Returns dict: sheet_name -> changes."""
+    # If neither is excel, fallback to single-sheet comparison named 'sheet'
+    if not (_is_excel(left) or _is_excel(right)):
+        return {"sheet": diff_sheets(left, right, key=key)}
+
+    left_sheets = []
+    right_sheets = []
+    if _is_excel(left):
+        with pd.ExcelFile(left) as xl:
+            left_sheets = xl.sheet_names
+    else:
+        left_sheets = ["sheet"]
+    if _is_excel(right):
+        with pd.ExcelFile(right) as xl:
+            right_sheets = xl.sheet_names
+    else:
+        right_sheets = ["sheet"]
+
+    all_sheets = list(dict.fromkeys(list(left_sheets) + list(right_sheets)))
+    result: Dict[str, List[Change]] = {}
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp:
+        empty_path = None
+
+        def _empty_csv() -> str:
+            nonlocal empty_path
+            if empty_path is None:
+                empty_path = os.path.join(tmp, "empty.csv")
+                pd.DataFrame().to_csv(empty_path, index=False)
+            return empty_path
+
+        for s in all_sheets:
+            # If a sheet is missing in one workbook, represent as added/removed row marker
+            Lpath = left if s in left_sheets else _empty_csv()
+            Rpath = right if s in right_sheets else _empty_csv()
+            result[s] = diff_sheets(Lpath, Rpath, key=key, sheet_name=s)
+    return result
+
+
+def compare_workbooks(left: str, right: str, key: Optional[str] = None) -> Dict[str, List[Change]]:
+    """High-level comparator that prefers a Rust backend when available.
+
+    Returns a mapping of sheet_name -> list of Change tuples.
+    """
+    if _RUST_AVAILABLE:
+        try:
+            # rust function returns JSON string report
+            raw = _rust.compare_workbooks_json(left, right, None, None)
+            report = json.loads(raw)
+            out: Dict[str, List[Change]] = {}
+            for sheet in report.get("sheets", []):
+                name = sheet.get("sheet_name")
+                deltas = []
+                for d in sheet.get("cell_deltas", []):
+                    status = d.get("status")
+                    row_old = d.get("row_idx_old")
+                    row_new = d.get("row_idx_new")
+                    col = d.get("col_idx")
+                    oldv = d.get("old_value")
+                    newv = d.get("new_value")
+                    if status == "Added":
+                        deltas.append(("added_row", row_new, None, None, None))
+                    elif status == "Deleted":
+                        deltas.append(("removed_row", row_old, None, None, None))
+                    else:
+                        deltas.append(("modified_cell", row_new if row_new is not None else row_old, col, oldv, newv))
+                out[name] = deltas
+            return out
+        except Exception:
+            # On any rust failure, fallback to Python implementation
+            pass
+
+    return diff_workbook(left, right, key=key)
+
+def diff_rows_full(
+    left: str,
+    right: str,
+    key: Optional[str] = None,
+    sheet_name: Optional[str] = None,
+    header: bool = False,
+    left_exists: bool = True,
+    right_exists: bool = True,
+) -> Dict[str, Any]:
+    """Full row-level diff for GitHub-style rendering.
+
+    By default every spreadsheet row is treated as data (no assumed header row),
+    so line numbers match real Excel row numbers and columns are labelled A, B, C...
+    Pass header=True to treat row 1 as column names instead (then `key` is a column name).
+
+    left_exists/right_exists: pass False when the sheet is entirely absent from
+    that workbook (e.g. a sheet that was added or deleted) - every row of the
+    other side is then reported as added/removed rather than raising.
+
+    Returns {"columns": [...], "rows": [{"status": "unchanged"|"added"|"removed"|"modified",
+    "left": {col: val}, "right": {col: val}, "changed": [col, ...]}, ...]}.
+    """
+    L = _norm(_load_grid(left, sheet_name, header=header)) if left_exists else pd.DataFrame()
+    R = _norm(_load_grid(right, sheet_name, header=header)) if right_exists else pd.DataFrame()
+    columns = list(dict.fromkeys(list(L.columns) + list(R.columns)))
+    rows: List[Dict[str, Any]] = []
+
+    def row_dict(row, key_val=None) -> Dict[str, str]:
+        d = {c: str(row.get(c, "")) for c in columns}
+        if key_val is not None:
+            d[key] = str(key_val)
+        return d
+
+    if key and key in L.columns and key in R.columns:
+        L2, R2 = L.set_index(key), R.set_index(key)
+        for k in sorted(set(L2.index) | set(R2.index)):
+            inL, inR = k in L2.index, k in R2.index
+            if inL and not inR:
+                rows.append({"status": "removed", "left": row_dict(L2.loc[k], k), "right": None, "changed": []})
+            elif inR and not inL:
+                rows.append({"status": "added", "left": None, "right": row_dict(R2.loc[k], k), "changed": []})
+            else:
+                lrow, rrow = row_dict(L2.loc[k], k), row_dict(R2.loc[k], k)
+                changed = [c for c in columns if lrow.get(c) != rrow.get(c)]
+                rows.append({"status": "modified" if changed else "unchanged", "left": lrow, "right": rrow, "changed": changed})
+    else:
+        maxr = max(len(L), len(R))
+        for i in range(maxr):
+            if i >= len(L):
+                rows.append({"status": "added", "left": None, "right": row_dict(R.iloc[i]), "changed": []}); continue
+            if i >= len(R):
+                rows.append({"status": "removed", "left": row_dict(L.iloc[i]), "right": None, "changed": []}); continue
+            lrow, rrow = row_dict(L.iloc[i]), row_dict(R.iloc[i])
+            changed = [c for c in columns if lrow.get(c) != rrow.get(c)]
+            rows.append({"status": "modified" if changed else "unchanged", "left": lrow, "right": rrow, "changed": changed})
+
+    old_ln = new_ln = 0
+    for r in rows:
+        if r["status"] in ("removed", "modified", "unchanged"):
+            old_ln += 1
+            r["old_line"] = old_ln
+        else:
+            r["old_line"] = None
+        if r["status"] in ("added", "modified", "unchanged"):
+            new_ln += 1
+            r["new_line"] = new_ln
+        else:
+            r["new_line"] = None
+
+    return {"columns": columns, "rows": rows}
+
+
+def diff_multi_grid(
+    paths: List[str],
+    present: List[bool],
+    key: Optional[str] = None,
+    sheet_name: Optional[str] = None,
+    header: bool = False,
+) -> Dict[str, Any]:
+    """N-way row-aligned comparison across several files, for a side-by-side split view.
+
+    `present[i]` is False when this sheet doesn't exist in paths[i] at all.
+    The first file with the sheet present is the baseline; any other file's
+    cell that differs from the baseline value (when both have the row) is flagged.
+
+    Returns {"columns": [...], "rows": [{"line": n, "status": "same"|"diff",
+    "values": [dict|None, ...], "changed": [set of col per file index], "missing": [bool,...]}]}.
+    """
+    frames = []
+    for p, ok in zip(paths, present):
+        frames.append(_norm(_load_grid(p, sheet_name, header=header)) if ok else pd.DataFrame())
+
+    columns: List[str] = []
+    for f in frames:
+        for c in f.columns:
+            if c not in columns:
+                columns.append(c)
+
+    def row_dict(row, key_val=None) -> Dict[str, str]:
+        d = {c: str(row.get(c, "")) for c in columns}
+        if key_val is not None and key:
+            d[key] = str(key_val)
+        return d
+
+    rows: List[Dict[str, Any]] = []
+    use_key = bool(key) and any(key in f.columns for f in frames if not f.empty)
+
+    if use_key:
+        indexed = [f.set_index(key) if (not f.empty and key in f.columns) else f for f in frames]
+        all_keys = sorted({k for f in indexed for k in (f.index if not f.empty else [])})
+        for k in all_keys:
+            values = []
+            for f in indexed:
+                has = (not f.empty) and (k in getattr(f, "index", []))
+                values.append(row_dict(f.loc[k], k) if has else None)
+            rows.append(_build_multi_row(k, values, columns))
+    else:
+        maxr = max((len(f) for f in frames), default=0)
+        for i in range(maxr):
+            values = [row_dict(f.iloc[i]) if i < len(f) else None for f in frames]
+            rows.append(_build_multi_row(i, values, columns))
+
+    return {"columns": columns, "rows": rows, "file_count": len(paths)}
+
+def _build_multi_row(idx, values: List[Optional[Dict[str, str]]], columns: List[str]) -> Dict[str, Any]:
+    baseline = next((v for v in values if v is not None), None)
+    changed = set()
+    if baseline is not None:
+        for v in values:
+            if v is None:
+                continue
+            for c in columns:
+                if v.get(c, "") != baseline.get(c, ""):
+                    changed.add(c)
+    missing = [v is None for v in values]
+    if missing[0] and not all(missing):
+        status = "added"
+    elif not missing[0] and len(missing) > 1 and all(missing[1:]):
+        status = "removed"
+    elif changed or any(missing):
+        status = "diff"
+    else:
+        status = "same"
+    return {"line": idx, "status": status, "cells": values, "changed": sorted(changed), "missing": missing, "baseline": baseline}
+
+def format_unified(changes: List[Change]) -> str:
+    out: List[str] = []
+    for t, r, col, a, b in changes:
+        if t == "added_row":
+            out.append(f"+ ROW {r}")
+        elif t == "removed_row":
+            out.append(f"- ROW {r}")
+        else:
+            out.append(f"- {r} | {col} = {a}")
+            out.append(f"+ {r} | {col} = {b}")
+    return "\n".join(out)
+
+def format_workbook(changes_map: Dict[str, List[Change]]) -> str:
+    parts: List[str] = []
+    for sheet, changes in changes_map.items():
+        parts.append(f"--- Sheet: {sheet} ---")
+        parts.append(format_unified(changes) or "(no changes)")
+    return "\n\n".join(parts)