datarecon 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datarecon/__init__.py +5 -0
- datarecon/cli.py +57 -0
- datarecon/core.py +178 -0
- datarecon/normalize.py +106 -0
- datarecon/result.py +124 -0
- datarecon-0.1.0.dist-info/METADATA +61 -0
- datarecon-0.1.0.dist-info/RECORD +11 -0
- datarecon-0.1.0.dist-info/WHEEL +5 -0
- datarecon-0.1.0.dist-info/entry_points.txt +2 -0
- datarecon-0.1.0.dist-info/licenses/LICENSE +21 -0
- datarecon-0.1.0.dist-info/top_level.txt +1 -0
datarecon/__init__.py
ADDED
datarecon/cli.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import sys
|
|
5
|
+
from typing import Optional, Sequence
|
|
6
|
+
|
|
7
|
+
from .core import reconcile
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
11
|
+
parser = argparse.ArgumentParser(prog="datarecon", description="Explainable dataset reconciliation.")
|
|
12
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
13
|
+
|
|
14
|
+
cmp = sub.add_parser("compare", help="Compare two files.")
|
|
15
|
+
cmp.add_argument("source")
|
|
16
|
+
cmp.add_argument("target")
|
|
17
|
+
cmp.add_argument("--key", action="append", required=True, help="Key column; repeat or comma-separate.")
|
|
18
|
+
cmp.add_argument("--output", "-o", help="Write report to .xlsx (full) or .csv (differences only).")
|
|
19
|
+
cmp.add_argument("--tolerance", action="append", default=[], metavar="COL=VALUE")
|
|
20
|
+
cmp.add_argument("--fuzzy", type=float, help="Similarity threshold (0-1) for near-identical text.")
|
|
21
|
+
cmp.add_argument("--ignore-column", action="append", default=[])
|
|
22
|
+
cmp.add_argument("--treat-as-equal", default="", help="Comma-separated causes, e.g. case,whitespace.")
|
|
23
|
+
cmp.add_argument("--fail-on-diff", action="store_true", help="Exit with status 1 if differences exist.")
|
|
24
|
+
return parser
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
28
|
+
args = _build_parser().parse_args(argv)
|
|
29
|
+
|
|
30
|
+
keys = [k.strip() for entry in args.key for k in entry.split(",") if k.strip()]
|
|
31
|
+
rules = {}
|
|
32
|
+
for item in args.tolerance:
|
|
33
|
+
column, _, value = item.partition("=")
|
|
34
|
+
rules[column] = {"tolerance": float(value)}
|
|
35
|
+
|
|
36
|
+
result = reconcile(
|
|
37
|
+
args.source,
|
|
38
|
+
args.target,
|
|
39
|
+
keys,
|
|
40
|
+
rules=rules,
|
|
41
|
+
ignore_columns=args.ignore_column,
|
|
42
|
+
treat_as_equal={c.strip() for c in args.treat_as_equal.split(",") if c.strip()},
|
|
43
|
+
fuzzy_threshold=args.fuzzy,
|
|
44
|
+
)
|
|
45
|
+
sys.stdout.write(result.summary() + "\n")
|
|
46
|
+
|
|
47
|
+
if args.output:
|
|
48
|
+
if args.output.lower().endswith(".csv"):
|
|
49
|
+
result.differences.to_csv(args.output, index=False)
|
|
50
|
+
else:
|
|
51
|
+
result.to_excel(args.output)
|
|
52
|
+
|
|
53
|
+
return 1 if (args.fail_on_diff and result.has_differences) else 0
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
if __name__ == "__main__":
|
|
57
|
+
raise SystemExit(main())
|
datarecon/core.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any, Iterable, Mapping, Optional, Sequence, Union
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
from .normalize import FORMATTING_CAUSES, classify, normalize_key_part
|
|
10
|
+
from .result import ReconResult
|
|
11
|
+
|
|
12
|
+
Source = Union[pd.DataFrame, str, Path]
|
|
13
|
+
|
|
14
|
+
_DEFAULT_IGNORED = frozenset({"numeric_tolerance"})
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _norm_col(name: Any) -> str:
|
|
18
|
+
return str(name).strip().casefold()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _read(obj: Source) -> pd.DataFrame:
|
|
22
|
+
if isinstance(obj, pd.DataFrame):
|
|
23
|
+
return obj
|
|
24
|
+
path = Path(obj)
|
|
25
|
+
suffix = path.suffix.lower()
|
|
26
|
+
if suffix in {".csv", ".txt"}:
|
|
27
|
+
return pd.read_csv(path, dtype=str, keep_default_na=False)
|
|
28
|
+
if suffix == ".tsv":
|
|
29
|
+
return pd.read_csv(path, sep="\t", dtype=str, keep_default_na=False)
|
|
30
|
+
if suffix in {".xlsx", ".xlsm", ".xls"}:
|
|
31
|
+
return pd.read_excel(path, dtype=str, keep_default_na=False)
|
|
32
|
+
if suffix == ".parquet":
|
|
33
|
+
return pd.read_parquet(path)
|
|
34
|
+
raise ValueError(f"Unsupported file type: {path.suffix!r}")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _is_na(value: Any) -> bool:
|
|
38
|
+
return value is None or (not isinstance(value, str) and bool(pd.isna(value)))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _prepare(obj: Source) -> tuple[pd.DataFrame, dict[str, str]]:
|
|
42
|
+
"""Convert to an all-string frame (None for nulls) with normalized column names."""
|
|
43
|
+
df = _read(obj)
|
|
44
|
+
data: dict[str, list] = {}
|
|
45
|
+
display: dict[str, str] = {}
|
|
46
|
+
for col in df.columns:
|
|
47
|
+
name = _norm_col(col)
|
|
48
|
+
if name in data:
|
|
49
|
+
raise ValueError(f"Duplicate column after normalization: {col!r}")
|
|
50
|
+
data[name] = [None if _is_na(v) else str(v) for v in df[col].tolist()]
|
|
51
|
+
display[name] = str(col).strip()
|
|
52
|
+
return pd.DataFrame(data), display
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _add_key(df: pd.DataFrame, keys: list[str]) -> pd.DataFrame:
|
|
56
|
+
parts = [df[k].map(normalize_key_part).tolist() for k in keys]
|
|
57
|
+
raw = [df[k].tolist() for k in keys]
|
|
58
|
+
df = df.copy()
|
|
59
|
+
df["__key__"] = [
|
|
60
|
+
None if any(p is None for p in row) else "|".join(row) for row in zip(*parts)
|
|
61
|
+
]
|
|
62
|
+
df["__label__"] = ["|".join("" if v is None else v for v in row) for row in zip(*raw)]
|
|
63
|
+
return df
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _duplicates(df: pd.DataFrame, side: str) -> pd.DataFrame:
|
|
67
|
+
counts = df.groupby("__key__").agg(key=("__label__", "first"), occurrences=("__label__", "size"))
|
|
68
|
+
counts = counts[counts["occurrences"] > 1].reset_index(drop=True)
|
|
69
|
+
counts.insert(0, "side", side)
|
|
70
|
+
return counts
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _public(df: pd.DataFrame, display: Mapping[str, str]) -> pd.DataFrame:
|
|
74
|
+
out = df.drop(columns=["__label__"], errors="ignore").reset_index(drop=True)
|
|
75
|
+
return out.rename(columns=display)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def reconcile(
|
|
79
|
+
source: Source,
|
|
80
|
+
target: Source,
|
|
81
|
+
key: Union[str, Sequence[str]],
|
|
82
|
+
*,
|
|
83
|
+
rules: Optional[Mapping[str, Mapping[str, Any]]] = None,
|
|
84
|
+
ignore_columns: Iterable[str] = (),
|
|
85
|
+
treat_as_equal: Optional[Iterable[str]] = None,
|
|
86
|
+
fuzzy_threshold: Optional[float] = None,
|
|
87
|
+
) -> ReconResult:
|
|
88
|
+
"""Compare two datasets and explain every difference.
|
|
89
|
+
|
|
90
|
+
Parameters
|
|
91
|
+
----------
|
|
92
|
+
key
|
|
93
|
+
Column name (or list of names) that identifies a row. Names are matched
|
|
94
|
+
case-insensitively. Key values are compared after trimming, case folding
|
|
95
|
+
and removing leading zeros, so ``001234`` matches ``1234``.
|
|
96
|
+
rules
|
|
97
|
+
Per-column options: ``tolerance`` (absolute numeric tolerance),
|
|
98
|
+
``fuzzy`` (0-1 similarity threshold), ``date_formats`` (list of strptime formats).
|
|
99
|
+
treat_as_equal
|
|
100
|
+
Cause labels to treat as equal, for example ``{"case", "whitespace"}``.
|
|
101
|
+
``numeric_tolerance`` is always treated as equal.
|
|
102
|
+
fuzzy_threshold
|
|
103
|
+
Default similarity threshold for labelling near-identical text ``similar_text``.
|
|
104
|
+
"""
|
|
105
|
+
keys = [key] if isinstance(key, str) else list(key)
|
|
106
|
+
keys_n = [_norm_col(k) for k in keys]
|
|
107
|
+
|
|
108
|
+
src, src_names = _prepare(source)
|
|
109
|
+
tgt, tgt_names = _prepare(target)
|
|
110
|
+
for original, name in zip(keys, keys_n):
|
|
111
|
+
for side, frame in (("source", src), ("target", tgt)):
|
|
112
|
+
if name not in frame.columns:
|
|
113
|
+
raise KeyError(f"Key column {original!r} not found in {side}")
|
|
114
|
+
|
|
115
|
+
ignore = {_norm_col(c) for c in ignore_columns}
|
|
116
|
+
rules_n = {_norm_col(c): dict(r) for c, r in (rules or {}).items()}
|
|
117
|
+
ignored_causes = set(_DEFAULT_IGNORED) | set(treat_as_equal or ())
|
|
118
|
+
|
|
119
|
+
common = [c for c in src.columns if c in tgt.columns and c not in keys_n and c not in ignore]
|
|
120
|
+
only_src = [src_names[c] for c in src.columns if c not in tgt.columns]
|
|
121
|
+
only_tgt = [tgt_names[c] for c in tgt.columns if c not in src.columns]
|
|
122
|
+
compared = [src_names[c] for c in common]
|
|
123
|
+
|
|
124
|
+
src_rows, tgt_rows = len(src), len(tgt)
|
|
125
|
+
src, tgt = _add_key(src, keys_n), _add_key(tgt, keys_n)
|
|
126
|
+
null_src, null_tgt = int(src["__key__"].isna().sum()), int(tgt["__key__"].isna().sum())
|
|
127
|
+
src, tgt = src[src["__key__"].notna()], tgt[tgt["__key__"].notna()]
|
|
128
|
+
|
|
129
|
+
duplicates = pd.concat([_duplicates(src, "source"), _duplicates(tgt, "target")], ignore_index=True)
|
|
130
|
+
|
|
131
|
+
s_idx = src.drop_duplicates("__key__", keep="first").set_index("__key__")
|
|
132
|
+
t_idx = tgt.drop_duplicates("__key__", keep="first").set_index("__key__")
|
|
133
|
+
|
|
134
|
+
s_in_t = s_idx.index.isin(t_idx.index)
|
|
135
|
+
t_in_s = t_idx.index.isin(s_idx.index)
|
|
136
|
+
missing_in_target = _public(s_idx[~s_in_t], src_names)
|
|
137
|
+
missing_in_source = _public(t_idx[~t_in_s], tgt_names)
|
|
138
|
+
|
|
139
|
+
both = s_idx.index[s_in_t]
|
|
140
|
+
sa, ta = s_idx.loc[both, common], t_idx.loc[both, common]
|
|
141
|
+
labels = s_idx.loc[both, "__label__"].tolist()
|
|
142
|
+
|
|
143
|
+
equal = ((sa == ta) | (sa.isna() & ta.isna())).to_numpy()
|
|
144
|
+
sv = sa.astype(object).where(sa.notna(), None).to_numpy()
|
|
145
|
+
tv = ta.astype(object).where(ta.notna(), None).to_numpy()
|
|
146
|
+
|
|
147
|
+
records, rows_any, rows_real, ignored_cells = [], set(), set(), 0
|
|
148
|
+
for r, c in zip(*np.nonzero(~equal)):
|
|
149
|
+
col = common[c]
|
|
150
|
+
cause = classify(sv[r, c], tv[r, c], rules_n.get(col, {}), fuzzy_threshold)
|
|
151
|
+
if cause in ignored_causes:
|
|
152
|
+
ignored_cells += 1
|
|
153
|
+
continue
|
|
154
|
+
rows_any.add(r)
|
|
155
|
+
if cause not in FORMATTING_CAUSES:
|
|
156
|
+
rows_real.add(r)
|
|
157
|
+
records.append((labels[r], src_names[col], sv[r, c], tv[r, c], cause))
|
|
158
|
+
|
|
159
|
+
differences = pd.DataFrame(records, columns=["key", "column", "source", "target", "cause"])
|
|
160
|
+
|
|
161
|
+
return ReconResult(
|
|
162
|
+
source_rows=src_rows,
|
|
163
|
+
target_rows=tgt_rows,
|
|
164
|
+
matched_keys=len(both),
|
|
165
|
+
exact_rows=len(both) - len(rows_any),
|
|
166
|
+
formatting_rows=len(rows_any - rows_real),
|
|
167
|
+
mismatch_rows=len(rows_real),
|
|
168
|
+
ignored_cells=ignored_cells,
|
|
169
|
+
differences=differences,
|
|
170
|
+
missing_in_target=missing_in_target,
|
|
171
|
+
missing_in_source=missing_in_source,
|
|
172
|
+
duplicates=duplicates,
|
|
173
|
+
null_keys_source=null_src,
|
|
174
|
+
null_keys_target=null_tgt,
|
|
175
|
+
columns_only_in_source=only_src,
|
|
176
|
+
columns_only_in_target=only_tgt,
|
|
177
|
+
compared_columns=compared,
|
|
178
|
+
)
|
datarecon/normalize.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Cell-level comparison logic: decide *why* two values differ."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import math
|
|
5
|
+
import re
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from difflib import SequenceMatcher
|
|
8
|
+
from typing import Optional, Sequence
|
|
9
|
+
|
|
10
|
+
FORMATTING_CAUSES = frozenset(
|
|
11
|
+
{"null_vs_blank", "whitespace", "case", "leading_zeros", "numeric_format", "date_format"}
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
DEFAULT_DATE_FORMATS = (
|
|
15
|
+
"%Y-%m-%d",
|
|
16
|
+
"%Y-%m-%d %H:%M:%S",
|
|
17
|
+
"%Y-%m-%dT%H:%M:%S",
|
|
18
|
+
"%m/%d/%Y",
|
|
19
|
+
"%m/%d/%Y %H:%M:%S",
|
|
20
|
+
"%Y/%m/%d",
|
|
21
|
+
"%m-%d-%Y",
|
|
22
|
+
"%Y%m%d",
|
|
23
|
+
"%d-%b-%Y",
|
|
24
|
+
"%b %d, %Y",
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
_NULL_TOKENS = frozenset({"", "null", "none", "nan"})
|
|
28
|
+
_NUM_RE = re.compile(r"^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?$")
|
|
29
|
+
_KEY_NUM_RE = re.compile(r"^\d+(\.0+)?$")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def is_null(value: Optional[str]) -> bool:
|
|
33
|
+
return value is None or value.strip().casefold() in _NULL_TOKENS
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def normalize_key_part(value: Optional[str]) -> Optional[str]:
|
|
37
|
+
"""Canonical form of one key column value used only for matching rows."""
|
|
38
|
+
if value is None:
|
|
39
|
+
return None
|
|
40
|
+
text = value.strip().casefold()
|
|
41
|
+
if not text:
|
|
42
|
+
return None
|
|
43
|
+
if _KEY_NUM_RE.match(text):
|
|
44
|
+
text = text.split(".")[0].lstrip("0") or "0"
|
|
45
|
+
return text
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _to_float(text: str) -> Optional[float]:
|
|
49
|
+
cleaned = text.replace(",", "")
|
|
50
|
+
if not _NUM_RE.match(cleaned):
|
|
51
|
+
return None
|
|
52
|
+
number = float(cleaned)
|
|
53
|
+
return number if math.isfinite(number) else None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _to_datetime(text: str, formats: Sequence[str]) -> Optional[datetime]:
|
|
57
|
+
for fmt in formats:
|
|
58
|
+
try:
|
|
59
|
+
return datetime.strptime(text, fmt)
|
|
60
|
+
except ValueError:
|
|
61
|
+
continue
|
|
62
|
+
return None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def classify(
|
|
66
|
+
source: Optional[str],
|
|
67
|
+
target: Optional[str],
|
|
68
|
+
rule: dict,
|
|
69
|
+
fuzzy_default: Optional[float] = None,
|
|
70
|
+
) -> str:
|
|
71
|
+
"""Return the cause label for a pair of values already known to differ."""
|
|
72
|
+
s_null, t_null = is_null(source), is_null(target)
|
|
73
|
+
if s_null and t_null:
|
|
74
|
+
return "null_vs_blank"
|
|
75
|
+
if s_null or t_null:
|
|
76
|
+
return "value_mismatch"
|
|
77
|
+
|
|
78
|
+
if source.strip() == target.strip():
|
|
79
|
+
return "whitespace"
|
|
80
|
+
a, b = " ".join(source.split()), " ".join(target.split())
|
|
81
|
+
if a == b:
|
|
82
|
+
return "whitespace"
|
|
83
|
+
if a.casefold() == b.casefold():
|
|
84
|
+
return "case"
|
|
85
|
+
if a.isdigit() and b.isdigit() and a.lstrip("0") == b.lstrip("0"):
|
|
86
|
+
return "leading_zeros"
|
|
87
|
+
|
|
88
|
+
fa, fb = _to_float(a), _to_float(b)
|
|
89
|
+
if fa is not None and fb is not None:
|
|
90
|
+
if fa == fb:
|
|
91
|
+
return "numeric_format"
|
|
92
|
+
tolerance = rule.get("tolerance")
|
|
93
|
+
if tolerance is not None and abs(fa - fb) <= tolerance:
|
|
94
|
+
return "numeric_tolerance"
|
|
95
|
+
return "value_mismatch"
|
|
96
|
+
|
|
97
|
+
formats = rule.get("date_formats", DEFAULT_DATE_FORMATS)
|
|
98
|
+
da, db = _to_datetime(a, formats), _to_datetime(b, formats)
|
|
99
|
+
if da is not None and db is not None:
|
|
100
|
+
return "date_format" if da == db else "value_mismatch"
|
|
101
|
+
|
|
102
|
+
threshold = rule.get("fuzzy", fuzzy_default)
|
|
103
|
+
if threshold is not None:
|
|
104
|
+
if SequenceMatcher(None, a.casefold(), b.casefold()).ratio() >= threshold:
|
|
105
|
+
return "similar_text"
|
|
106
|
+
return "value_mismatch"
|
datarecon/result.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Union
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class ReconResult:
|
|
12
|
+
source_rows: int
|
|
13
|
+
target_rows: int
|
|
14
|
+
matched_keys: int
|
|
15
|
+
exact_rows: int
|
|
16
|
+
formatting_rows: int
|
|
17
|
+
mismatch_rows: int
|
|
18
|
+
ignored_cells: int
|
|
19
|
+
differences: pd.DataFrame
|
|
20
|
+
missing_in_target: pd.DataFrame
|
|
21
|
+
missing_in_source: pd.DataFrame
|
|
22
|
+
duplicates: pd.DataFrame
|
|
23
|
+
null_keys_source: int
|
|
24
|
+
null_keys_target: int
|
|
25
|
+
columns_only_in_source: list[str]
|
|
26
|
+
columns_only_in_target: list[str]
|
|
27
|
+
compared_columns: list[str]
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def has_differences(self) -> bool:
|
|
31
|
+
return bool(
|
|
32
|
+
len(self.differences)
|
|
33
|
+
or len(self.missing_in_target)
|
|
34
|
+
or len(self.missing_in_source)
|
|
35
|
+
or len(self.duplicates)
|
|
36
|
+
or self.null_keys_source
|
|
37
|
+
or self.null_keys_target
|
|
38
|
+
or self.columns_only_in_source
|
|
39
|
+
or self.columns_only_in_target
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
def cause_summary(self) -> pd.DataFrame:
|
|
43
|
+
return (
|
|
44
|
+
self.differences["cause"]
|
|
45
|
+
.value_counts()
|
|
46
|
+
.rename_axis("cause")
|
|
47
|
+
.reset_index(name="cells")
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def column_summary(self) -> pd.DataFrame:
|
|
51
|
+
if self.differences.empty:
|
|
52
|
+
return pd.DataFrame(columns=["column", "differences", "top_cause"])
|
|
53
|
+
grouped = self.differences.groupby("column")["cause"]
|
|
54
|
+
out = pd.DataFrame(
|
|
55
|
+
{"differences": grouped.size(), "top_cause": grouped.agg(lambda s: s.value_counts().index[0])}
|
|
56
|
+
)
|
|
57
|
+
return out.sort_values("differences", ascending=False, kind="stable").reset_index()
|
|
58
|
+
|
|
59
|
+
def metrics(self) -> pd.DataFrame:
|
|
60
|
+
dup_src = int((self.duplicates["side"] == "source").sum()) if len(self.duplicates) else 0
|
|
61
|
+
dup_tgt = int((self.duplicates["side"] == "target").sum()) if len(self.duplicates) else 0
|
|
62
|
+
rows = [
|
|
63
|
+
("Source rows", self.source_rows),
|
|
64
|
+
("Target rows", self.target_rows),
|
|
65
|
+
("Matched keys", self.matched_keys),
|
|
66
|
+
("Exact matches", self.exact_rows),
|
|
67
|
+
("Formatting differences only", self.formatting_rows),
|
|
68
|
+
("Value mismatches", self.mismatch_rows),
|
|
69
|
+
("Missing in target", len(self.missing_in_target)),
|
|
70
|
+
("Missing in source", len(self.missing_in_source)),
|
|
71
|
+
("Duplicate keys in source", dup_src),
|
|
72
|
+
("Duplicate keys in target", dup_tgt),
|
|
73
|
+
("Null keys in source", self.null_keys_source),
|
|
74
|
+
("Null keys in target", self.null_keys_target),
|
|
75
|
+
]
|
|
76
|
+
return pd.DataFrame(rows, columns=["metric", "value"])
|
|
77
|
+
|
|
78
|
+
def summary(self, top: int = 10) -> str:
|
|
79
|
+
lines = ["Reconciliation Summary", "-" * 42]
|
|
80
|
+
metrics = self.metrics()
|
|
81
|
+
always = {"Source rows", "Target rows", "Matched keys", "Exact matches"}
|
|
82
|
+
for label, value in metrics.itertuples(index=False):
|
|
83
|
+
if label in always or value:
|
|
84
|
+
indent = " " if label in {"Exact matches", "Formatting differences only", "Value mismatches"} else ""
|
|
85
|
+
lines.append(f"{indent}{label + ':':<30}{value:>10,}")
|
|
86
|
+
if self.ignored_cells:
|
|
87
|
+
lines.append(f"{'Cells treated as equal:':<32}{self.ignored_cells:>10,}")
|
|
88
|
+
|
|
89
|
+
if len(self.differences):
|
|
90
|
+
lines += ["", "Differences by cause:"]
|
|
91
|
+
for cause, cells in self.cause_summary().itertuples(index=False):
|
|
92
|
+
lines.append(f" {cause:<28}{cells:>10,}")
|
|
93
|
+
lines += ["", "Most problematic columns:"]
|
|
94
|
+
for i, (col, n, cause) in enumerate(self.column_summary().head(top).itertuples(index=False), 1):
|
|
95
|
+
lines.append(f" {i:>2}. {col:<24}{n:>8,} (mostly {cause})")
|
|
96
|
+
|
|
97
|
+
if self.columns_only_in_source:
|
|
98
|
+
lines += ["", "Columns only in source: " + ", ".join(self.columns_only_in_source)]
|
|
99
|
+
if self.columns_only_in_target:
|
|
100
|
+
lines += ["Columns only in target: " + ", ".join(self.columns_only_in_target)]
|
|
101
|
+
return "\n".join(lines)
|
|
102
|
+
|
|
103
|
+
def to_excel(self, path: Union[str, Path]) -> Path:
|
|
104
|
+
path = Path(path)
|
|
105
|
+
schema = pd.DataFrame(
|
|
106
|
+
[("compared", c) for c in self.compared_columns]
|
|
107
|
+
+ [("only_in_source", c) for c in self.columns_only_in_source]
|
|
108
|
+
+ [("only_in_target", c) for c in self.columns_only_in_target],
|
|
109
|
+
columns=["status", "column"],
|
|
110
|
+
)
|
|
111
|
+
with pd.ExcelWriter(path, engine="openpyxl") as writer:
|
|
112
|
+
self.metrics().to_excel(writer, sheet_name="Summary", index=False)
|
|
113
|
+
if len(self.differences):
|
|
114
|
+
start = len(self.metrics()) + 3
|
|
115
|
+
self.cause_summary().to_excel(writer, sheet_name="Summary", index=False, startrow=start)
|
|
116
|
+
self.column_summary().to_excel(
|
|
117
|
+
writer, sheet_name="Summary", index=False, startrow=start + len(self.cause_summary()) + 3
|
|
118
|
+
)
|
|
119
|
+
self.differences.to_excel(writer, sheet_name="Differences", index=False)
|
|
120
|
+
self.missing_in_target.to_excel(writer, sheet_name="Missing in target", index=False)
|
|
121
|
+
self.missing_in_source.to_excel(writer, sheet_name="Missing in source", index=False)
|
|
122
|
+
self.duplicates.to_excel(writer, sheet_name="Duplicate keys", index=False)
|
|
123
|
+
schema.to_excel(writer, sheet_name="Schema", index=False)
|
|
124
|
+
return path
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: datarecon
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Explainable dataset reconciliation: find out why two datasets do not match.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: pandas>=2.0
|
|
10
|
+
Requires-Dist: numpy>=1.23
|
|
11
|
+
Provides-Extra: excel
|
|
12
|
+
Requires-Dist: openpyxl>=3.1; extra == "excel"
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
15
|
+
Requires-Dist: openpyxl>=3.1; extra == "dev"
|
|
16
|
+
Dynamic: license-file
|
|
17
|
+
|
|
18
|
+
# datarecon
|
|
19
|
+
|
|
20
|
+
Compare two datasets and get an explanation of *why* they differ, not just a list of mismatched cells.
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from datarecon import reconcile
|
|
24
|
+
|
|
25
|
+
result = reconcile(
|
|
26
|
+
"source.csv", "warehouse_export.csv",
|
|
27
|
+
key="student_id",
|
|
28
|
+
rules={"score": {"tolerance": 0.01}},
|
|
29
|
+
treat_as_equal={"case", "whitespace"},
|
|
30
|
+
)
|
|
31
|
+
print(result.summary())
|
|
32
|
+
result.to_excel("reconciliation.xlsx")
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
datarecon compare source.csv target.csv --key student_id -o report.xlsx --fail-on-diff
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## What it detects
|
|
40
|
+
|
|
41
|
+
| Cause | Example |
|
|
42
|
+
|---|---|
|
|
43
|
+
| `leading_zeros` | `00123` vs `123` |
|
|
44
|
+
| `null_vs_blank` | `None` vs `""` vs `NULL` |
|
|
45
|
+
| `whitespace` / `case` | `Mary Jones` vs `MARY JONES` |
|
|
46
|
+
| `numeric_format` | `0.0` vs `00`, `10` vs `10.00` |
|
|
47
|
+
| `numeric_tolerance` | `10.004` vs `10.00` with `tolerance=0.01` |
|
|
48
|
+
| `date_format` | `2026-09-01` vs `09/01/2026` (month-first) |
|
|
49
|
+
| `similar_text` | `Ann Lee` vs `Anne Lee` (opt-in via `fuzzy_threshold`) |
|
|
50
|
+
| `value_mismatch` | everything else |
|
|
51
|
+
|
|
52
|
+
Also reported: rows missing on either side, duplicate keys, null keys, columns present on only one side.
|
|
53
|
+
|
|
54
|
+
Key values are matched after trimming, case folding and stripping leading zeros. Files are read as strings so no information is lost before comparison.
|
|
55
|
+
|
|
56
|
+
## Development
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
pip install -e ".[dev]"
|
|
60
|
+
pytest
|
|
61
|
+
```
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
datarecon/__init__.py,sha256=ofBcjKgzG7xOLR0bLTrTjk7RnrCOxP4iE8MWZZSbOiM,122
|
|
2
|
+
datarecon/cli.py,sha256=oe5WQ3Co5YlNVsPu_FdRq3WcxtcFqqD2rgDEFmJwVgs,2152
|
|
3
|
+
datarecon/core.py,sha256=fh4gwRUdHEKsG9Vu6ccGRNS220fvxm9G2a2MMkgIaY8,6888
|
|
4
|
+
datarecon/normalize.py,sha256=FksTutXvCq7ujZB3ddeN8sWPydxY5__cbBS0wUYZ2wU,3148
|
|
5
|
+
datarecon/result.py,sha256=6DzlVtquoFOYETSnf7nE-ceIKH-k3EP4lHs-97Z4SF8,5411
|
|
6
|
+
datarecon-0.1.0.dist-info/licenses/LICENSE,sha256=R2WgoW5qAvbDAqQlNMhcfAoQNQO8OZUcwjhuAgCrU48,1069
|
|
7
|
+
datarecon-0.1.0.dist-info/METADATA,sha256=lmc0vvn3rM__UDzSSPKex3iVk20DqjcaWHCE0v2Fp70,1803
|
|
8
|
+
datarecon-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
9
|
+
datarecon-0.1.0.dist-info/entry_points.txt,sha256=wkM5ipaHR-aY_CUPwpAc9FuBD7xpIlqa3HX5REhlQfE,49
|
|
10
|
+
datarecon-0.1.0.dist-info/top_level.txt,sha256=oRDUAb8ymI8WyfzvMLXLm0BdOxUwISCNirxaRz9V2h8,10
|
|
11
|
+
datarecon-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sameer Nimse
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
datarecon
|