tieback 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tieback/__init__.py ADDED
@@ -0,0 +1,49 @@
1
+ """tieback: join insurance extracts that were never meant to be joined, and
2
+ prove the result ties back to the totals.
3
+
4
+ import tieback
5
+ tables = tieback.read(["policies.csv", "vehicles.csv", "claims.csv"])
6
+ book, plan = tieback.assemble(tables)
7
+ print(tieback.report(plan))
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from pathlib import Path
13
+
14
+ import polars as pl
15
+
16
+ from tieback.keys import (
17
+ JoinCandidate,
18
+ JoinDiagnostics,
19
+ JoinSpec,
20
+ ReconciliationResult,
21
+ detect_join_keys,
22
+ execute_join,
23
+ normalize_key,
24
+ )
25
+ from tieback.plan import KeyPair, Plan, Step, Window, assemble, parse_dates, replay
26
+ from tieback.report import TieOut, check_total, load_plan, report, save_plan
27
+
28
+ __version__ = "0.2.0"
29
+
30
+ __all__ = [
31
+ "JoinCandidate", "JoinDiagnostics", "JoinSpec", "KeyPair", "Plan", "ReconciliationResult", "Step",
32
+ "TieOut", "Window", "assemble", "check_total", "detect_join_keys", "execute_join", "load_plan",
33
+ "normalize_key", "parse_dates", "read", "replay", "report", "save_plan",
34
+ ]
35
+
36
+
37
+ def read(paths: list[str | Path]) -> dict[str, pl.DataFrame]:
38
+ """Read CSV or Parquet files into the {name: frame} mapping `assemble` takes.
39
+ Text dates are parsed later, by the plan. Ids that differ only by a prefix
40
+ or leading zeros ('ON-0482113' and 482113) are matched by the plan too."""
41
+ out: dict[str, pl.DataFrame] = {}
42
+ for p in map(Path, paths):
43
+ if p.name in out:
44
+ raise ValueError(f"Two files are named {p.name}; rename one.")
45
+ if p.suffix.lower() == ".parquet":
46
+ out[p.name] = pl.read_parquet(p)
47
+ else:
48
+ out[p.name] = pl.read_csv(p, infer_schema_length=10_000)
49
+ return out
tieback/checks.py ADDED
@@ -0,0 +1,281 @@
1
+ """Everything else that is wrong with the extracts, found after the join.
2
+
3
+ Each finding says what is wrong, how many rows, what it does to the numbers
4
+ (exposure or money affected), a few examples, and what to do. Severity:
5
+
6
+ error the numbers are wrong if nothing is done (double exposure, a table
7
+ that did not join, a total that does not tie)
8
+ warning needs a decision (orphan claims, a spelling that splits a rating
9
+ group, a negative amount that may be a recovery)
10
+ info done for you and listed for the record (corrected keys)
11
+
12
+ Pure compute: frames and the plan in, a list of dicts out.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import re
18
+ from datetime import date
19
+
20
+ import polars as pl
21
+
22
+ from tieback.repair import _distance_one, _loose
23
+
24
+ _PAID = re.compile(r"paid", re.IGNORECASE)
25
+ _INCURRED = re.compile(r"incurred|incd", re.IGNORECASE)
26
+ _LOSS_DATE = re.compile(r"loss|accident|occur|incident", re.IGNORECASE)
27
+ _REPORT_DATE = re.compile(r"report|notif|fnol|opened", re.IGNORECASE)
28
+ # Findings whose examples are the whole list (each one needs a look).
29
+ _FULL = {"key_likely_typo", "key_suspect", "category_spellings"}
30
+
31
+
32
+ def _f(check: str, severity: str, message: str, *, table: str | None = None, column: str | None = None,
33
+ count: int = 0, impact: dict | None = None, examples: list | None = None, action: str = "") -> dict:
34
+ return {"check": check, "severity": severity, "table": table, "column": column, "count": int(count),
35
+ "message": message, "impact": impact or {}, "examples": (examples or [])[:3] if check not in _FULL else (examples or []),
36
+ "action": action}
37
+
38
+
39
+ def _money(x: float) -> str:
40
+ return f"{x:,.0f}"
41
+
42
+
43
+ def _examples(df: pl.DataFrame, cols: list[str]) -> list[dict]:
44
+ cols = [c for c in cols if c in df.columns]
45
+ return [{k: (str(v) if v is not None else None) for k, v in r.items()} for r in df.select(cols).head(3).iter_rows(named=True)]
46
+
47
+
48
+ def run_checks(inputs: dict[str, pl.DataFrame], plan, book: pl.DataFrame, controls: dict[str, float]) -> list[dict]:
49
+ out: list[dict] = []
50
+ expo = plan.exposure
51
+ events = {s.table: s for s in plan.steps if s.kind in ("rollup", "group")}
52
+ key_cols = {(s.table, k.other) for s in plan.steps for k in s.keys} | {(plan.base, k.base) for s in plan.steps for k in s.keys}
53
+ id_cols = {(s.table, s.count_of) for s in plan.steps if s.count_of}
54
+
55
+ # ── Tables that did not join ─────────────────────────────────────────────
56
+ for u in plan.unresolved:
57
+ out.append(_f("not_joined", "error", f"{u['table']} could not be joined. {u['reason']}", table=u["table"],
58
+ count=inputs[u["table"]].height,
59
+ action="Check that it shares an id with the other files, or join it by hand and pass the result."))
60
+
61
+ # ── Duplicate rows ───────────────────────────────────────────────────────
62
+ for name, df in inputs.items():
63
+ extra = df.height - df.unique().height
64
+ if not extra:
65
+ continue
66
+ dup = df.filter(df.is_duplicated()).unique(maintain_order=True)
67
+ impact, sev = {}, "warning"
68
+ if name == plan.base and expo and expo in df.columns:
69
+ sev = "error"
70
+ impact[expo] = round(float(df[expo].sum() or 0) - float(df.unique()[expo].sum() or 0), 4)
71
+ if name in events:
72
+ sev = "error"
73
+ for c in events[name].sums:
74
+ impact[c] = round(float(df[c].sum() or 0) - float(df.unique()[c].sum() or 0), 2)
75
+ out.append(_f("duplicate_rows", sev, f"{name} has {extra:,} row{'s' if extra > 1 else ''} that repeat another row exactly.",
76
+ table=name, count=extra, impact=impact, examples=_examples(dup, dup.columns[:5]),
77
+ action="Drop the repeats if the source system exported them twice."))
78
+
79
+ # ── Overlapping periods and exposure that does not fit them ──────────────
80
+ win = next((s for s in plan.steps if s.window), None)
81
+ if win is not None and win.window.start in book.columns and win.window.end in book.columns:
82
+ st, en = win.window.start, win.window.end
83
+ ent = [k.base for k in win.keys if k.base in book.columns]
84
+ bad = book.filter(pl.col(en) <= pl.col(st))
85
+ if bad.height:
86
+ out.append(_f("period_ends_before_start", "error", f"{bad.height:,} {plan.base} rows end on or before they start ({st} to {en}).",
87
+ table=plan.base, count=bad.height, examples=_examples(bad, ent + [st, en])))
88
+ if ent:
89
+ srt = book.sort(ent + [st]).with_columns(pl.col(en).shift(1).over(ent).alias("__prev_end"))
90
+ ov = srt.filter(pl.col("__prev_end").is_not_null() & (pl.col(st) < pl.col("__prev_end")))
91
+ if ov.height:
92
+ days = ov.select((pl.min_horizontal(pl.col("__prev_end"), pl.col(en)) - pl.col(st)).dt.total_days().sum()).item() or 0
93
+ out.append(_f("overlapping_periods", "error",
94
+ f"{ov.height:,} {plan.base} rows overlap an earlier period of the same " + " and ".join(ent)
95
+ + ": the overlap is exposure counted twice.",
96
+ table=plan.base, count=ov.height, impact={"overlap_years": round(days / 365.25, 2)},
97
+ examples=_examples(ov, ent + [st, en]),
98
+ action="Keep one row per period: usually a mid-term change exported as a second full term."))
99
+ if expo and expo in book.columns:
100
+ length = (pl.col(en) - pl.col(st)).dt.total_days() / 365.25
101
+ over = book.filter(pl.col(expo) > length * 1.01 + 0.003)
102
+ if over.height:
103
+ out.append(_f("exposure_exceeds_period", "error",
104
+ f"{over.height:,} {plan.base} rows carry more {expo} than their period is long.",
105
+ table=plan.base, column=expo, count=over.height,
106
+ impact={expo: round(float(over.select(pl.col(expo) - length).sum().item() or 0), 3)},
107
+ examples=_examples(over, ent + [st, en, expo])))
108
+ if expo and expo in book.columns:
109
+ bad = book.filter(pl.col(expo).is_null() | (pl.col(expo) <= 0))
110
+ if bad.height:
111
+ out.append(_f("exposure_missing_or_zero", "warning", f"{bad.height:,} {plan.base} rows have no positive {expo}.",
112
+ table=plan.base, column=expo, count=bad.height, examples=_examples(bad, [expo])))
113
+
114
+ # ── Amounts and dates in the event files ─────────────────────────────────
115
+ today = date.today()
116
+ for name, df in inputs.items():
117
+ step = events.get(name)
118
+ sums = step.sums if step else []
119
+ for c in sums:
120
+ neg = df.filter(pl.col(c) < 0)
121
+ if neg.height:
122
+ out.append(_f("negative_amount", "warning", f"{neg.height:,} {name} rows have a negative {c} (total {_money(float(neg[c].sum()))}).",
123
+ table=name, column=c, count=neg.height, impact={c: round(float(neg[c].sum()), 2)},
124
+ examples=_examples(neg, ([step.count_of] if step.count_of else []) + [c]),
125
+ action="Recoveries and reversals are negative on purpose; data-entry signs are not. Decide which."))
126
+ paid = next((c for c in sums if _PAID.search(c)), None)
127
+ inc = next((c for c in sums if _INCURRED.search(c)), None)
128
+ if paid and inc:
129
+ over = df.filter(pl.col(paid) > pl.col(inc) + 0.005)
130
+ if over.height:
131
+ out.append(_f("paid_exceeds_incurred", "warning", f"{over.height:,} {name} rows have {paid} above {inc}.",
132
+ table=name, count=over.height, impact={paid: round(float(over.select(pl.col(paid) - pl.col(inc)).sum().item()), 2)},
133
+ examples=_examples(over, ([step.count_of] if step and step.count_of else []) + [paid, inc]),
134
+ action="Incurred should include paid. Usually a stale case reserve or a mis-keyed amount."))
135
+ dates = [c for c, t in df.schema.items() if t == pl.Date]
136
+ loss = next((c for c in dates if _LOSS_DATE.search(c)), None)
137
+ rep = next((c for c in dates if _REPORT_DATE.search(c)), None)
138
+ if loss and rep:
139
+ bad = df.filter(pl.col(rep) < pl.col(loss))
140
+ if bad.height:
141
+ out.append(_f("reported_before_loss", "warning", f"{bad.height:,} {name} rows were reported before the loss happened ({rep} < {loss}).",
142
+ table=name, count=bad.height, examples=_examples(bad, ([step.count_of] if step and step.count_of else []) + [loss, rep]),
143
+ action="Often day and month swapped, or the two dates swapped. The loss date places the claim in its term."))
144
+ for c in dates:
145
+ odd = df.filter((pl.col(c) < date(1950, 1, 1)) | (pl.col(c) > date(today.year + 1, today.month, 1)))
146
+ if odd.height:
147
+ out.append(_f("implausible_date", "warning", f"{odd.height:,} {name} rows have a {c} before 1950 or more than a year ahead.",
148
+ table=name, column=c, count=odd.height, examples=_examples(odd, [c])))
149
+
150
+ # ── Spellings that split one category in two ─────────────────────────────
151
+ for name, df in inputs.items():
152
+ for c, t in df.schema.items():
153
+ if t != pl.Utf8 or (name, c) in key_cols or (name, c) in id_cols:
154
+ continue
155
+ vc = df[c].drop_nulls().value_counts(sort=True)
156
+ if vc.height < 2 or vc.height > 1_000 or vc.height > 0.5 * max(df.height, 1):
157
+ continue
158
+ vals = vc[c].to_list()
159
+ counts = dict(zip(vals, vc["count"].to_list()))
160
+ if sum(1 for v in vals if re.fullmatch(r"[\d.\-/ ]+", v)) > 0.5 * len(vals):
161
+ continue
162
+ mapping = {}
163
+ for i, v in enumerate(vals):
164
+ for u in vals[:i]:
165
+ if u in mapping:
166
+ continue
167
+ lv, lu = _loose(v), _loose(u)
168
+ same = lv == lu and v != u
169
+ near = (len(lu) >= 5 and _distance_one(lv, lu) is not None and counts[v] <= 0.1 * counts[u])
170
+ if same or near:
171
+ mapping[v] = u
172
+ break
173
+ if mapping:
174
+ rows = sum(counts[v] for v in mapping)
175
+ out.append(_f("category_spellings", "warning",
176
+ f"{name}.{c} spells {len(mapping)} value{'s' if len(mapping) > 1 else ''} two ways: "
177
+ + "; ".join(f"'{v}' ({counts[v]:,}) looks like '{u}' ({counts[u]:,})" for v, u in list(mapping.items())[:4]) + ".",
178
+ table=name, column=c, count=rows, examples=[{"from": v, "to": u} for v, u in mapping.items()],
179
+ action="Each spelling becomes its own rating level. Map them to one before fitting."))
180
+
181
+ # ── Attributes that disagree between files ───────────────────────────────
182
+ for s in plan.steps:
183
+ if s.kind != "lookup":
184
+ continue
185
+ label = re.sub(r"\.(csv|xlsx?|txt|parquet)$", "", s.table, flags=re.IGNORECASE)
186
+ for c in s.added_columns:
187
+ if not c.endswith(f"_{label}"):
188
+ continue
189
+ mine = c[: -len(label) - 1]
190
+ if mine not in book.columns:
191
+ continue
192
+ both = book.filter(pl.col(mine).is_not_null() & pl.col(c).is_not_null())
193
+ try:
194
+ diff = both.filter(pl.col(mine).cast(pl.Utf8) != pl.col(c).cast(pl.Utf8))
195
+ except Exception:
196
+ continue
197
+ if diff.height:
198
+ out.append(_f("files_disagree", "warning",
199
+ f"{mine} differs between {plan.base} and {s.table} on {diff.height:,} rows.",
200
+ table=s.table, column=mine, count=diff.height, examples=_examples(diff, [mine, c]),
201
+ action="Decide which system is the source of truth for this field."))
202
+
203
+ # ── What did not attach ──────────────────────────────────────────────────
204
+ for s in plan.steps:
205
+ t = s.totals
206
+ if s.kind == "lookup":
207
+ if t.get("base_rows_unmatched"):
208
+ out.append(_f("rows_without_partner", "warning",
209
+ f"{t['base_rows_unmatched']:,} {plan.base} rows found no row in {s.table}: their {s.table} columns are empty.",
210
+ table=s.table, count=t["base_rows_unmatched"]))
211
+ continue
212
+ unit = t.get("unit", "row")
213
+ lost = {k: t.get(f"units_{k}", 0) for k in ("no_key", "outside_window", "ambiguous")}
214
+ n = sum(lost.values())
215
+ if n:
216
+ money = {c: round(a.get("no_key", 0) + a.get("outside_window", 0) + a.get("ambiguous", 0), 2)
217
+ for c, a in (t.get("amounts") or {}).items()}
218
+ parts = [f"{lost['no_key']:,} with no matching key" if lost["no_key"] else "",
219
+ f"{lost['outside_window']:,} outside every period" if lost["outside_window"] else "",
220
+ f"{lost['ambiguous']:,} matching several rows" if lost["ambiguous"] else ""]
221
+ losses = any(_PAID.search(c) or _INCURRED.search(c) or re.search(r"loss", c, re.I) for c in money)
222
+ unit_s = "rows" if unit == "row" else f"{unit}s"
223
+ out.append(_f("unattached_events", "warning",
224
+ f"{n:,} {unit_s} in {s.table} are not in the table: " + ", ".join(p for p in parts if p) + ".",
225
+ table=s.table, count=n, impact=money,
226
+ action=("Losses with no exposure: keep them in a separate load or chase the missing policies."
227
+ if losses else "They have no exposure row to sit on.")
228
+ + " `tieback --replay` lists every one."))
229
+
230
+ # ── Keys: corrected, and suspected ───────────────────────────────────────
231
+ for s in plan.steps:
232
+ for r in s.repairs:
233
+ out.append(_f("key_corrected", "info",
234
+ f"{r['table']}.{r['column']}: '{r['original']}' read as '{r['corrected']}' ({r['kind']}; {r['evidence']}).",
235
+ table=r["table"], column=r["column"], count=r["rows"]))
236
+ groups: dict[tuple, list[dict]] = {}
237
+ for sp in plan.suspects:
238
+ groups.setdefault((sp["table"], sp["column"], bool(sp.get("likely"))), []).append(sp)
239
+ for (table, column, likely), sps in groups.items():
240
+ rows = sum(sp["rows"] for sp in sps)
241
+ n = len(sps)
242
+ if likely:
243
+ out.append(_f("key_likely_typo", "warning",
244
+ f"{table}.{column}: {n} value{'s match' if n > 1 else ' matches'} nothing but "
245
+ f"{'sit' if n > 1 else 'sits'} one character from a real key "
246
+ "whose other columns and dates fit, and only that one. Not applied: a policy missing from the "
247
+ "extract can sit one digit from a real one by chance.",
248
+ table=table, column=column, count=rows,
249
+ examples=[{"value": sp["value"], "likely": sp["likely"]} for sp in sps],
250
+ action="Confirm in the source system, or run with repair='all' to apply them."))
251
+ else:
252
+ out.append(_f("key_suspect", "warning",
253
+ f"{table}.{column}: {n} value{'s match' if n > 1 else ' matches'} nothing and "
254
+ f"{'sit' if n > 1 else 'sits'} one keystroke from real keys "
255
+ "that the other columns cannot tell apart.",
256
+ table=table, column=column, count=rows,
257
+ examples=[{"value": sp["value"], "near": ", ".join(sp["candidates"])} for sp in sps],
258
+ action="Look them up in the source system."))
259
+ # ── Control totals ───────────────────────────────────────────────────────
260
+ for col, expected in (controls or {}).items():
261
+ if col not in book.columns:
262
+ out.append(_f("control_total", "error", f"No column {col} in the table to tie to {expected:,}.", column=col))
263
+ continue
264
+ total = float(book[col].sum() or 0.0)
265
+ rel = (total - expected) / expected if expected else float("inf")
266
+ ok = abs(rel) <= 0.001
267
+ out.append(_f("control_total", "info" if ok else "error",
268
+ f"{col}: {total:,.2f} against {expected:,.2f} from finance, {rel:+.3%}" + (" (ties)" if ok else " (does not tie)"),
269
+ column=col, impact={col: round(total - expected, 2)}))
270
+
271
+ for f in out:
272
+ f["message"] = readable(f["message"])
273
+ if f.get("column"):
274
+ f["column"] = readable(f["column"])
275
+ order = {"error": 0, "warning": 1, "info": 2}
276
+ return sorted(out, key=lambda f: order[f["severity"]])
277
+
278
+
279
+ def readable(text: str) -> str:
280
+ """Part columns split out of a two-number id, as a person would say them."""
281
+ return re.sub(r"·(\d)", r" (part \1)", text)
tieback/cli.py ADDED
@@ -0,0 +1,80 @@
1
+ """Command line.
2
+
3
+ tieback sample/*.csv -o book.parquet --plan plan.json --findings findings.json
4
+ tieback sample/*.csv --control EARNED_CAR_YRS=21130 --repair all
5
+ tieback --replay plan.json sample/*.csv
6
+ tieback demo ./sample
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import sys
13
+ from pathlib import Path
14
+
15
+ import tieback
16
+
17
+
18
+ def main(argv: list[str] | None = None) -> int:
19
+ argv = sys.argv[1:] if argv is None else argv
20
+ if argv[:1] == ["demo"]:
21
+ from tieback.demo import build
22
+ out = Path(argv[1]) if len(argv) > 1 else Path("tieback_demo")
23
+ for name, df in build(out, messy=True).items():
24
+ print(f"{out / name}: {df.height:,} rows")
25
+ print("\nFive extracts from five systems, with the problems real ones have.")
26
+ print(f"Now run: tieback {out}/*.csv")
27
+ return 0
28
+
29
+ ap = argparse.ArgumentParser(prog="tieback", description="Join insurance extracts and check they tie back to the totals.")
30
+ ap.add_argument("files", nargs="+", help="CSV or Parquet extracts (two or more)")
31
+ ap.add_argument("-o", "--out", help="write the joined table here (.parquet or .csv)")
32
+ ap.add_argument("--plan", help="save the plan as JSON here")
33
+ ap.add_argument("--replay", metavar="PLAN", help="run a saved plan instead of detecting keys, and list the rows it missed")
34
+ ap.add_argument("--exposure", help="name of the exposure column, if it is not detected")
35
+ ap.add_argument("--repair", choices=["safe", "all", "off"], default="safe",
36
+ help="correct mistyped keys: safe (case, spacing, swapped neighbours; default), all (also a changed character), off")
37
+ ap.add_argument("--control", action="append", default=[], metavar="COLUMN=TOTAL",
38
+ help="a total from finance the table must tie to, e.g. EARNED_CAR_YRS=21130 (repeatable)")
39
+ ap.add_argument("--findings", metavar="FILE", help="write every finding as JSON here")
40
+ ap.add_argument("--version", action="version", version=f"tieback {tieback.__version__}")
41
+ a = ap.parse_args(argv)
42
+
43
+ tables = tieback.read(a.files)
44
+ if a.replay:
45
+ result = tieback.replay(tables, tieback.load_plan(a.replay))
46
+ from tieback.checks import readable
47
+ for name, grain in result["grain"].items():
48
+ print(readable(f"{name}: {grain}"))
49
+ for s in result["steps"]:
50
+ for m in s["misses"]:
51
+ print(f" {s['table']}: {m['rows'].height:,} rows not used ({m['why'].replace('_', ' ')})")
52
+ return 0
53
+
54
+ hints = {name: a.exposure for name in tables} if a.exposure else None
55
+ controls = {}
56
+ for c in a.control:
57
+ col, _, val = c.partition("=")
58
+ try:
59
+ controls[col] = float(val.replace(",", ""))
60
+ except ValueError:
61
+ ap.error(f"--control needs COLUMN=NUMBER, got {c!r}")
62
+ repair = {"safe": True, "all": "all", "off": False}[a.repair]
63
+ book, plan = tieback.assemble(tables, exposure_hints=hints, repair=repair, controls=controls)
64
+ print(tieback.report(plan))
65
+ if a.out:
66
+ out = Path(a.out)
67
+ book.write_csv(out) if out.suffix.lower() == ".csv" else book.write_parquet(out)
68
+ print(f"\nwrote {out} ({book.height:,} rows, {book.width} columns)")
69
+ if a.plan:
70
+ tieback.save_plan(plan, a.plan)
71
+ print(f"wrote {a.plan}")
72
+ if a.findings:
73
+ import json
74
+ Path(a.findings).write_text(json.dumps(plan.findings, indent=2, default=str))
75
+ print(f"wrote {a.findings} ({len(plan.findings)} findings)")
76
+ return 0
77
+
78
+
79
+ if __name__ == "__main__":
80
+ sys.exit(main())