tieback 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tieback/__init__.py +49 -0
- tieback/checks.py +281 -0
- tieback/cli.py +80 -0
- tieback/demo.py +281 -0
- tieback/keys.py +567 -0
- tieback/plan.py +891 -0
- tieback/repair.py +277 -0
- tieback/report.py +148 -0
- tieback-0.2.0.dist-info/METADATA +158 -0
- tieback-0.2.0.dist-info/RECORD +13 -0
- tieback-0.2.0.dist-info/WHEEL +4 -0
- tieback-0.2.0.dist-info/entry_points.txt +2 -0
- tieback-0.2.0.dist-info/licenses/LICENSE +202 -0
tieback/__init__.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""tieback: join insurance extracts that were never meant to be joined, and
|
|
2
|
+
prove the result ties back to the totals.
|
|
3
|
+
|
|
4
|
+
import tieback
|
|
5
|
+
tables = tieback.read(["policies.csv", "vehicles.csv", "claims.csv"])
|
|
6
|
+
book, plan = tieback.assemble(tables)
|
|
7
|
+
print(tieback.report(plan))
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import polars as pl
|
|
15
|
+
|
|
16
|
+
from tieback.keys import (
|
|
17
|
+
JoinCandidate,
|
|
18
|
+
JoinDiagnostics,
|
|
19
|
+
JoinSpec,
|
|
20
|
+
ReconciliationResult,
|
|
21
|
+
detect_join_keys,
|
|
22
|
+
execute_join,
|
|
23
|
+
normalize_key,
|
|
24
|
+
)
|
|
25
|
+
from tieback.plan import KeyPair, Plan, Step, Window, assemble, parse_dates, replay
|
|
26
|
+
from tieback.report import TieOut, check_total, load_plan, report, save_plan
|
|
27
|
+
|
|
28
|
+
__version__ = "0.2.0"
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"JoinCandidate", "JoinDiagnostics", "JoinSpec", "KeyPair", "Plan", "ReconciliationResult", "Step",
|
|
32
|
+
"TieOut", "Window", "assemble", "check_total", "detect_join_keys", "execute_join", "load_plan",
|
|
33
|
+
"normalize_key", "parse_dates", "read", "replay", "report", "save_plan",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def read(paths: list[str | Path]) -> dict[str, pl.DataFrame]:
|
|
38
|
+
"""Read CSV or Parquet files into the {name: frame} mapping `assemble` takes.
|
|
39
|
+
Text dates are parsed later, by the plan. Ids that differ only by a prefix
|
|
40
|
+
or leading zeros ('ON-0482113' and 482113) are matched by the plan too."""
|
|
41
|
+
out: dict[str, pl.DataFrame] = {}
|
|
42
|
+
for p in map(Path, paths):
|
|
43
|
+
if p.name in out:
|
|
44
|
+
raise ValueError(f"Two files are named {p.name}; rename one.")
|
|
45
|
+
if p.suffix.lower() == ".parquet":
|
|
46
|
+
out[p.name] = pl.read_parquet(p)
|
|
47
|
+
else:
|
|
48
|
+
out[p.name] = pl.read_csv(p, infer_schema_length=10_000)
|
|
49
|
+
return out
|
tieback/checks.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
"""Everything else that is wrong with the extracts, found after the join.
|
|
2
|
+
|
|
3
|
+
Each finding says what is wrong, how many rows, what it does to the numbers
|
|
4
|
+
(exposure or money affected), a few examples, and what to do. Severity:
|
|
5
|
+
|
|
6
|
+
error the numbers are wrong if nothing is done (double exposure, a table
|
|
7
|
+
that did not join, a total that does not tie)
|
|
8
|
+
warning needs a decision (orphan claims, a spelling that splits a rating
|
|
9
|
+
group, a negative amount that may be a recovery)
|
|
10
|
+
info done for you and listed for the record (corrected keys)
|
|
11
|
+
|
|
12
|
+
Pure compute: frames and the plan in, a list of dicts out.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
from datetime import date
|
|
19
|
+
|
|
20
|
+
import polars as pl
|
|
21
|
+
|
|
22
|
+
from tieback.repair import _distance_one, _loose
|
|
23
|
+
|
|
24
|
+
_PAID = re.compile(r"paid", re.IGNORECASE)
|
|
25
|
+
_INCURRED = re.compile(r"incurred|incd", re.IGNORECASE)
|
|
26
|
+
_LOSS_DATE = re.compile(r"loss|accident|occur|incident", re.IGNORECASE)
|
|
27
|
+
_REPORT_DATE = re.compile(r"report|notif|fnol|opened", re.IGNORECASE)
|
|
28
|
+
# Findings whose examples are the whole list (each one needs a look).
|
|
29
|
+
_FULL = {"key_likely_typo", "key_suspect", "category_spellings"}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _f(check: str, severity: str, message: str, *, table: str | None = None, column: str | None = None,
|
|
33
|
+
count: int = 0, impact: dict | None = None, examples: list | None = None, action: str = "") -> dict:
|
|
34
|
+
return {"check": check, "severity": severity, "table": table, "column": column, "count": int(count),
|
|
35
|
+
"message": message, "impact": impact or {}, "examples": (examples or [])[:3] if check not in _FULL else (examples or []),
|
|
36
|
+
"action": action}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _money(x: float) -> str:
|
|
40
|
+
return f"{x:,.0f}"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _examples(df: pl.DataFrame, cols: list[str]) -> list[dict]:
|
|
44
|
+
cols = [c for c in cols if c in df.columns]
|
|
45
|
+
return [{k: (str(v) if v is not None else None) for k, v in r.items()} for r in df.select(cols).head(3).iter_rows(named=True)]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def run_checks(inputs: dict[str, pl.DataFrame], plan, book: pl.DataFrame, controls: dict[str, float]) -> list[dict]:
|
|
49
|
+
out: list[dict] = []
|
|
50
|
+
expo = plan.exposure
|
|
51
|
+
events = {s.table: s for s in plan.steps if s.kind in ("rollup", "group")}
|
|
52
|
+
key_cols = {(s.table, k.other) for s in plan.steps for k in s.keys} | {(plan.base, k.base) for s in plan.steps for k in s.keys}
|
|
53
|
+
id_cols = {(s.table, s.count_of) for s in plan.steps if s.count_of}
|
|
54
|
+
|
|
55
|
+
# ── Tables that did not join ─────────────────────────────────────────────
|
|
56
|
+
for u in plan.unresolved:
|
|
57
|
+
out.append(_f("not_joined", "error", f"{u['table']} could not be joined. {u['reason']}", table=u["table"],
|
|
58
|
+
count=inputs[u["table"]].height,
|
|
59
|
+
action="Check that it shares an id with the other files, or join it by hand and pass the result."))
|
|
60
|
+
|
|
61
|
+
# ── Duplicate rows ───────────────────────────────────────────────────────
|
|
62
|
+
for name, df in inputs.items():
|
|
63
|
+
extra = df.height - df.unique().height
|
|
64
|
+
if not extra:
|
|
65
|
+
continue
|
|
66
|
+
dup = df.filter(df.is_duplicated()).unique(maintain_order=True)
|
|
67
|
+
impact, sev = {}, "warning"
|
|
68
|
+
if name == plan.base and expo and expo in df.columns:
|
|
69
|
+
sev = "error"
|
|
70
|
+
impact[expo] = round(float(df[expo].sum() or 0) - float(df.unique()[expo].sum() or 0), 4)
|
|
71
|
+
if name in events:
|
|
72
|
+
sev = "error"
|
|
73
|
+
for c in events[name].sums:
|
|
74
|
+
impact[c] = round(float(df[c].sum() or 0) - float(df.unique()[c].sum() or 0), 2)
|
|
75
|
+
out.append(_f("duplicate_rows", sev, f"{name} has {extra:,} row{'s' if extra > 1 else ''} that repeat another row exactly.",
|
|
76
|
+
table=name, count=extra, impact=impact, examples=_examples(dup, dup.columns[:5]),
|
|
77
|
+
action="Drop the repeats if the source system exported them twice."))
|
|
78
|
+
|
|
79
|
+
# ── Overlapping periods and exposure that does not fit them ──────────────
|
|
80
|
+
win = next((s for s in plan.steps if s.window), None)
|
|
81
|
+
if win is not None and win.window.start in book.columns and win.window.end in book.columns:
|
|
82
|
+
st, en = win.window.start, win.window.end
|
|
83
|
+
ent = [k.base for k in win.keys if k.base in book.columns]
|
|
84
|
+
bad = book.filter(pl.col(en) <= pl.col(st))
|
|
85
|
+
if bad.height:
|
|
86
|
+
out.append(_f("period_ends_before_start", "error", f"{bad.height:,} {plan.base} rows end on or before they start ({st} to {en}).",
|
|
87
|
+
table=plan.base, count=bad.height, examples=_examples(bad, ent + [st, en])))
|
|
88
|
+
if ent:
|
|
89
|
+
srt = book.sort(ent + [st]).with_columns(pl.col(en).shift(1).over(ent).alias("__prev_end"))
|
|
90
|
+
ov = srt.filter(pl.col("__prev_end").is_not_null() & (pl.col(st) < pl.col("__prev_end")))
|
|
91
|
+
if ov.height:
|
|
92
|
+
days = ov.select((pl.min_horizontal(pl.col("__prev_end"), pl.col(en)) - pl.col(st)).dt.total_days().sum()).item() or 0
|
|
93
|
+
out.append(_f("overlapping_periods", "error",
|
|
94
|
+
f"{ov.height:,} {plan.base} rows overlap an earlier period of the same " + " and ".join(ent)
|
|
95
|
+
+ ": the overlap is exposure counted twice.",
|
|
96
|
+
table=plan.base, count=ov.height, impact={"overlap_years": round(days / 365.25, 2)},
|
|
97
|
+
examples=_examples(ov, ent + [st, en]),
|
|
98
|
+
action="Keep one row per period: usually a mid-term change exported as a second full term."))
|
|
99
|
+
if expo and expo in book.columns:
|
|
100
|
+
length = (pl.col(en) - pl.col(st)).dt.total_days() / 365.25
|
|
101
|
+
over = book.filter(pl.col(expo) > length * 1.01 + 0.003)
|
|
102
|
+
if over.height:
|
|
103
|
+
out.append(_f("exposure_exceeds_period", "error",
|
|
104
|
+
f"{over.height:,} {plan.base} rows carry more {expo} than their period is long.",
|
|
105
|
+
table=plan.base, column=expo, count=over.height,
|
|
106
|
+
impact={expo: round(float(over.select(pl.col(expo) - length).sum().item() or 0), 3)},
|
|
107
|
+
examples=_examples(over, ent + [st, en, expo])))
|
|
108
|
+
if expo and expo in book.columns:
|
|
109
|
+
bad = book.filter(pl.col(expo).is_null() | (pl.col(expo) <= 0))
|
|
110
|
+
if bad.height:
|
|
111
|
+
out.append(_f("exposure_missing_or_zero", "warning", f"{bad.height:,} {plan.base} rows have no positive {expo}.",
|
|
112
|
+
table=plan.base, column=expo, count=bad.height, examples=_examples(bad, [expo])))
|
|
113
|
+
|
|
114
|
+
# ── Amounts and dates in the event files ─────────────────────────────────
|
|
115
|
+
today = date.today()
|
|
116
|
+
for name, df in inputs.items():
|
|
117
|
+
step = events.get(name)
|
|
118
|
+
sums = step.sums if step else []
|
|
119
|
+
for c in sums:
|
|
120
|
+
neg = df.filter(pl.col(c) < 0)
|
|
121
|
+
if neg.height:
|
|
122
|
+
out.append(_f("negative_amount", "warning", f"{neg.height:,} {name} rows have a negative {c} (total {_money(float(neg[c].sum()))}).",
|
|
123
|
+
table=name, column=c, count=neg.height, impact={c: round(float(neg[c].sum()), 2)},
|
|
124
|
+
examples=_examples(neg, ([step.count_of] if step.count_of else []) + [c]),
|
|
125
|
+
action="Recoveries and reversals are negative on purpose; data-entry signs are not. Decide which."))
|
|
126
|
+
paid = next((c for c in sums if _PAID.search(c)), None)
|
|
127
|
+
inc = next((c for c in sums if _INCURRED.search(c)), None)
|
|
128
|
+
if paid and inc:
|
|
129
|
+
over = df.filter(pl.col(paid) > pl.col(inc) + 0.005)
|
|
130
|
+
if over.height:
|
|
131
|
+
out.append(_f("paid_exceeds_incurred", "warning", f"{over.height:,} {name} rows have {paid} above {inc}.",
|
|
132
|
+
table=name, count=over.height, impact={paid: round(float(over.select(pl.col(paid) - pl.col(inc)).sum().item()), 2)},
|
|
133
|
+
examples=_examples(over, ([step.count_of] if step and step.count_of else []) + [paid, inc]),
|
|
134
|
+
action="Incurred should include paid. Usually a stale case reserve or a mis-keyed amount."))
|
|
135
|
+
dates = [c for c, t in df.schema.items() if t == pl.Date]
|
|
136
|
+
loss = next((c for c in dates if _LOSS_DATE.search(c)), None)
|
|
137
|
+
rep = next((c for c in dates if _REPORT_DATE.search(c)), None)
|
|
138
|
+
if loss and rep:
|
|
139
|
+
bad = df.filter(pl.col(rep) < pl.col(loss))
|
|
140
|
+
if bad.height:
|
|
141
|
+
out.append(_f("reported_before_loss", "warning", f"{bad.height:,} {name} rows were reported before the loss happened ({rep} < {loss}).",
|
|
142
|
+
table=name, count=bad.height, examples=_examples(bad, ([step.count_of] if step and step.count_of else []) + [loss, rep]),
|
|
143
|
+
action="Often day and month swapped, or the two dates swapped. The loss date places the claim in its term."))
|
|
144
|
+
for c in dates:
|
|
145
|
+
odd = df.filter((pl.col(c) < date(1950, 1, 1)) | (pl.col(c) > date(today.year + 1, today.month, 1)))
|
|
146
|
+
if odd.height:
|
|
147
|
+
out.append(_f("implausible_date", "warning", f"{odd.height:,} {name} rows have a {c} before 1950 or more than a year ahead.",
|
|
148
|
+
table=name, column=c, count=odd.height, examples=_examples(odd, [c])))
|
|
149
|
+
|
|
150
|
+
# ── Spellings that split one category in two ─────────────────────────────
|
|
151
|
+
for name, df in inputs.items():
|
|
152
|
+
for c, t in df.schema.items():
|
|
153
|
+
if t != pl.Utf8 or (name, c) in key_cols or (name, c) in id_cols:
|
|
154
|
+
continue
|
|
155
|
+
vc = df[c].drop_nulls().value_counts(sort=True)
|
|
156
|
+
if vc.height < 2 or vc.height > 1_000 or vc.height > 0.5 * max(df.height, 1):
|
|
157
|
+
continue
|
|
158
|
+
vals = vc[c].to_list()
|
|
159
|
+
counts = dict(zip(vals, vc["count"].to_list()))
|
|
160
|
+
if sum(1 for v in vals if re.fullmatch(r"[\d.\-/ ]+", v)) > 0.5 * len(vals):
|
|
161
|
+
continue
|
|
162
|
+
mapping = {}
|
|
163
|
+
for i, v in enumerate(vals):
|
|
164
|
+
for u in vals[:i]:
|
|
165
|
+
if u in mapping:
|
|
166
|
+
continue
|
|
167
|
+
lv, lu = _loose(v), _loose(u)
|
|
168
|
+
same = lv == lu and v != u
|
|
169
|
+
near = (len(lu) >= 5 and _distance_one(lv, lu) is not None and counts[v] <= 0.1 * counts[u])
|
|
170
|
+
if same or near:
|
|
171
|
+
mapping[v] = u
|
|
172
|
+
break
|
|
173
|
+
if mapping:
|
|
174
|
+
rows = sum(counts[v] for v in mapping)
|
|
175
|
+
out.append(_f("category_spellings", "warning",
|
|
176
|
+
f"{name}.{c} spells {len(mapping)} value{'s' if len(mapping) > 1 else ''} two ways: "
|
|
177
|
+
+ "; ".join(f"'{v}' ({counts[v]:,}) looks like '{u}' ({counts[u]:,})" for v, u in list(mapping.items())[:4]) + ".",
|
|
178
|
+
table=name, column=c, count=rows, examples=[{"from": v, "to": u} for v, u in mapping.items()],
|
|
179
|
+
action="Each spelling becomes its own rating level. Map them to one before fitting."))
|
|
180
|
+
|
|
181
|
+
# ── Attributes that disagree between files ───────────────────────────────
|
|
182
|
+
for s in plan.steps:
|
|
183
|
+
if s.kind != "lookup":
|
|
184
|
+
continue
|
|
185
|
+
label = re.sub(r"\.(csv|xlsx?|txt|parquet)$", "", s.table, flags=re.IGNORECASE)
|
|
186
|
+
for c in s.added_columns:
|
|
187
|
+
if not c.endswith(f"_{label}"):
|
|
188
|
+
continue
|
|
189
|
+
mine = c[: -len(label) - 1]
|
|
190
|
+
if mine not in book.columns:
|
|
191
|
+
continue
|
|
192
|
+
both = book.filter(pl.col(mine).is_not_null() & pl.col(c).is_not_null())
|
|
193
|
+
try:
|
|
194
|
+
diff = both.filter(pl.col(mine).cast(pl.Utf8) != pl.col(c).cast(pl.Utf8))
|
|
195
|
+
except Exception:
|
|
196
|
+
continue
|
|
197
|
+
if diff.height:
|
|
198
|
+
out.append(_f("files_disagree", "warning",
|
|
199
|
+
f"{mine} differs between {plan.base} and {s.table} on {diff.height:,} rows.",
|
|
200
|
+
table=s.table, column=mine, count=diff.height, examples=_examples(diff, [mine, c]),
|
|
201
|
+
action="Decide which system is the source of truth for this field."))
|
|
202
|
+
|
|
203
|
+
# ── What did not attach ──────────────────────────────────────────────────
|
|
204
|
+
for s in plan.steps:
|
|
205
|
+
t = s.totals
|
|
206
|
+
if s.kind == "lookup":
|
|
207
|
+
if t.get("base_rows_unmatched"):
|
|
208
|
+
out.append(_f("rows_without_partner", "warning",
|
|
209
|
+
f"{t['base_rows_unmatched']:,} {plan.base} rows found no row in {s.table}: their {s.table} columns are empty.",
|
|
210
|
+
table=s.table, count=t["base_rows_unmatched"]))
|
|
211
|
+
continue
|
|
212
|
+
unit = t.get("unit", "row")
|
|
213
|
+
lost = {k: t.get(f"units_{k}", 0) for k in ("no_key", "outside_window", "ambiguous")}
|
|
214
|
+
n = sum(lost.values())
|
|
215
|
+
if n:
|
|
216
|
+
money = {c: round(a.get("no_key", 0) + a.get("outside_window", 0) + a.get("ambiguous", 0), 2)
|
|
217
|
+
for c, a in (t.get("amounts") or {}).items()}
|
|
218
|
+
parts = [f"{lost['no_key']:,} with no matching key" if lost["no_key"] else "",
|
|
219
|
+
f"{lost['outside_window']:,} outside every period" if lost["outside_window"] else "",
|
|
220
|
+
f"{lost['ambiguous']:,} matching several rows" if lost["ambiguous"] else ""]
|
|
221
|
+
losses = any(_PAID.search(c) or _INCURRED.search(c) or re.search(r"loss", c, re.I) for c in money)
|
|
222
|
+
unit_s = "rows" if unit == "row" else f"{unit}s"
|
|
223
|
+
out.append(_f("unattached_events", "warning",
|
|
224
|
+
f"{n:,} {unit_s} in {s.table} are not in the table: " + ", ".join(p for p in parts if p) + ".",
|
|
225
|
+
table=s.table, count=n, impact=money,
|
|
226
|
+
action=("Losses with no exposure: keep them in a separate load or chase the missing policies."
|
|
227
|
+
if losses else "They have no exposure row to sit on.")
|
|
228
|
+
+ " `tieback --replay` lists every one."))
|
|
229
|
+
|
|
230
|
+
# ── Keys: corrected, and suspected ───────────────────────────────────────
|
|
231
|
+
for s in plan.steps:
|
|
232
|
+
for r in s.repairs:
|
|
233
|
+
out.append(_f("key_corrected", "info",
|
|
234
|
+
f"{r['table']}.{r['column']}: '{r['original']}' read as '{r['corrected']}' ({r['kind']}; {r['evidence']}).",
|
|
235
|
+
table=r["table"], column=r["column"], count=r["rows"]))
|
|
236
|
+
groups: dict[tuple, list[dict]] = {}
|
|
237
|
+
for sp in plan.suspects:
|
|
238
|
+
groups.setdefault((sp["table"], sp["column"], bool(sp.get("likely"))), []).append(sp)
|
|
239
|
+
for (table, column, likely), sps in groups.items():
|
|
240
|
+
rows = sum(sp["rows"] for sp in sps)
|
|
241
|
+
n = len(sps)
|
|
242
|
+
if likely:
|
|
243
|
+
out.append(_f("key_likely_typo", "warning",
|
|
244
|
+
f"{table}.{column}: {n} value{'s match' if n > 1 else ' matches'} nothing but "
|
|
245
|
+
f"{'sit' if n > 1 else 'sits'} one character from a real key "
|
|
246
|
+
"whose other columns and dates fit, and only that one. Not applied: a policy missing from the "
|
|
247
|
+
"extract can sit one digit from a real one by chance.",
|
|
248
|
+
table=table, column=column, count=rows,
|
|
249
|
+
examples=[{"value": sp["value"], "likely": sp["likely"]} for sp in sps],
|
|
250
|
+
action="Confirm in the source system, or run with repair='all' to apply them."))
|
|
251
|
+
else:
|
|
252
|
+
out.append(_f("key_suspect", "warning",
|
|
253
|
+
f"{table}.{column}: {n} value{'s match' if n > 1 else ' matches'} nothing and "
|
|
254
|
+
f"{'sit' if n > 1 else 'sits'} one keystroke from real keys "
|
|
255
|
+
"that the other columns cannot tell apart.",
|
|
256
|
+
table=table, column=column, count=rows,
|
|
257
|
+
examples=[{"value": sp["value"], "near": ", ".join(sp["candidates"])} for sp in sps],
|
|
258
|
+
action="Look them up in the source system."))
|
|
259
|
+
# ── Control totals ───────────────────────────────────────────────────────
|
|
260
|
+
for col, expected in (controls or {}).items():
|
|
261
|
+
if col not in book.columns:
|
|
262
|
+
out.append(_f("control_total", "error", f"No column {col} in the table to tie to {expected:,}.", column=col))
|
|
263
|
+
continue
|
|
264
|
+
total = float(book[col].sum() or 0.0)
|
|
265
|
+
rel = (total - expected) / expected if expected else float("inf")
|
|
266
|
+
ok = abs(rel) <= 0.001
|
|
267
|
+
out.append(_f("control_total", "info" if ok else "error",
|
|
268
|
+
f"{col}: {total:,.2f} against {expected:,.2f} from finance, {rel:+.3%}" + (" (ties)" if ok else " (does not tie)"),
|
|
269
|
+
column=col, impact={col: round(total - expected, 2)}))
|
|
270
|
+
|
|
271
|
+
for f in out:
|
|
272
|
+
f["message"] = readable(f["message"])
|
|
273
|
+
if f.get("column"):
|
|
274
|
+
f["column"] = readable(f["column"])
|
|
275
|
+
order = {"error": 0, "warning": 1, "info": 2}
|
|
276
|
+
return sorted(out, key=lambda f: order[f["severity"]])
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def readable(text: str) -> str:
|
|
280
|
+
"""Part columns split out of a two-number id, as a person would say them."""
|
|
281
|
+
return re.sub(r"·(\d)", r" (part \1)", text)
|
tieback/cli.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Command line.
|
|
2
|
+
|
|
3
|
+
tieback sample/*.csv -o book.parquet --plan plan.json --findings findings.json
|
|
4
|
+
tieback sample/*.csv --control EARNED_CAR_YRS=21130 --repair all
|
|
5
|
+
tieback --replay plan.json sample/*.csv
|
|
6
|
+
tieback demo ./sample
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import tieback
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def main(argv: list[str] | None = None) -> int:
|
|
19
|
+
argv = sys.argv[1:] if argv is None else argv
|
|
20
|
+
if argv[:1] == ["demo"]:
|
|
21
|
+
from tieback.demo import build
|
|
22
|
+
out = Path(argv[1]) if len(argv) > 1 else Path("tieback_demo")
|
|
23
|
+
for name, df in build(out, messy=True).items():
|
|
24
|
+
print(f"{out / name}: {df.height:,} rows")
|
|
25
|
+
print("\nFive extracts from five systems, with the problems real ones have.")
|
|
26
|
+
print(f"Now run: tieback {out}/*.csv")
|
|
27
|
+
return 0
|
|
28
|
+
|
|
29
|
+
ap = argparse.ArgumentParser(prog="tieback", description="Join insurance extracts and check they tie back to the totals.")
|
|
30
|
+
ap.add_argument("files", nargs="+", help="CSV or Parquet extracts (two or more)")
|
|
31
|
+
ap.add_argument("-o", "--out", help="write the joined table here (.parquet or .csv)")
|
|
32
|
+
ap.add_argument("--plan", help="save the plan as JSON here")
|
|
33
|
+
ap.add_argument("--replay", metavar="PLAN", help="run a saved plan instead of detecting keys, and list the rows it missed")
|
|
34
|
+
ap.add_argument("--exposure", help="name of the exposure column, if it is not detected")
|
|
35
|
+
ap.add_argument("--repair", choices=["safe", "all", "off"], default="safe",
|
|
36
|
+
help="correct mistyped keys: safe (case, spacing, swapped neighbours; default), all (also a changed character), off")
|
|
37
|
+
ap.add_argument("--control", action="append", default=[], metavar="COLUMN=TOTAL",
|
|
38
|
+
help="a total from finance the table must tie to, e.g. EARNED_CAR_YRS=21130 (repeatable)")
|
|
39
|
+
ap.add_argument("--findings", metavar="FILE", help="write every finding as JSON here")
|
|
40
|
+
ap.add_argument("--version", action="version", version=f"tieback {tieback.__version__}")
|
|
41
|
+
a = ap.parse_args(argv)
|
|
42
|
+
|
|
43
|
+
tables = tieback.read(a.files)
|
|
44
|
+
if a.replay:
|
|
45
|
+
result = tieback.replay(tables, tieback.load_plan(a.replay))
|
|
46
|
+
from tieback.checks import readable
|
|
47
|
+
for name, grain in result["grain"].items():
|
|
48
|
+
print(readable(f"{name}: {grain}"))
|
|
49
|
+
for s in result["steps"]:
|
|
50
|
+
for m in s["misses"]:
|
|
51
|
+
print(f" {s['table']}: {m['rows'].height:,} rows not used ({m['why'].replace('_', ' ')})")
|
|
52
|
+
return 0
|
|
53
|
+
|
|
54
|
+
hints = {name: a.exposure for name in tables} if a.exposure else None
|
|
55
|
+
controls = {}
|
|
56
|
+
for c in a.control:
|
|
57
|
+
col, _, val = c.partition("=")
|
|
58
|
+
try:
|
|
59
|
+
controls[col] = float(val.replace(",", ""))
|
|
60
|
+
except ValueError:
|
|
61
|
+
ap.error(f"--control needs COLUMN=NUMBER, got {c!r}")
|
|
62
|
+
repair = {"safe": True, "all": "all", "off": False}[a.repair]
|
|
63
|
+
book, plan = tieback.assemble(tables, exposure_hints=hints, repair=repair, controls=controls)
|
|
64
|
+
print(tieback.report(plan))
|
|
65
|
+
if a.out:
|
|
66
|
+
out = Path(a.out)
|
|
67
|
+
book.write_csv(out) if out.suffix.lower() == ".csv" else book.write_parquet(out)
|
|
68
|
+
print(f"\nwrote {out} ({book.height:,} rows, {book.width} columns)")
|
|
69
|
+
if a.plan:
|
|
70
|
+
tieback.save_plan(plan, a.plan)
|
|
71
|
+
print(f"wrote {a.plan}")
|
|
72
|
+
if a.findings:
|
|
73
|
+
import json
|
|
74
|
+
Path(a.findings).write_text(json.dumps(plan.findings, indent=2, default=str))
|
|
75
|
+
print(f"wrote {a.findings} ({len(plan.findings)} findings)")
|
|
76
|
+
return 0
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
if __name__ == "__main__":
|
|
80
|
+
sys.exit(main())
|