tieback 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tieback/demo.py ADDED
@@ -0,0 +1,281 @@
1
+ #!/usr/bin/env python
2
+ """Build the three-extract sample book: policies, vehicles and claims.
3
+
4
+ A pricing submission as it actually arrives, from three systems that were
5
+ never designed to be joined:
6
+
7
+ policies.csv policy admin: one row per policy term. POLICY_NO reads
8
+ 'ON-0482113'; a renewal repeats it with the next term.
9
+ vehicles.csv vehicle schedule: one row per vehicle per term, with the
10
+ earned car-years (the exposure). POL_NUM is the bare number
11
+ 482113, and the term is identified by its start date.
12
+ claims.csv claims system: one row per claim per coverage. POLICY_REF
13
+ reads 'P0482113'; there is no term, only a loss date, and a
14
+ claim with three coverages is three rows.
15
+
16
+ Planted, as in real extracts: claims on policies missing from the policy
17
+ extract, losses dated after a mid-term cancellation, policy terms with no
18
+ vehicle on the schedule, and a reported date next to the loss date (only
19
+ the loss date places a claim in its term).
20
+
21
+ The frequency signal is textbook (U-shaped driver age, region, at-fault
22
+ history, use, rate group, distance), so the joined book models sensibly.
23
+
24
+ Usage: python -m tieback.demo [out_dir]
25
+ """
26
+ from __future__ import annotations
27
+
28
+ import sys
29
+ from datetime import date, timedelta
30
+ from pathlib import Path
31
+
32
+ import numpy as np
33
+ import polars as pl
34
+
35
+ OUT = Path("tieback_demo")
36
+
37
+ REGIONS = { # weight, frequency relativity
38
+ "Toronto": (0.22, 1.45), "GTA 905": (0.28, 1.25), "Hamilton-Niagara": (0.12, 1.05),
39
+ "Ottawa": (0.12, 0.90), "Southwestern": (0.16, 0.85), "Northern": (0.10, 0.75),
40
+ }
41
+ USES = {"Pleasure": (0.45, 0.85), "Commute": (0.45, 1.00), "Business": (0.10, 1.30)}
42
+ AT_FAULT = {0: (0.82, 1.00), 1: (0.14, 1.35), 2: (0.035, 1.70), 3: (0.005, 2.10)}
43
+ COVERAGES = { # weight as the first coverage, mean incurred, lognormal sigma
44
+ "DCPD": (0.45, 4_500, 0.7), "COLL": (0.30, 7_000, 0.8), "BI": (0.10, 45_000, 1.1), "AB": (0.15, 18_000, 1.0),
45
+ }
46
+ BASE_FREQ = 0.065
47
+
48
+
49
+ def _pick(rng, table: dict, n: int):
50
+ keys = list(table)
51
+ w = np.array([table[k][0] for k in keys])
52
+ return np.array(keys, dtype=object)[rng.choice(len(keys), n, p=w / w.sum())]
53
+
54
+
55
+ def build(out: Path = OUT, n_policies: int = 9_000, seed: int = 2027, *, messy: bool = False) -> dict[str, pl.DataFrame]:
56
+ """Write the sample book. `messy=True` adds two more files (finance
57
+ premium, driver convictions) and plants the problems real extracts have;
58
+ `build_with_truth` also returns what was planted."""
59
+ return build_with_truth(out, n_policies, seed, messy=messy)[0]
60
+
61
+
62
+ def build_with_truth(out: Path = OUT, n_policies: int = 9_000, seed: int = 2027, *,
63
+ messy: bool = False) -> tuple[dict[str, pl.DataFrame], dict]:
64
+ rng = np.random.default_rng(seed)
65
+ numbers = rng.choice(np.arange(100_000, 1_000_000), n_policies + 40, replace=False)
66
+ pol_nums, ghost_nums = numbers[:n_policies], numbers[n_policies:]
67
+
68
+ region = _pick(rng, REGIONS, n_policies)
69
+ age0 = np.clip(np.round(18 + rng.gamma(2.2, 11, n_policies)), 18, 88).astype(int)
70
+ licensed0 = np.minimum(age0 - 16, np.round(rng.uniform(0.3, 1.0, n_policies) * (age0 - 16))).astype(int)
71
+ at_fault = _pick(rng, AT_FAULT, n_policies).astype(int)
72
+ home = np.where(rng.random(n_policies) < 0.45, "Y", "N")
73
+ n_veh = rng.choice([1, 2, 3], n_policies, p=[0.60, 0.32, 0.08])
74
+ inception = np.array([date(2023, 1, 1) + timedelta(days=int(d)) for d in rng.integers(0, 365, n_policies)])
75
+
76
+ pol_rows, veh_rows, terms = [], [], []
77
+ for i in range(n_policies):
78
+ units = [
79
+ {"use": _pick(rng, USES, 1)[0], "group": int(rng.integers(1, 21)),
80
+ "km": int(rng.choice(np.arange(5_000, 40_001, 1_000))), "year": int(rng.integers(2006, 2025))}
81
+ for _ in range(n_veh[i])
82
+ ]
83
+ eff, term_no = inception[i], 1
84
+ while True:
85
+ cancelled = rng.random() < 0.08
86
+ length = int(rng.integers(30, 330)) if cancelled else 365
87
+ exp = eff + timedelta(days=length)
88
+ terms.append((i, term_no, eff, exp, cancelled, units))
89
+ pol_rows.append({
90
+ "POLICY_NO": f"ON-{pol_nums[i]:07d}", "TERM_NO": term_no, "TERM_EFF": eff, "TERM_EXP": exp,
91
+ "ORIG_INCEPTION": inception[i],
92
+ "REGION": region[i], "DRIVER_AGE": int(age0[i] + term_no - 1),
93
+ "YEARS_LICENSED": int(licensed0[i] + term_no - 1), "AT_FAULT_3YR": int(at_fault[i]), "HOME_POLICY": home[i],
94
+ })
95
+ if cancelled or term_no == 2 or rng.random() > 0.70:
96
+ break
97
+ eff, term_no = exp, term_no + 1
98
+
99
+ claim_rows, claim_seq = [], 0
100
+
101
+ def add_claim(ref: str, unit: int, loss: date):
102
+ nonlocal claim_seq
103
+ claim_seq += 1
104
+ first = _pick(rng, COVERAGES, 1)[0]
105
+ covs = [first]
106
+ if first in ("BI", "COLL") and rng.random() < 0.4:
107
+ covs.append("AB")
108
+ if rng.random() < 0.2:
109
+ covs.append("COLL" if first == "DCPD" else "DCPD")
110
+ reported = loss + timedelta(days=int(rng.geometric(1 / 12)))
111
+ for cov in dict.fromkeys(covs):
112
+ _, mean, sigma = COVERAGES[cov]
113
+ incurred = round(float(rng.lognormal(np.log(mean) - sigma**2 / 2, sigma)), 2)
114
+ claim_rows.append({
115
+ "CLAIM_NO": f"CLM-{claim_seq:07d}", "POLICY_REF": ref, "VEHICLE": unit, "COVERAGE": cov,
116
+ "LOSS_DATE": loss, "REPORTED_DATE": reported,
117
+ "INCURRED": incurred, "PAID": round(incurred * float(rng.uniform(0.3, 1.0)), 2),
118
+ })
119
+
120
+ for i, term_no, eff, exp, cancelled, units in terms:
121
+ age = age0[i] + term_no - 1
122
+ age_rel = 1 + 1.6 * np.exp(-(age - 18) / 6) + 0.012 * max(age - 65, 0)
123
+ base = BASE_FREQ * REGIONS[region[i]][1] * age_rel * AT_FAULT[at_fault[i]][1] * (0.9 if home[i] == "Y" else 1.0)
124
+ days = (exp - eff).days
125
+ for u, unit in enumerate(units, start=1):
126
+ expo = days / 365.25
127
+ veh_rows.append({
128
+ "POL_NUM": int(pol_nums[i]), "TERM_START": eff, "UNIT_NO": u, "VEH_YEAR": unit["year"],
129
+ "VEH_USE": unit["use"], "RATE_GROUP": unit["group"], "ANNUAL_KM": unit["km"],
130
+ "EARNED_CAR_YRS": round(expo, 4),
131
+ })
132
+ lam = base * USES[unit["use"]][1] * np.exp(0.03 * (unit["group"] - 10)) * (unit["km"] / 15_000) ** 0.3 * expo
133
+ for _ in range(rng.poisson(lam)):
134
+ add_claim(f"P{pol_nums[i]:07d}", u, eff + timedelta(days=int(rng.integers(0, days))))
135
+ # Losses reported after a mid-term cancellation: no term covers them.
136
+ if cancelled and rng.random() < 0.03:
137
+ add_claim(f"P{pol_nums[i]:07d}", 1, exp + timedelta(days=int(rng.integers(5, 90))))
138
+
139
+ # Claims on policies the policy extract does not contain.
140
+ for n in ghost_nums[:30]:
141
+ add_claim(f"P{n:07d}", 1, date(2024, 1, 1) + timedelta(days=int(rng.integers(0, 600))))
142
+ # Policy terms with no vehicle on the schedule (dropped by the vehicle system).
143
+ for j, n in enumerate(ghost_nums[30:]):
144
+ eff = date(2024, 1, 1) + timedelta(days=int(rng.integers(0, 300)))
145
+ pol_rows.append({
146
+ "POLICY_NO": f"ON-{n:07d}", "TERM_NO": 1, "TERM_EFF": eff, "TERM_EXP": eff + timedelta(days=365),
147
+ "ORIG_INCEPTION": eff, "REGION": "Toronto", "DRIVER_AGE": 40, "YEARS_LICENSED": 20,
148
+ "AT_FAULT_3YR": 0, "HOME_POLICY": "N",
149
+ })
150
+
151
+ # Each system exports in its own order.
152
+ frames = {
153
+ "policies.csv": pl.DataFrame(pol_rows).sample(fraction=1.0, shuffle=True, seed=seed),
154
+ "vehicles.csv": pl.DataFrame(veh_rows).sort("POL_NUM", "TERM_START", "UNIT_NO"),
155
+ "claims.csv": pl.DataFrame(claim_rows).sort("REPORTED_DATE", "CLAIM_NO"),
156
+ }
157
+ truth: dict = {}
158
+ if messy:
159
+ frames, truth = _mess_up(frames, np.random.default_rng(seed + 1))
160
+ out.mkdir(parents=True, exist_ok=True)
161
+ for name, df in frames.items():
162
+ df.write_csv(out / name)
163
+ return frames, truth
164
+
165
+
166
+ def _typo(rng, digits: str, kind: str, taken: set[str]) -> str | None:
167
+ """A mistyped copy of a digit string that is not itself a real key."""
168
+ for _ in range(50):
169
+ if kind == "swap":
170
+ i = int(rng.integers(0, len(digits) - 1))
171
+ if digits[i] == digits[i + 1]:
172
+ continue
173
+ new = digits[:i] + digits[i + 1] + digits[i] + digits[i + 2:]
174
+ else:
175
+ i = int(rng.integers(1, len(digits)))
176
+ new = digits[:i] + str((int(digits[i]) + int(rng.integers(1, 10))) % 10) + digits[i + 1:]
177
+ if new != digits and new.lstrip("0") not in taken:
178
+ return new
179
+ return None
180
+
181
+
182
+ def _mess_up(frames: dict[str, pl.DataFrame], rng) -> tuple[dict[str, pl.DataFrame], dict]:
183
+ """Two more files and the problems real extracts carry. Returns the files
184
+ and a record of every problem planted, for testing."""
185
+ pol, veh, clm = frames["policies.csv"], frames["vehicles.csv"], frames["claims.csv"]
186
+ taken = set(pol["POLICY_NO"].str.slice(3).str.strip_chars_start("0").to_list())
187
+ truth: dict = {}
188
+
189
+ # Finance premium: one row per policy term, id '0482113/1', dates day first.
190
+ prem = pol.select(
191
+ (pl.col("POLICY_NO").str.slice(3) + "/" + pl.col("TERM_NO").cast(pl.Utf8)).alias("GL_POLICY_TERM"),
192
+ pl.col("TERM_EFF").dt.strftime("%d/%m/%Y").alias("BOOKED_FROM"),
193
+ pl.Series("WRITTEN_PREMIUM", np.round(rng.uniform(700, 3_200, pol.height), 2)),
194
+ ).sample(fraction=1.0, shuffle=True, seed=1)
195
+ # Driver convictions: per policy, dated, several per policy term.
196
+ pick = rng.integers(0, pol.height, max(pol.height // 5, 50))
197
+ eff = pol["TERM_EFF"].to_list()
198
+ days = (pol["TERM_EXP"] - pol["TERM_EFF"]).dt.total_days().to_list()
199
+ conv = pl.DataFrame({
200
+ "CONVICTION_ID": [f"MTO-{i:06d}" for i in range(len(pick))],
201
+ "POLICYNUM": [pol["POLICY_NO"][int(i)][3:] for i in pick],
202
+ "CONV_DATE": [eff[int(i)] + timedelta(days=int(rng.integers(0, max(int(days[int(i)]), 1)))) for i in pick],
203
+ "OFFENCE": rng.choice(["Speeding", "Distracted driving", "Careless driving", "Fail to stop"], len(pick)),
204
+ })
205
+
206
+ # Mistyped policy numbers on claims: neighbours swapped (applied by
207
+ # default) and one digit changed (reported as likely).
208
+ claim_ids = clm.filter(pl.col("POLICY_REF").str.slice(1).str.strip_chars_start("0").is_in(list(taken)))["CLAIM_NO"].unique().sort()
209
+ chosen = rng.choice(claim_ids.to_list(), 12, replace=False)
210
+ truth["claim_swaps"], truth["claim_changes"] = {}, {}
211
+ fix = {}
212
+ for j, cid in enumerate(chosen):
213
+ ref = clm.filter(pl.col("CLAIM_NO") == cid)["POLICY_REF"][0]
214
+ kind = "swap" if j < 8 else "change"
215
+ new = _typo(rng, ref[1:], kind, taken)
216
+ if new:
217
+ fix[cid] = "P" + new
218
+ truth["claim_swaps" if kind == "swap" else "claim_changes"][cid] = (ref, "P" + new)
219
+ clm = clm.with_columns(pl.col("CLAIM_NO").replace_strict(fix, default=pl.col("POLICY_REF")).alias("POLICY_REF"))
220
+
221
+ # A mistyped policy number on the vehicle schedule (the exposure file).
222
+ vrows = rng.choice(veh.height, 3, replace=False)
223
+ truth["vehicle_swaps"] = {}
224
+ nums = veh["POL_NUM"].to_list()
225
+ for r in vrows:
226
+ new = _typo(rng, f"{nums[r]}", "swap", taken)
227
+ if new:
228
+ truth["vehicle_swaps"][int(r)] = (nums[r], int(new))
229
+ nums[r] = int(new)
230
+ veh = veh.with_columns(pl.Series("POL_NUM", nums))
231
+
232
+ # Exposure longer than its term.
233
+ over = rng.choice(veh.height, 2, replace=False)
234
+ expo = veh["EARNED_CAR_YRS"].to_list()
235
+ for r in over:
236
+ expo[r] = round(expo[r] + 0.75, 4)
237
+ veh = veh.with_columns(pl.Series("EARNED_CAR_YRS", expo))
238
+ truth["exposure_over"] = 2
239
+
240
+ # One region spelt three ways.
241
+ reg = pol["REGION"].to_list()
242
+ idx = [i for i, r in enumerate(reg) if r == "Toronto"]
243
+ for i in idx[:20]:
244
+ reg[i] = "TORONTO"
245
+ for i in [i for i, r in enumerate(reg) if r == "Ottawa"][:5]:
246
+ reg[i] = "Otawa"
247
+ pol = pol.with_columns(pl.Series("REGION", reg))
248
+ truth["region_variants"] = {"TORONTO": "Toronto", "Otawa": "Ottawa"}
249
+
250
+ # Claim amounts: a sign error, paid above incurred, dates swapped.
251
+ rows = rng.choice(clm.height, 9, replace=False)
252
+ inc, paid = clm["INCURRED"].to_list(), clm["PAID"].to_list()
253
+ loss, rep = clm["LOSS_DATE"].to_list(), clm["REPORTED_DATE"].to_list()
254
+ for r in rows[:2]:
255
+ inc[r], paid[r] = -inc[r], 0.0
256
+ for r in rows[2:6]:
257
+ paid[r] = round(inc[r] * 1.3, 2)
258
+ for r in rows[6:9]:
259
+ loss[r], rep[r] = rep[r] + timedelta(days=3), loss[r]
260
+ clm = clm.with_columns(pl.Series("INCURRED", inc), pl.Series("PAID", paid),
261
+ pl.Series("LOSS_DATE", loss), pl.Series("REPORTED_DATE", rep))
262
+ truth.update(negative=2, paid_over=4, reported_before=3)
263
+
264
+ # Rows exported twice.
265
+ veh = pl.concat([veh, veh[rng.choice(veh.height, 3, replace=False)]]).sort("POL_NUM", "TERM_START", "UNIT_NO")
266
+ clm = pl.concat([clm, clm[rng.choice(clm.height, 4, replace=False)]]).sort("REPORTED_DATE", "CLAIM_NO")
267
+ truth.update(dup_vehicles=3, dup_claims=4)
268
+
269
+ return ({"policies.csv": pol, "vehicles.csv": veh, "claims.csv": clm,
270
+ "premium.csv": prem, "convictions.csv": conv}, truth)
271
+
272
+
273
+ def main() -> int:
274
+ out = Path(sys.argv[1]) if len(sys.argv) > 1 else OUT
275
+ for name, df in build(out, messy=True).items():
276
+ print(f"{name}: {df.height:,} rows x {df.width} cols")
277
+ return 0
278
+
279
+
280
+ if __name__ == "__main__":
281
+ sys.exit(main())