tieback 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tieback/__init__.py +49 -0
- tieback/checks.py +281 -0
- tieback/cli.py +80 -0
- tieback/demo.py +281 -0
- tieback/keys.py +567 -0
- tieback/plan.py +891 -0
- tieback/repair.py +277 -0
- tieback/report.py +148 -0
- tieback-0.2.0.dist-info/METADATA +158 -0
- tieback-0.2.0.dist-info/RECORD +13 -0
- tieback-0.2.0.dist-info/WHEEL +4 -0
- tieback-0.2.0.dist-info/entry_points.txt +2 -0
- tieback-0.2.0.dist-info/licenses/LICENSE +202 -0
tieback/demo.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
"""Build the three-extract sample book: policies, vehicles and claims.
|
|
3
|
+
|
|
4
|
+
A pricing submission as it actually arrives, from three systems that were
|
|
5
|
+
never designed to be joined:
|
|
6
|
+
|
|
7
|
+
policies.csv policy admin: one row per policy term. POLICY_NO reads
|
|
8
|
+
'ON-0482113'; a renewal repeats it with the next term.
|
|
9
|
+
vehicles.csv vehicle schedule: one row per vehicle per term, with the
|
|
10
|
+
earned car-years (the exposure). POL_NUM is the bare number
|
|
11
|
+
482113, and the term is identified by its start date.
|
|
12
|
+
claims.csv claims system: one row per claim per coverage. POLICY_REF
|
|
13
|
+
reads 'P0482113'; there is no term, only a loss date, and a
|
|
14
|
+
claim with three coverages is three rows.
|
|
15
|
+
|
|
16
|
+
Planted, as in real extracts: claims on policies missing from the policy
|
|
17
|
+
extract, losses dated after a mid-term cancellation, policy terms with no
|
|
18
|
+
vehicle on the schedule, and a reported date next to the loss date (only
|
|
19
|
+
the loss date places a claim in its term).
|
|
20
|
+
|
|
21
|
+
The frequency signal is textbook (U-shaped driver age, region, at-fault
|
|
22
|
+
history, use, rate group, distance), so the joined book models sensibly.
|
|
23
|
+
|
|
24
|
+
Usage: python -m tieback.demo [out_dir]
|
|
25
|
+
"""
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import sys
|
|
29
|
+
from datetime import date, timedelta
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
import numpy as np
|
|
33
|
+
import polars as pl
|
|
34
|
+
|
|
35
|
+
OUT = Path("tieback_demo")
|
|
36
|
+
|
|
37
|
+
REGIONS = { # weight, frequency relativity
|
|
38
|
+
"Toronto": (0.22, 1.45), "GTA 905": (0.28, 1.25), "Hamilton-Niagara": (0.12, 1.05),
|
|
39
|
+
"Ottawa": (0.12, 0.90), "Southwestern": (0.16, 0.85), "Northern": (0.10, 0.75),
|
|
40
|
+
}
|
|
41
|
+
USES = {"Pleasure": (0.45, 0.85), "Commute": (0.45, 1.00), "Business": (0.10, 1.30)}
|
|
42
|
+
AT_FAULT = {0: (0.82, 1.00), 1: (0.14, 1.35), 2: (0.035, 1.70), 3: (0.005, 2.10)}
|
|
43
|
+
COVERAGES = { # weight as the first coverage, mean incurred, lognormal sigma
|
|
44
|
+
"DCPD": (0.45, 4_500, 0.7), "COLL": (0.30, 7_000, 0.8), "BI": (0.10, 45_000, 1.1), "AB": (0.15, 18_000, 1.0),
|
|
45
|
+
}
|
|
46
|
+
BASE_FREQ = 0.065
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _pick(rng, table: dict, n: int):
|
|
50
|
+
keys = list(table)
|
|
51
|
+
w = np.array([table[k][0] for k in keys])
|
|
52
|
+
return np.array(keys, dtype=object)[rng.choice(len(keys), n, p=w / w.sum())]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def build(out: Path = OUT, n_policies: int = 9_000, seed: int = 2027, *, messy: bool = False) -> dict[str, pl.DataFrame]:
|
|
56
|
+
"""Write the sample book. `messy=True` adds two more files (finance
|
|
57
|
+
premium, driver convictions) and plants the problems real extracts have;
|
|
58
|
+
`build_with_truth` also returns what was planted."""
|
|
59
|
+
return build_with_truth(out, n_policies, seed, messy=messy)[0]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def build_with_truth(out: Path = OUT, n_policies: int = 9_000, seed: int = 2027, *,
|
|
63
|
+
messy: bool = False) -> tuple[dict[str, pl.DataFrame], dict]:
|
|
64
|
+
rng = np.random.default_rng(seed)
|
|
65
|
+
numbers = rng.choice(np.arange(100_000, 1_000_000), n_policies + 40, replace=False)
|
|
66
|
+
pol_nums, ghost_nums = numbers[:n_policies], numbers[n_policies:]
|
|
67
|
+
|
|
68
|
+
region = _pick(rng, REGIONS, n_policies)
|
|
69
|
+
age0 = np.clip(np.round(18 + rng.gamma(2.2, 11, n_policies)), 18, 88).astype(int)
|
|
70
|
+
licensed0 = np.minimum(age0 - 16, np.round(rng.uniform(0.3, 1.0, n_policies) * (age0 - 16))).astype(int)
|
|
71
|
+
at_fault = _pick(rng, AT_FAULT, n_policies).astype(int)
|
|
72
|
+
home = np.where(rng.random(n_policies) < 0.45, "Y", "N")
|
|
73
|
+
n_veh = rng.choice([1, 2, 3], n_policies, p=[0.60, 0.32, 0.08])
|
|
74
|
+
inception = np.array([date(2023, 1, 1) + timedelta(days=int(d)) for d in rng.integers(0, 365, n_policies)])
|
|
75
|
+
|
|
76
|
+
pol_rows, veh_rows, terms = [], [], []
|
|
77
|
+
for i in range(n_policies):
|
|
78
|
+
units = [
|
|
79
|
+
{"use": _pick(rng, USES, 1)[0], "group": int(rng.integers(1, 21)),
|
|
80
|
+
"km": int(rng.choice(np.arange(5_000, 40_001, 1_000))), "year": int(rng.integers(2006, 2025))}
|
|
81
|
+
for _ in range(n_veh[i])
|
|
82
|
+
]
|
|
83
|
+
eff, term_no = inception[i], 1
|
|
84
|
+
while True:
|
|
85
|
+
cancelled = rng.random() < 0.08
|
|
86
|
+
length = int(rng.integers(30, 330)) if cancelled else 365
|
|
87
|
+
exp = eff + timedelta(days=length)
|
|
88
|
+
terms.append((i, term_no, eff, exp, cancelled, units))
|
|
89
|
+
pol_rows.append({
|
|
90
|
+
"POLICY_NO": f"ON-{pol_nums[i]:07d}", "TERM_NO": term_no, "TERM_EFF": eff, "TERM_EXP": exp,
|
|
91
|
+
"ORIG_INCEPTION": inception[i],
|
|
92
|
+
"REGION": region[i], "DRIVER_AGE": int(age0[i] + term_no - 1),
|
|
93
|
+
"YEARS_LICENSED": int(licensed0[i] + term_no - 1), "AT_FAULT_3YR": int(at_fault[i]), "HOME_POLICY": home[i],
|
|
94
|
+
})
|
|
95
|
+
if cancelled or term_no == 2 or rng.random() > 0.70:
|
|
96
|
+
break
|
|
97
|
+
eff, term_no = exp, term_no + 1
|
|
98
|
+
|
|
99
|
+
claim_rows, claim_seq = [], 0
|
|
100
|
+
|
|
101
|
+
def add_claim(ref: str, unit: int, loss: date):
|
|
102
|
+
nonlocal claim_seq
|
|
103
|
+
claim_seq += 1
|
|
104
|
+
first = _pick(rng, COVERAGES, 1)[0]
|
|
105
|
+
covs = [first]
|
|
106
|
+
if first in ("BI", "COLL") and rng.random() < 0.4:
|
|
107
|
+
covs.append("AB")
|
|
108
|
+
if rng.random() < 0.2:
|
|
109
|
+
covs.append("COLL" if first == "DCPD" else "DCPD")
|
|
110
|
+
reported = loss + timedelta(days=int(rng.geometric(1 / 12)))
|
|
111
|
+
for cov in dict.fromkeys(covs):
|
|
112
|
+
_, mean, sigma = COVERAGES[cov]
|
|
113
|
+
incurred = round(float(rng.lognormal(np.log(mean) - sigma**2 / 2, sigma)), 2)
|
|
114
|
+
claim_rows.append({
|
|
115
|
+
"CLAIM_NO": f"CLM-{claim_seq:07d}", "POLICY_REF": ref, "VEHICLE": unit, "COVERAGE": cov,
|
|
116
|
+
"LOSS_DATE": loss, "REPORTED_DATE": reported,
|
|
117
|
+
"INCURRED": incurred, "PAID": round(incurred * float(rng.uniform(0.3, 1.0)), 2),
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
for i, term_no, eff, exp, cancelled, units in terms:
|
|
121
|
+
age = age0[i] + term_no - 1
|
|
122
|
+
age_rel = 1 + 1.6 * np.exp(-(age - 18) / 6) + 0.012 * max(age - 65, 0)
|
|
123
|
+
base = BASE_FREQ * REGIONS[region[i]][1] * age_rel * AT_FAULT[at_fault[i]][1] * (0.9 if home[i] == "Y" else 1.0)
|
|
124
|
+
days = (exp - eff).days
|
|
125
|
+
for u, unit in enumerate(units, start=1):
|
|
126
|
+
expo = days / 365.25
|
|
127
|
+
veh_rows.append({
|
|
128
|
+
"POL_NUM": int(pol_nums[i]), "TERM_START": eff, "UNIT_NO": u, "VEH_YEAR": unit["year"],
|
|
129
|
+
"VEH_USE": unit["use"], "RATE_GROUP": unit["group"], "ANNUAL_KM": unit["km"],
|
|
130
|
+
"EARNED_CAR_YRS": round(expo, 4),
|
|
131
|
+
})
|
|
132
|
+
lam = base * USES[unit["use"]][1] * np.exp(0.03 * (unit["group"] - 10)) * (unit["km"] / 15_000) ** 0.3 * expo
|
|
133
|
+
for _ in range(rng.poisson(lam)):
|
|
134
|
+
add_claim(f"P{pol_nums[i]:07d}", u, eff + timedelta(days=int(rng.integers(0, days))))
|
|
135
|
+
# Losses reported after a mid-term cancellation: no term covers them.
|
|
136
|
+
if cancelled and rng.random() < 0.03:
|
|
137
|
+
add_claim(f"P{pol_nums[i]:07d}", 1, exp + timedelta(days=int(rng.integers(5, 90))))
|
|
138
|
+
|
|
139
|
+
# Claims on policies the policy extract does not contain.
|
|
140
|
+
for n in ghost_nums[:30]:
|
|
141
|
+
add_claim(f"P{n:07d}", 1, date(2024, 1, 1) + timedelta(days=int(rng.integers(0, 600))))
|
|
142
|
+
# Policy terms with no vehicle on the schedule (dropped by the vehicle system).
|
|
143
|
+
for j, n in enumerate(ghost_nums[30:]):
|
|
144
|
+
eff = date(2024, 1, 1) + timedelta(days=int(rng.integers(0, 300)))
|
|
145
|
+
pol_rows.append({
|
|
146
|
+
"POLICY_NO": f"ON-{n:07d}", "TERM_NO": 1, "TERM_EFF": eff, "TERM_EXP": eff + timedelta(days=365),
|
|
147
|
+
"ORIG_INCEPTION": eff, "REGION": "Toronto", "DRIVER_AGE": 40, "YEARS_LICENSED": 20,
|
|
148
|
+
"AT_FAULT_3YR": 0, "HOME_POLICY": "N",
|
|
149
|
+
})
|
|
150
|
+
|
|
151
|
+
# Each system exports in its own order.
|
|
152
|
+
frames = {
|
|
153
|
+
"policies.csv": pl.DataFrame(pol_rows).sample(fraction=1.0, shuffle=True, seed=seed),
|
|
154
|
+
"vehicles.csv": pl.DataFrame(veh_rows).sort("POL_NUM", "TERM_START", "UNIT_NO"),
|
|
155
|
+
"claims.csv": pl.DataFrame(claim_rows).sort("REPORTED_DATE", "CLAIM_NO"),
|
|
156
|
+
}
|
|
157
|
+
truth: dict = {}
|
|
158
|
+
if messy:
|
|
159
|
+
frames, truth = _mess_up(frames, np.random.default_rng(seed + 1))
|
|
160
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
161
|
+
for name, df in frames.items():
|
|
162
|
+
df.write_csv(out / name)
|
|
163
|
+
return frames, truth
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _typo(rng, digits: str, kind: str, taken: set[str]) -> str | None:
|
|
167
|
+
"""A mistyped copy of a digit string that is not itself a real key."""
|
|
168
|
+
for _ in range(50):
|
|
169
|
+
if kind == "swap":
|
|
170
|
+
i = int(rng.integers(0, len(digits) - 1))
|
|
171
|
+
if digits[i] == digits[i + 1]:
|
|
172
|
+
continue
|
|
173
|
+
new = digits[:i] + digits[i + 1] + digits[i] + digits[i + 2:]
|
|
174
|
+
else:
|
|
175
|
+
i = int(rng.integers(1, len(digits)))
|
|
176
|
+
new = digits[:i] + str((int(digits[i]) + int(rng.integers(1, 10))) % 10) + digits[i + 1:]
|
|
177
|
+
if new != digits and new.lstrip("0") not in taken:
|
|
178
|
+
return new
|
|
179
|
+
return None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _mess_up(frames: dict[str, pl.DataFrame], rng) -> tuple[dict[str, pl.DataFrame], dict]:
|
|
183
|
+
"""Two more files and the problems real extracts carry. Returns the files
|
|
184
|
+
and a record of every problem planted, for testing."""
|
|
185
|
+
pol, veh, clm = frames["policies.csv"], frames["vehicles.csv"], frames["claims.csv"]
|
|
186
|
+
taken = set(pol["POLICY_NO"].str.slice(3).str.strip_chars_start("0").to_list())
|
|
187
|
+
truth: dict = {}
|
|
188
|
+
|
|
189
|
+
# Finance premium: one row per policy term, id '0482113/1', dates day first.
|
|
190
|
+
prem = pol.select(
|
|
191
|
+
(pl.col("POLICY_NO").str.slice(3) + "/" + pl.col("TERM_NO").cast(pl.Utf8)).alias("GL_POLICY_TERM"),
|
|
192
|
+
pl.col("TERM_EFF").dt.strftime("%d/%m/%Y").alias("BOOKED_FROM"),
|
|
193
|
+
pl.Series("WRITTEN_PREMIUM", np.round(rng.uniform(700, 3_200, pol.height), 2)),
|
|
194
|
+
).sample(fraction=1.0, shuffle=True, seed=1)
|
|
195
|
+
# Driver convictions: per policy, dated, several per policy term.
|
|
196
|
+
pick = rng.integers(0, pol.height, max(pol.height // 5, 50))
|
|
197
|
+
eff = pol["TERM_EFF"].to_list()
|
|
198
|
+
days = (pol["TERM_EXP"] - pol["TERM_EFF"]).dt.total_days().to_list()
|
|
199
|
+
conv = pl.DataFrame({
|
|
200
|
+
"CONVICTION_ID": [f"MTO-{i:06d}" for i in range(len(pick))],
|
|
201
|
+
"POLICYNUM": [pol["POLICY_NO"][int(i)][3:] for i in pick],
|
|
202
|
+
"CONV_DATE": [eff[int(i)] + timedelta(days=int(rng.integers(0, max(int(days[int(i)]), 1)))) for i in pick],
|
|
203
|
+
"OFFENCE": rng.choice(["Speeding", "Distracted driving", "Careless driving", "Fail to stop"], len(pick)),
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
# Mistyped policy numbers on claims: neighbours swapped (applied by
|
|
207
|
+
# default) and one digit changed (reported as likely).
|
|
208
|
+
claim_ids = clm.filter(pl.col("POLICY_REF").str.slice(1).str.strip_chars_start("0").is_in(list(taken)))["CLAIM_NO"].unique().sort()
|
|
209
|
+
chosen = rng.choice(claim_ids.to_list(), 12, replace=False)
|
|
210
|
+
truth["claim_swaps"], truth["claim_changes"] = {}, {}
|
|
211
|
+
fix = {}
|
|
212
|
+
for j, cid in enumerate(chosen):
|
|
213
|
+
ref = clm.filter(pl.col("CLAIM_NO") == cid)["POLICY_REF"][0]
|
|
214
|
+
kind = "swap" if j < 8 else "change"
|
|
215
|
+
new = _typo(rng, ref[1:], kind, taken)
|
|
216
|
+
if new:
|
|
217
|
+
fix[cid] = "P" + new
|
|
218
|
+
truth["claim_swaps" if kind == "swap" else "claim_changes"][cid] = (ref, "P" + new)
|
|
219
|
+
clm = clm.with_columns(pl.col("CLAIM_NO").replace_strict(fix, default=pl.col("POLICY_REF")).alias("POLICY_REF"))
|
|
220
|
+
|
|
221
|
+
# A mistyped policy number on the vehicle schedule (the exposure file).
|
|
222
|
+
vrows = rng.choice(veh.height, 3, replace=False)
|
|
223
|
+
truth["vehicle_swaps"] = {}
|
|
224
|
+
nums = veh["POL_NUM"].to_list()
|
|
225
|
+
for r in vrows:
|
|
226
|
+
new = _typo(rng, f"{nums[r]}", "swap", taken)
|
|
227
|
+
if new:
|
|
228
|
+
truth["vehicle_swaps"][int(r)] = (nums[r], int(new))
|
|
229
|
+
nums[r] = int(new)
|
|
230
|
+
veh = veh.with_columns(pl.Series("POL_NUM", nums))
|
|
231
|
+
|
|
232
|
+
# Exposure longer than its term.
|
|
233
|
+
over = rng.choice(veh.height, 2, replace=False)
|
|
234
|
+
expo = veh["EARNED_CAR_YRS"].to_list()
|
|
235
|
+
for r in over:
|
|
236
|
+
expo[r] = round(expo[r] + 0.75, 4)
|
|
237
|
+
veh = veh.with_columns(pl.Series("EARNED_CAR_YRS", expo))
|
|
238
|
+
truth["exposure_over"] = 2
|
|
239
|
+
|
|
240
|
+
# One region spelt three ways.
|
|
241
|
+
reg = pol["REGION"].to_list()
|
|
242
|
+
idx = [i for i, r in enumerate(reg) if r == "Toronto"]
|
|
243
|
+
for i in idx[:20]:
|
|
244
|
+
reg[i] = "TORONTO"
|
|
245
|
+
for i in [i for i, r in enumerate(reg) if r == "Ottawa"][:5]:
|
|
246
|
+
reg[i] = "Otawa"
|
|
247
|
+
pol = pol.with_columns(pl.Series("REGION", reg))
|
|
248
|
+
truth["region_variants"] = {"TORONTO": "Toronto", "Otawa": "Ottawa"}
|
|
249
|
+
|
|
250
|
+
# Claim amounts: a sign error, paid above incurred, dates swapped.
|
|
251
|
+
rows = rng.choice(clm.height, 9, replace=False)
|
|
252
|
+
inc, paid = clm["INCURRED"].to_list(), clm["PAID"].to_list()
|
|
253
|
+
loss, rep = clm["LOSS_DATE"].to_list(), clm["REPORTED_DATE"].to_list()
|
|
254
|
+
for r in rows[:2]:
|
|
255
|
+
inc[r], paid[r] = -inc[r], 0.0
|
|
256
|
+
for r in rows[2:6]:
|
|
257
|
+
paid[r] = round(inc[r] * 1.3, 2)
|
|
258
|
+
for r in rows[6:9]:
|
|
259
|
+
loss[r], rep[r] = rep[r] + timedelta(days=3), loss[r]
|
|
260
|
+
clm = clm.with_columns(pl.Series("INCURRED", inc), pl.Series("PAID", paid),
|
|
261
|
+
pl.Series("LOSS_DATE", loss), pl.Series("REPORTED_DATE", rep))
|
|
262
|
+
truth.update(negative=2, paid_over=4, reported_before=3)
|
|
263
|
+
|
|
264
|
+
# Rows exported twice.
|
|
265
|
+
veh = pl.concat([veh, veh[rng.choice(veh.height, 3, replace=False)]]).sort("POL_NUM", "TERM_START", "UNIT_NO")
|
|
266
|
+
clm = pl.concat([clm, clm[rng.choice(clm.height, 4, replace=False)]]).sort("REPORTED_DATE", "CLAIM_NO")
|
|
267
|
+
truth.update(dup_vehicles=3, dup_claims=4)
|
|
268
|
+
|
|
269
|
+
return ({"policies.csv": pol, "vehicles.csv": veh, "claims.csv": clm,
|
|
270
|
+
"premium.csv": prem, "convictions.csv": conv}, truth)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def main() -> int:
|
|
274
|
+
out = Path(sys.argv[1]) if len(sys.argv) > 1 else OUT
|
|
275
|
+
for name, df in build(out, messy=True).items():
|
|
276
|
+
print(f"{name}: {df.height:,} rows x {df.width} cols")
|
|
277
|
+
return 0
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
if __name__ == "__main__":
|
|
281
|
+
sys.exit(main())
|