ins-pricing 0.1.6__py3-none-any.whl

ins_pricing v2/modelling/watchdog_run.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+import threading
+import time
+from typing import List, Optional
+
+try:
+    from .run_logging import configure_run_logging  # type: ignore
+except Exception:  # pragma: no cover
+    try:
+        from run_logging import configure_run_logging  # type: ignore
+    except Exception:  # pragma: no cover
+        configure_run_logging = None  # type: ignore
+
+
+def _split_argv(argv: List[str]) -> tuple[List[str], List[str]]:
+    if "--" not in argv:
+        raise ValueError("Missing '--' separator before the command to run.")
+    idx = argv.index("--")
+    return argv[:idx], argv[idx + 1 :]
+
+
+def _kill_process_tree(pid: int) -> None:
+    if pid <= 0:
+        return
+    if os.name == "nt":
+        subprocess.run(
+            ["taskkill", "/PID", str(pid), "/T", "/F"],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            check=False,
+        )
+        return
+    try:
+        os.killpg(pid, 15)
+        time.sleep(2)
+        os.killpg(pid, 9)
+    except Exception:
+        try:
+            os.kill(pid, 9)
+        except Exception:
+            pass
+
+
+def _reader_thread(
+    proc: subprocess.Popen, last_output_ts: dict, prefix: str = ""
+) -> None:
+    assert proc.stdout is not None
+    for line in proc.stdout:
+        last_output_ts["ts"] = time.time()
+        if prefix:
+            sys.stdout.write(prefix)
+        sys.stdout.write(line)
+        sys.stdout.flush()
+
+
+def _parse_args(before_cmd: List[str], cmd: List[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Run a command under a simple watchdog: if there is no stdout/stderr "
+            "output for N seconds, kill the whole process tree and restart. "
+            "Designed to pair with optuna_storage so BayesOpt can resume."
+        )
+    )
+    parser.add_argument(
+        "--idle-seconds",
+        type=int,
+        default=7200,
+        help="Restart if there is no output for this many seconds (default: 7200).",
+    )
+    parser.add_argument(
+        "--max-restarts",
+        type=int,
+        default=50,
+        help="Maximum restart attempts (default: 50).",
+    )
+    parser.add_argument(
+        "--restart-delay-seconds",
+        type=int,
+        default=10,
+        help="Delay between restarts (default: 10).",
+    )
+    parser.add_argument(
+        "--stop-on-nonzero-exit",
+        action="store_true",
+        help="If the command exits non-zero, stop instead of restarting.",
+    )
+    args = parser.parse_args(before_cmd)
+    if not cmd:
+        parser.error("Empty command after '--'.")
+    return args
+
+
+def run_with_watchdog(
+    cmd: List[str],
+    idle_seconds: int,
+    max_restarts: int,
+    restart_delay_seconds: int,
+    stop_on_nonzero_exit: bool,
+) -> int:
+    idle_seconds = max(1, int(idle_seconds))
+    max_restarts = max(0, int(max_restarts))
+    restart_delay_seconds = max(0, int(restart_delay_seconds))
+
+    attempt = 0
+    while True:
+        attempt += 1
+        print(
+            f"[watchdog] start attempt={attempt} idle_seconds={idle_seconds} cmd={cmd}",
+            flush=True,
+        )
+
+        creationflags = 0
+        start_new_session = False
+        if os.name == "nt":
+            creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
+        else:
+            start_new_session = True
+
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+            universal_newlines=True,
+            creationflags=creationflags,
+            start_new_session=start_new_session,
+        )
+
+        last_output_ts: dict = {"ts": time.time()}
+        reader = threading.Thread(
+            target=_reader_thread,
+            args=(proc, last_output_ts),
+            kwargs={"prefix": ""},
+            daemon=True,
+        )
+        reader.start()
+
+        killed_for_idle = False
+        exit_code: Optional[int] = None
+        while True:
+            exit_code = proc.poll()
+            if exit_code is not None:
+                break
+            idle_for = time.time() - float(last_output_ts["ts"])
+            if idle_for > idle_seconds:
+                killed_for_idle = True
+                print(
+                    f"[watchdog] idle>{idle_seconds}s (idle_for={int(idle_for)}s), killing pid={proc.pid}",
+                    flush=True,
+                )
+                _kill_process_tree(proc.pid)
+                break
+            time.sleep(5)
+
+        try:
+            proc.wait(timeout=30)
+        except Exception:
+            _kill_process_tree(proc.pid)
+
+        if exit_code is None:
+            exit_code = proc.poll() or 1
+
+        if exit_code == 0:
+            print("[watchdog] finished with exit_code=0", flush=True)
+            return 0
+
+        if stop_on_nonzero_exit and not killed_for_idle:
+            print(
+                f"[watchdog] command exited non-zero (exit_code={exit_code}); stop.",
+                flush=True,
+            )
+            return int(exit_code)
+
+        if attempt > max_restarts + 1:
+            print(
+                f"[watchdog] exceeded max_restarts={max_restarts}; last exit_code={exit_code}",
+                flush=True,
+            )
+            return int(exit_code)
+
+        print(
+            f"[watchdog] restart in {restart_delay_seconds}s (exit_code={exit_code}, killed_for_idle={killed_for_idle})",
+            flush=True,
+        )
+        if restart_delay_seconds:
+            time.sleep(restart_delay_seconds)
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    if configure_run_logging:
+        configure_run_logging(prefix="watchdog")
+    argv = list(sys.argv[1:] if argv is None else argv)
+    before_cmd, cmd = _split_argv(argv)
+    args = _parse_args(before_cmd, cmd)
+    return run_with_watchdog(
+        cmd=cmd,
+        idle_seconds=args.idle_seconds,
+        max_restarts=args.max_restarts,
+        restart_delay_seconds=args.restart_delay_seconds,
+        stop_on_nonzero_exit=bool(args.stop_on_nonzero_exit),
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ins_pricing v2/pricing/__init__.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from .calibration import apply_calibration, fit_calibration_factor
+from .data_quality import detect_leakage, profile_columns, validate_schema
+from .exposure import aggregate_policy_level, build_frequency_severity, compute_exposure
+from .factors import bin_numeric, build_factor_table
+from .monitoring import population_stability_index, psi_report
+from .rate_table import RateTable, apply_factor_tables, compute_base_rate, rate_premium
+
+__all__ = [
+    "apply_calibration",
+    "fit_calibration_factor",
+    "detect_leakage",
+    "profile_columns",
+    "validate_schema",
+    "aggregate_policy_level",
+    "build_frequency_severity",
+    "compute_exposure",
+    "bin_numeric",
+    "build_factor_table",
+    "population_stability_index",
+    "psi_report",
+    "RateTable",
+    "apply_factor_tables",
+    "compute_base_rate",
+    "rate_premium",
+]

ins_pricing v2/pricing/calibration.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing import Optional
+
+import numpy as np
+
+
+def fit_calibration_factor(
+    pred: np.ndarray,
+    actual: np.ndarray,
+    *,
+    weight: Optional[np.ndarray] = None,
+    target_lr: Optional[float] = None,
+) -> float:
+    """Fit a scalar calibration factor for premiums or pure premiums."""
+    pred = np.asarray(pred, dtype=float).reshape(-1)
+    actual = np.asarray(actual, dtype=float).reshape(-1)
+    if weight is not None:
+        weight = np.asarray(weight, dtype=float).reshape(-1)
+        if weight.shape[0] != pred.shape[0]:
+            raise ValueError("weight length must match pred length.")
+        pred = pred * weight
+        actual = actual * weight
+
+    pred_sum = float(np.sum(pred))
+    actual_sum = float(np.sum(actual))
+    if pred_sum <= 0:
+        return 1.0
+
+    if target_lr is None:
+        return actual_sum / pred_sum
+    if target_lr <= 0:
+        raise ValueError("target_lr must be positive.")
+    return actual_sum / (target_lr * pred_sum)
+
+
+def apply_calibration(pred: np.ndarray, factor: float) -> np.ndarray:
+    pred = np.asarray(pred, dtype=float)
+    return pred * float(factor)

ins_pricing v2/pricing/data_quality.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+from typing import Callable, Dict, Iterable, Optional
+
+import numpy as np
+import pandas as pd
+
+
+def _dtype_matches(actual: np.dtype, expected) -> bool:
+    if callable(expected):
+        return bool(expected(actual))
+    if isinstance(expected, (list, tuple, set)):
+        return any(_dtype_matches(actual, item) for item in expected)
+    try:
+        expected_dtype = np.dtype(expected)
+    except Exception:
+        return False
+    if pd.api.types.is_categorical_dtype(actual) and expected_dtype == np.dtype("category"):
+        return True
+    if pd.api.types.is_string_dtype(actual) and expected_dtype.kind in {"U", "S", "O"}:
+        return True
+    if np.issubdtype(actual, expected_dtype):
+        return True
+    return pd.api.types.is_dtype_equal(actual, expected_dtype)
+
+
+def validate_schema(
+    df: pd.DataFrame,
+    required_cols: Iterable[str],
+    dtypes: Optional[Dict[str, object]] = None,
+    *,
+    raise_on_error: bool = True,
+) -> Dict[str, object]:
+    """Validate required columns and optional dtypes."""
+    required = list(required_cols)
+    missing = [col for col in required if col not in df.columns]
+    dtype_mismatch: Dict[str, Dict[str, str]] = {}
+    if dtypes:
+        for col, expected in dtypes.items():
+            if col not in df.columns:
+                continue
+            actual = df[col].dtype
+            if not _dtype_matches(actual, expected):
+                dtype_mismatch[col] = {
+                    "expected": str(expected),
+                    "actual": str(actual),
+                }
+
+    ok = not missing and not dtype_mismatch
+    result = {"ok": ok, "missing": missing, "dtype_mismatch": dtype_mismatch}
+    if raise_on_error and not ok:
+        raise ValueError(f"Schema validation failed: {result}")
+    return result
+
+
+def profile_columns(
+    df: pd.DataFrame, cols: Optional[Iterable[str]] = None
+) -> pd.DataFrame:
+    """Basic column profiling for missing/uniques and numeric stats."""
+    columns = list(cols) if cols is not None else list(df.columns)
+    rows = []
+    for col in columns:
+        series = df[col]
+        n = len(series)
+        missing_ratio = float(series.isna().mean()) if n else 0.0
+        nunique = int(series.nunique(dropna=True))
+        unique_ratio = float(nunique / n) if n else 0.0
+        entry = {
+            "column": col,
+            "dtype": str(series.dtype),
+            "missing_ratio": missing_ratio,
+            "n_unique": nunique,
+            "unique_ratio": unique_ratio,
+        }
+        if pd.api.types.is_numeric_dtype(series):
+            entry.update(
+                {
+                    "min": float(series.min(skipna=True)),
+                    "max": float(series.max(skipna=True)),
+                    "mean": float(series.mean(skipna=True)),
+                }
+            )
+        rows.append(entry)
+    return pd.DataFrame(rows)
+
+
+def detect_leakage(
+    df: pd.DataFrame,
+    target_col: str,
+    *,
+    exclude_cols: Optional[Iterable[str]] = None,
+    corr_threshold: float = 0.995,
+) -> pd.DataFrame:
+    """Detect simple leakage via identical columns or very high correlation."""
+    if target_col not in df.columns:
+        raise ValueError("target_col not found.")
+    exclude = set(exclude_cols or [])
+    exclude.add(target_col)
+    target = df[target_col]
+    results = []
+    for col in df.columns:
+        if col in exclude:
+            continue
+        series = df[col]
+        reason = None
+        score = None
+        if series.equals(target):
+            reason = "identical"
+            score = 1.0
+        elif pd.api.types.is_numeric_dtype(series) and pd.api.types.is_numeric_dtype(target):
+            corr = series.corr(target)
+            if pd.notna(corr) and abs(corr) >= corr_threshold:
+                reason = "high_corr"
+                score = float(corr)
+        if reason:
+            results.append({"feature": col, "reason": reason, "score": score})
+    return pd.DataFrame(results).sort_values(by="score", ascending=False).reset_index(drop=True)

ins_pricing v2/pricing/exposure.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from typing import Iterable, Optional
+
+import numpy as np
+import pandas as pd
+
+
+def compute_exposure(
+    df: pd.DataFrame,
+    start_col: str,
+    end_col: str,
+    *,
+    unit: str = "year",
+    inclusive: bool = False,
+    clip_min: Optional[float] = 0.0,
+    clip_max: Optional[float] = None,
+) -> pd.Series:
+    """Compute exposure from start/end date columns."""
+    start = pd.to_datetime(df[start_col])
+    end = pd.to_datetime(df[end_col])
+    delta_days = (end - start).dt.days.astype(float)
+    if inclusive:
+        delta_days = delta_days + 1.0
+    if unit == "day":
+        exposure = delta_days
+    elif unit == "month":
+        exposure = delta_days / 30.0
+    elif unit == "year":
+        exposure = delta_days / 365.25
+    else:
+        raise ValueError("unit must be one of: day, month, year.")
+
+    exposure = exposure.replace([np.inf, -np.inf], np.nan).fillna(0.0)
+    if clip_min is not None:
+        exposure = exposure.clip(lower=clip_min)
+    if clip_max is not None:
+        exposure = exposure.clip(upper=clip_max)
+    return exposure
+
+
+def aggregate_policy_level(
+    df: pd.DataFrame,
+    policy_keys: Iterable[str],
+    *,
+    exposure_col: str,
+    claim_count_col: Optional[str] = None,
+    claim_amount_col: Optional[str] = None,
+    weight_col: Optional[str] = None,
+) -> pd.DataFrame:
+    """Aggregate event-level rows to policy-level records."""
+    agg = {exposure_col: "sum"}
+    if claim_count_col:
+        agg[claim_count_col] = "sum"
+    if claim_amount_col:
+        agg[claim_amount_col] = "sum"
+    if weight_col:
+        agg[weight_col] = "sum"
+    grouped = df.groupby(list(policy_keys), dropna=False).agg(agg).reset_index()
+    return grouped
+
+
+def build_frequency_severity(
+    df: pd.DataFrame,
+    *,
+    exposure_col: str,
+    claim_count_col: str,
+    claim_amount_col: str,
+    zero_severity: float = 0.0,
+) -> pd.DataFrame:
+    """Compute frequency, severity and pure premium from counts and losses."""
+    exposure = df[exposure_col].to_numpy(dtype=float, copy=False)
+    counts = df[claim_count_col].to_numpy(dtype=float, copy=False)
+    amounts = df[claim_amount_col].to_numpy(dtype=float, copy=False)
+
+    with np.errstate(divide="ignore", invalid="ignore"):
+        frequency = np.where(exposure > 0, counts / exposure, 0.0)
+        severity = np.where(counts > 0, amounts / counts, zero_severity)
+        pure_premium = frequency * severity
+
+    out = df.copy()
+    out["frequency"] = frequency
+    out["severity"] = severity
+    out["pure_premium"] = pure_premium
+    return out

ins_pricing v2/pricing/factors.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from typing import Optional, Tuple
+
+import numpy as np
+import pandas as pd
+
+
+def bin_numeric(
+    series: pd.Series,
+    *,
+    bins: int = 10,
+    method: str = "quantile",
+    labels: Optional[list] = None,
+    include_lowest: bool = True,
+) -> Tuple[pd.Series, np.ndarray]:
+    """Bin numeric series and return (binned, bin_edges)."""
+    if method == "quantile":
+        binned = pd.qcut(series, q=bins, duplicates="drop", labels=labels)
+        bin_edges = binned.cat.categories.left.to_numpy()
+    elif method == "uniform":
+        binned = pd.cut(series, bins=bins, include_lowest=include_lowest, labels=labels)
+        bin_edges = binned.cat.categories.left.to_numpy()
+    else:
+        raise ValueError("method must be one of: quantile, uniform.")
+    return binned, bin_edges
+
+
+def build_factor_table(
+    df: pd.DataFrame,
+    *,
+    factor_col: str,
+    loss_col: str,
+    exposure_col: str,
+    weight_col: Optional[str] = None,
+    base_rate: Optional[float] = None,
+    smoothing: float = 0.0,
+    min_exposure: Optional[float] = None,
+) -> pd.DataFrame:
+    """Build a factor table with rate and relativity."""
+    if weight_col and weight_col in df.columns:
+        weights = df[weight_col].to_numpy(dtype=float, copy=False)
+    else:
+        weights = None
+
+    loss = df[loss_col].to_numpy(dtype=float, copy=False)
+    exposure = df[exposure_col].to_numpy(dtype=float, copy=False)
+
+    if weights is not None:
+        loss = loss * weights
+        exposure = exposure * weights
+
+    data = pd.DataFrame(
+        {
+            "factor": df[factor_col],
+            "loss": loss,
+            "exposure": exposure,
+        }
+    )
+    grouped = data.groupby("factor", dropna=False).agg({"loss": "sum", "exposure": "sum"})
+    grouped = grouped.reset_index().rename(columns={"factor": "level"})
+
+    if base_rate is None:
+        total_loss = float(grouped["loss"].sum())
+        total_exposure = float(grouped["exposure"].sum())
+        base_rate = total_loss / total_exposure if total_exposure > 0 else 0.0
+
+    exposure_vals = grouped["exposure"].to_numpy(dtype=float, copy=False)
+    loss_vals = grouped["loss"].to_numpy(dtype=float, copy=False)
+
+    with np.errstate(divide="ignore", invalid="ignore"):
+        rate = np.where(
+            exposure_vals > 0,
+            (loss_vals + smoothing * base_rate) / (exposure_vals + smoothing),
+            0.0,
+        )
+        relativity = np.where(base_rate > 0, rate / base_rate, 1.0)
+
+    grouped["rate"] = rate
+    grouped["relativity"] = relativity
+    grouped["base_rate"] = float(base_rate)
+
+    if min_exposure is not None:
+        low_exposure = grouped["exposure"] < float(min_exposure)
+        grouped.loc[low_exposure, "relativity"] = 1.0
+        grouped.loc[low_exposure, "rate"] = float(base_rate)
+        grouped["is_low_exposure"] = low_exposure
+    else:
+        grouped["is_low_exposure"] = False
+
+    return grouped

ins_pricing v2/pricing/monitoring.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from typing import Iterable, Optional
+
+import numpy as np
+import pandas as pd
+
+
+def psi_numeric(
+    expected: np.ndarray,
+    actual: np.ndarray,
+    *,
+    bins: int = 10,
+    strategy: str = "quantile",
+    eps: float = 1e-6,
+) -> float:
+    expected = np.asarray(expected, dtype=float)
+    actual = np.asarray(actual, dtype=float)
+    expected = expected[~np.isnan(expected)]
+    actual = actual[~np.isnan(actual)]
+    if expected.size == 0 or actual.size == 0:
+        return 0.0
+
+    if strategy == "quantile":
+        quantiles = np.linspace(0, 1, bins + 1)
+        bin_edges = np.quantile(expected, quantiles)
+        bin_edges = np.unique(bin_edges)
+    elif strategy == "uniform":
+        min_val = min(expected.min(), actual.min())
+        max_val = max(expected.max(), actual.max())
+        bin_edges = np.linspace(min_val, max_val, bins + 1)
+    else:
+        raise ValueError("strategy must be one of: quantile, uniform.")
+
+    if bin_edges.size < 2:
+        return 0.0
+
+    exp_counts, _ = np.histogram(expected, bins=bin_edges)
+    act_counts, _ = np.histogram(actual, bins=bin_edges)
+    exp_pct = exp_counts / max(exp_counts.sum(), 1)
+    act_pct = act_counts / max(act_counts.sum(), 1)
+    exp_pct = np.clip(exp_pct, eps, 1.0)
+    act_pct = np.clip(act_pct, eps, 1.0)
+    return float(np.sum((act_pct - exp_pct) * np.log(act_pct / exp_pct)))
+
+
+def psi_categorical(
+    expected: Iterable,
+    actual: Iterable,
+    *,
+    eps: float = 1e-6,
+) -> float:
+    expected = pd.Series(expected)
+    actual = pd.Series(actual)
+    categories = pd.Index(expected.dropna().unique()).union(actual.dropna().unique())
+    if categories.empty:
+        return 0.0
+    exp_counts = expected.value_counts().reindex(categories, fill_value=0)
+    act_counts = actual.value_counts().reindex(categories, fill_value=0)
+    exp_pct = exp_counts / max(exp_counts.sum(), 1)
+    act_pct = act_counts / max(act_counts.sum(), 1)
+    exp_pct = np.clip(exp_pct.to_numpy(dtype=float), eps, 1.0)
+    act_pct = np.clip(act_pct.to_numpy(dtype=float), eps, 1.0)
+    return float(np.sum((act_pct - exp_pct) * np.log(act_pct / exp_pct)))
+
+
+def population_stability_index(
+    expected: np.ndarray,
+    actual: np.ndarray,
+    *,
+    bins: int = 10,
+    strategy: str = "quantile",
+) -> float:
+    if pd.api.types.is_numeric_dtype(expected) and pd.api.types.is_numeric_dtype(actual):
+        return psi_numeric(expected, actual, bins=bins, strategy=strategy)
+    return psi_categorical(expected, actual)
+
+
+def psi_report(
+    expected_df: pd.DataFrame,
+    actual_df: pd.DataFrame,
+    *,
+    features: Optional[Iterable[str]] = None,
+    bins: int = 10,
+    strategy: str = "quantile",
+) -> pd.DataFrame:
+    feats = list(features) if features is not None else list(expected_df.columns)
+    rows = []
+    for feat in feats:
+        if feat not in expected_df.columns or feat not in actual_df.columns:
+            continue
+        psi = population_stability_index(
+            expected_df[feat].to_numpy(),
+            actual_df[feat].to_numpy(),
+            bins=bins,
+            strategy=strategy,
+        )
+        rows.append({"feature": feat, "psi": psi})
+    return pd.DataFrame(rows).sort_values(by="psi", ascending=False).reset_index(drop=True)