ins-pricing 0.1.6__py3-none-any.whl

ins_pricing/modelling/tests/test_explain.py

@@ -0,0 +1,56 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+
+def test_permutation_importance_simple():
+    from ins_pricing.explain import permutation_importance
+
+    rng = np.random.default_rng(0)
+    X = pd.DataFrame(
+        {
+            "x1": rng.normal(size=200),
+            "x2": rng.normal(size=200),
+        }
+    )
+    y = 3.0 * X["x1"].to_numpy() + rng.normal(scale=0.1, size=200)
+
+    def predict_fn(df):
+        return 3.0 * df["x1"].to_numpy()
+
+    imp = permutation_importance(
+        predict_fn,
+        X,
+        y,
+        metric="rmse",
+        n_repeats=3,
+        max_rows=None,
+        random_state=0,
+    )
+
+    assert imp.loc[0, "feature"] == "x1"
+    assert imp["importance_mean"].iloc[0] > imp["importance_mean"].iloc[1]
+
+
+def test_integrated_gradients_linear():
+    torch = pytest.importorskip("torch")
+    from ins_pricing.explain import integrated_gradients_torch
+
+    torch.manual_seed(0)
+    model = torch.nn.Linear(3, 1, bias=False)
+    with torch.no_grad():
+        model.weight[:] = torch.tensor([[1.0, 2.0, -1.0]])
+
+    X = torch.tensor(
+        [[1.0, 2.0, 3.0], [0.5, -1.0, 0.0]],
+        dtype=torch.float32,
+    )
+
+    def forward(x):
+        return model(x).squeeze(1)
+
+    attrs = integrated_gradients_torch(forward, X, baseline="zeros", steps=10)
+    expected = X.numpy() * np.array([1.0, 2.0, -1.0])
+
+    assert attrs.shape == X.shape
+    np.testing.assert_allclose(attrs, expected, rtol=1e-2, atol=1e-2)

ins_pricing/modelling/tests/test_geo_tokens_split.py

@@ -0,0 +1,49 @@
+import types
+
+import pandas as pd
+import pytest
+
+pytest.importorskip("torch")
+pytest.importorskip("optuna")
+pytest.importorskip("xgboost")
+pytest.importorskip("statsmodels")
+
+from ins_pricing.bayesopt.trainers import FTTrainer
+
+
+class DummyCtx:
+    def __init__(self, train_df: pd.DataFrame, test_df: pd.DataFrame):
+        self.task_type = "regression"
+        self.config = types.SimpleNamespace(use_ft_ddp=False, geo_feature_nmes=["geo"])
+        self.train_data = train_df
+        self.test_data = test_df
+        self._build_calls = []
+
+    def _build_geo_tokens(self, _params=None):
+        self._build_calls.append(
+            (self.train_data.copy(deep=True), self.test_data.copy(deep=True))
+        )
+        return self.train_data.copy(deep=True), self.test_data.copy(deep=True), ["geo_token"], None
+
+
+def test_geo_token_split_uses_fold_and_restores_context():
+    train = pd.DataFrame({"geo": ["a", "b", "c", "d"], "x": [1, 2, 3, 4]})
+    test = pd.DataFrame({"geo": ["e"], "x": [5]})
+    ctx = DummyCtx(train, test)
+    trainer = FTTrainer(ctx)
+
+    X_train = train.iloc[[0, 1]]
+    X_val = train.iloc[[2, 3]]
+
+    orig_train = ctx.train_data
+    orig_test = ctx.test_data
+
+    result = trainer._build_geo_tokens_for_split(X_train, X_val, geo_params={"k": 1})
+
+    assert ctx.train_data is orig_train
+    assert ctx.test_data is orig_test
+    assert result is not None
+
+    train_snapshot, test_snapshot = ctx._build_calls[0]
+    assert train_snapshot.equals(train.loc[X_train.index])
+    assert test_snapshot.equals(train.loc[X_val.index])

ins_pricing/modelling/tests/test_graph_cache.py

@@ -0,0 +1,33 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+pytest.importorskip("torch")
+pytest.importorskip("sklearn")
+
+from ins_pricing.bayesopt.models import GraphNeuralNetSklearn
+
+
+def test_graph_cache_meta_invalidation(tmp_path):
+    X = pd.DataFrame({"a": [0.1, 0.2, 0.3], "b": [1.0, 2.0, 3.0]})
+    cache_path = tmp_path / "gnn_cache.pt"
+
+    model = GraphNeuralNetSklearn(
+        model_nme="demo",
+        input_dim=2,
+        k_neighbors=1,
+        epochs=1,
+        use_approx_knn=False,
+        graph_cache_path=str(cache_path),
+    )
+
+    adj = model._build_graph_from_df(X)
+    assert adj is not None
+    assert cache_path.exists()
+
+    cached = model._load_cached_adj(X)
+    assert cached is not None
+
+    X_changed = X.copy()
+    X_changed.iloc[0, 0] += 1.0
+    assert model._load_cached_adj(X_changed) is None

ins_pricing/modelling/tests/test_plotting.py

@@ -0,0 +1,63 @@
+import os
+
+import numpy as np
+import pandas as pd
+import pytest
+
+pytest.importorskip("torch")
+pytest.importorskip("xgboost")
+pytest.importorskip("optuna")
+pytest.importorskip("statsmodels")
+pytest.importorskip("shap")
+
+
+def test_plotting_outputs(tmp_path, monkeypatch):
+    monkeypatch.setenv("MPLBACKEND", "Agg")
+    mpl_cfg = tmp_path / ".mplconfig"
+    cache_dir = tmp_path / ".cache"
+    (cache_dir / "fontconfig").mkdir(parents=True, exist_ok=True)
+    mpl_cfg.mkdir(parents=True, exist_ok=True)
+    monkeypatch.setenv("MPLCONFIGDIR", str(mpl_cfg))
+    monkeypatch.setenv("XDG_CACHE_HOME", str(cache_dir))
+
+    from ins_pricing.BayesOpt import BayesOptModel
+    import matplotlib.pyplot as plt
+
+    monkeypatch.setattr(plt, "show", lambda *args, **kwargs: None)
+
+    rng = np.random.default_rng(0)
+    train = pd.DataFrame(
+        {
+            "x1": rng.normal(size=30),
+            "y": rng.normal(size=30),
+            "w": rng.uniform(0.5, 1.5, size=30),
+        }
+    )
+    test = pd.DataFrame({"x1": rng.normal(size=20)})
+
+    model = BayesOptModel(
+        train,
+        test,
+        model_nme="demo",
+        resp_nme="y",
+        weight_nme="w",
+        factor_nmes=["x1"],
+        task_type="regression",
+        use_gpu=False,
+        output_dir=str(tmp_path),
+    )
+
+    for df in (model.train_data, model.test_data):
+        df["pred_xgb"] = rng.normal(size=len(df))
+        df["pred_resn"] = rng.normal(size=len(df))
+        df["w_pred_xgb"] = df["pred_xgb"] * df[model.weight_nme]
+        df["w_pred_resn"] = df["pred_resn"] * df[model.weight_nme]
+
+    model.plot_lift("Xgboost", "pred_xgb", n_bins=5)
+    model.plot_dlift(["xgb", "resn"], n_bins=5)
+
+    lift_path = tmp_path / "plot" / "01_demo_Xgboost_lift.png"
+    dlift_path = tmp_path / "plot" / "02_demo_dlift_xgb_vs_resn.png"
+
+    assert lift_path.exists()
+    assert dlift_path.exists()

ins_pricing/modelling/tests/test_plotting_library.py

@@ -0,0 +1,150 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+pytest.importorskip("matplotlib")
+pytest.importorskip("sklearn")
+
+
+def _configure_matplotlib(tmp_path, monkeypatch):
+    monkeypatch.setenv("MPLBACKEND", "Agg")
+    mpl_cfg = tmp_path / ".mplconfig"
+    cache_dir = tmp_path / ".cache"
+    (cache_dir / "fontconfig").mkdir(parents=True, exist_ok=True)
+    mpl_cfg.mkdir(parents=True, exist_ok=True)
+    monkeypatch.setenv("MPLCONFIGDIR", str(mpl_cfg))
+    monkeypatch.setenv("XDG_CACHE_HOME", str(cache_dir))
+
+
+def test_plotting_library_outputs(tmp_path, monkeypatch):
+    _configure_matplotlib(tmp_path, monkeypatch)
+
+    from ins_pricing.plotting import curves, diagnostics, geo, importance
+
+    rng = np.random.default_rng(42)
+    n = 80
+    pred1 = rng.normal(loc=0.2, scale=1.0, size=n)
+    pred2 = rng.normal(loc=0.1, scale=1.0, size=n)
+    actual = np.abs(rng.normal(loc=1.0, scale=0.5, size=n))
+    weight = rng.uniform(0.5, 2.0, size=n)
+
+    curves.plot_lift_curve(
+        pred1,
+        actual * weight,
+        weight,
+        n_bins=8,
+        save_path=str(tmp_path / "lift.png"),
+    )
+    curves.plot_double_lift_curve(
+        pred1,
+        pred2,
+        actual * weight,
+        weight,
+        n_bins=8,
+        save_path=str(tmp_path / "dlift.png"),
+    )
+
+    y_true = rng.integers(0, 2, size=n)
+    curves.plot_roc_curves(
+        y_true,
+        {"m1": pred1, "m2": pred2},
+        save_path=str(tmp_path / "roc.png"),
+    )
+
+    importance.plot_feature_importance(
+        {"x1": 0.3, "x2": 0.1, "x3": 0.05},
+        save_path=str(tmp_path / "importance.png"),
+    )
+
+    diagnostics.plot_loss_curve(
+        history={"train": [1.0, 0.7, 0.5], "val": [1.2, 0.8, 0.6]},
+        save_path=str(tmp_path / "loss.png"),
+    )
+    diagnostics.plot_oneway(
+        pd.DataFrame(
+            {
+                "x1": rng.normal(size=n),
+                "w_act": actual * weight,
+                "w": weight,
+            }
+        ),
+        feature="x1",
+        weight_col="w",
+        target_col="w_act",
+        n_bins=6,
+        save_path=str(tmp_path / "oneway.png"),
+    )
+
+    df_geo = pd.DataFrame(
+        {
+            "lon": rng.uniform(100, 120, size=n),
+            "lat": rng.uniform(20, 40, size=n),
+            "loss": actual,
+        }
+    )
+    geo.plot_geo_heatmap(
+        df_geo,
+        x_col="lon",
+        y_col="lat",
+        value_col="loss",
+        bins=10,
+        save_path=str(tmp_path / "geo_heat.png"),
+    )
+    geo.plot_geo_contour(
+        df_geo,
+        x_col="lon",
+        y_col="lat",
+        value_col="loss",
+        max_points=40,
+        levels=6,
+        save_path=str(tmp_path / "geo_contour.png"),
+    )
+
+    assert (tmp_path / "lift.png").exists()
+    assert (tmp_path / "dlift.png").exists()
+    assert (tmp_path / "roc.png").exists()
+    assert (tmp_path / "importance.png").exists()
+    assert (tmp_path / "loss.png").exists()
+    assert (tmp_path / "oneway.png").exists()
+    assert (tmp_path / "geo_heat.png").exists()
+    assert (tmp_path / "geo_contour.png").exists()
+
+
+def test_geo_plotting_on_map_optional(tmp_path, monkeypatch):
+    _configure_matplotlib(tmp_path, monkeypatch)
+    pytest.importorskip("contextily")
+
+    from ins_pricing.plotting import geo
+
+    rng = np.random.default_rng(7)
+    n = 60
+    df_geo = pd.DataFrame(
+        {
+            "lon": rng.uniform(105, 115, size=n),
+            "lat": rng.uniform(25, 35, size=n),
+            "loss": np.abs(rng.normal(loc=1.0, scale=0.4, size=n)),
+        }
+    )
+
+    geo.plot_geo_heatmap_on_map(
+        df_geo,
+        lon_col="lon",
+        lat_col="lat",
+        value_col="loss",
+        bins=12,
+        basemap=None,
+        save_path=str(tmp_path / "geo_heat_map.png"),
+    )
+    geo.plot_geo_contour_on_map(
+        df_geo,
+        lon_col="lon",
+        lat_col="lat",
+        value_col="loss",
+        max_points=30,
+        levels=5,
+        basemap=None,
+        save_path=str(tmp_path / "geo_contour_map.png"),
+    )
+
+    assert (tmp_path / "geo_heat_map.png").exists()
+    assert (tmp_path / "geo_contour_map.png").exists()

ins_pricing/modelling/tests/test_preprocessor.py

@@ -0,0 +1,48 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from ins_pricing.bayesopt.config_preprocess import BayesOptConfig, DatasetPreprocessor
+
+
+def _build_config(binary_resp: bool = False) -> BayesOptConfig:
+    return BayesOptConfig(
+        model_nme="demo",
+        resp_nme="y",
+        weight_nme="w",
+        factor_nmes=["x1"],
+        task_type="regression",
+        binary_resp_nme="y_bin" if binary_resp else None,
+    )
+
+
+def test_preprocessor_fills_missing_test_labels():
+    train = pd.DataFrame(
+        {
+            "x1": [1.0, 2.0, 3.0],
+            "y": [10.0, 20.0, 30.0],
+            "w": [1.0, 2.0, 3.0],
+            "y_bin": [0, 1, 0],
+        }
+    )
+    test = pd.DataFrame({"x1": [4.0, 5.0]})
+
+    cfg = _build_config(binary_resp=True)
+    result = DatasetPreprocessor(train, test, cfg).run()
+
+    assert "w_act" in result.train_data.columns
+    assert "w_act" not in result.test_data.columns
+    assert "w_binary_act" in result.train_data.columns
+    assert "w_binary_act" not in result.test_data.columns
+    assert result.test_data["w"].eq(1.0).all()
+    assert result.test_data["y"].isna().all()
+    assert result.test_data["y_bin"].isna().all()
+
+
+def test_preprocessor_missing_train_columns_raises():
+    train = pd.DataFrame({"x1": [1.0]})
+    test = pd.DataFrame({"x1": [2.0]})
+
+    cfg = _build_config(binary_resp=False)
+    with pytest.raises(KeyError):
+        DatasetPreprocessor(train, test, cfg).run()

ins_pricing/modelling/watchdog_run.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+import threading
+import time
+from typing import List, Optional
+
+try:
+    from .run_logging import configure_run_logging  # type: ignore
+except Exception:  # pragma: no cover
+    try:
+        from run_logging import configure_run_logging  # type: ignore
+    except Exception:  # pragma: no cover
+        configure_run_logging = None  # type: ignore
+
+
+def _split_argv(argv: List[str]) -> tuple[List[str], List[str]]:
+    if "--" not in argv:
+        raise ValueError("Missing '--' separator before the command to run.")
+    idx = argv.index("--")
+    return argv[:idx], argv[idx + 1 :]
+
+
+def _kill_process_tree(pid: int) -> None:
+    if pid <= 0:
+        return
+    if os.name == "nt":
+        subprocess.run(
+            ["taskkill", "/PID", str(pid), "/T", "/F"],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            check=False,
+        )
+        return
+    try:
+        os.killpg(pid, 15)
+        time.sleep(2)
+        os.killpg(pid, 9)
+    except Exception:
+        try:
+            os.kill(pid, 9)
+        except Exception:
+            pass
+
+
+def _reader_thread(
+    proc: subprocess.Popen, last_output_ts: dict, prefix: str = ""
+) -> None:
+    assert proc.stdout is not None
+    for line in proc.stdout:
+        last_output_ts["ts"] = time.time()
+        if prefix:
+            sys.stdout.write(prefix)
+        sys.stdout.write(line)
+        sys.stdout.flush()
+
+
+def _parse_args(before_cmd: List[str], cmd: List[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Run a command under a simple watchdog: if there is no stdout/stderr "
+            "output for N seconds, kill the whole process tree and restart. "
+            "Designed to pair with optuna_storage so BayesOpt can resume."
+        )
+    )
+    parser.add_argument(
+        "--idle-seconds",
+        type=int,
+        default=7200,
+        help="Restart if there is no output for this many seconds (default: 7200).",
+    )
+    parser.add_argument(
+        "--max-restarts",
+        type=int,
+        default=50,
+        help="Maximum restart attempts (default: 50).",
+    )
+    parser.add_argument(
+        "--restart-delay-seconds",
+        type=int,
+        default=10,
+        help="Delay between restarts (default: 10).",
+    )
+    parser.add_argument(
+        "--stop-on-nonzero-exit",
+        action="store_true",
+        help="If the command exits non-zero, stop instead of restarting.",
+    )
+    args = parser.parse_args(before_cmd)
+    if not cmd:
+        parser.error("Empty command after '--'.")
+    return args
+
+
+def run_with_watchdog(
+    cmd: List[str],
+    idle_seconds: int,
+    max_restarts: int,
+    restart_delay_seconds: int,
+    stop_on_nonzero_exit: bool,
+) -> int:
+    idle_seconds = max(1, int(idle_seconds))
+    max_restarts = max(0, int(max_restarts))
+    restart_delay_seconds = max(0, int(restart_delay_seconds))
+
+    attempt = 0
+    while True:
+        attempt += 1
+        print(
+            f"[watchdog] start attempt={attempt} idle_seconds={idle_seconds} cmd={cmd}",
+            flush=True,
+        )
+
+        creationflags = 0
+        start_new_session = False
+        if os.name == "nt":
+            creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
+        else:
+            start_new_session = True
+
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+            universal_newlines=True,
+            creationflags=creationflags,
+            start_new_session=start_new_session,
+        )
+
+        last_output_ts: dict = {"ts": time.time()}
+        reader = threading.Thread(
+            target=_reader_thread,
+            args=(proc, last_output_ts),
+            kwargs={"prefix": ""},
+            daemon=True,
+        )
+        reader.start()
+
+        killed_for_idle = False
+        exit_code: Optional[int] = None
+        while True:
+            exit_code = proc.poll()
+            if exit_code is not None:
+                break
+            idle_for = time.time() - float(last_output_ts["ts"])
+            if idle_for > idle_seconds:
+                killed_for_idle = True
+                print(
+                    f"[watchdog] idle>{idle_seconds}s (idle_for={int(idle_for)}s), killing pid={proc.pid}",
+                    flush=True,
+                )
+                _kill_process_tree(proc.pid)
+                break
+            time.sleep(5)
+
+        try:
+            proc.wait(timeout=30)
+        except Exception:
+            _kill_process_tree(proc.pid)
+
+        if exit_code is None:
+            exit_code = proc.poll() or 1
+
+        if exit_code == 0:
+            print("[watchdog] finished with exit_code=0", flush=True)
+            return 0
+
+        if stop_on_nonzero_exit and not killed_for_idle:
+            print(
+                f"[watchdog] command exited non-zero (exit_code={exit_code}); stop.",
+                flush=True,
+            )
+            return int(exit_code)
+
+        if attempt > max_restarts + 1:
+            print(
+                f"[watchdog] exceeded max_restarts={max_restarts}; last exit_code={exit_code}",
+                flush=True,
+            )
+            return int(exit_code)
+
+        print(
+            f"[watchdog] restart in {restart_delay_seconds}s (exit_code={exit_code}, killed_for_idle={killed_for_idle})",
+            flush=True,
+        )
+        if restart_delay_seconds:
+            time.sleep(restart_delay_seconds)
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    if configure_run_logging:
+        configure_run_logging(prefix="watchdog")
+    argv = list(sys.argv[1:] if argv is None else argv)
+    before_cmd, cmd = _split_argv(argv)
+    args = _parse_args(before_cmd, cmd)
+    return run_with_watchdog(
+        cmd=cmd,
+        idle_seconds=args.idle_seconds,
+        max_restarts=args.max_restarts,
+        restart_delay_seconds=args.restart_delay_seconds,
+        stop_on_nonzero_exit=bool(args.stop_on_nonzero_exit),
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ins_pricing/pricing/README.md

@@ -0,0 +1,44 @@
+# pricing
+
+Lightweight pricing loop utilities: data quality checks, exposure/targets,
+factor tables, rate tables, calibration, and monitoring (PSI).
+
+Quick start:
+
+```python
+from ins_pricing.pricing import (
+    compute_exposure,
+    build_frequency_severity,
+    build_factor_table,
+    compute_base_rate,
+    rate_premium,
+    fit_calibration_factor,
+)
+
+df["exposure"] = compute_exposure(df, "start_date", "end_date")
+df = build_frequency_severity(
+    df,
+    exposure_col="exposure",
+    claim_count_col="claim_cnt",
+    claim_amount_col="claim_amt",
+)
+
+base_rate = compute_base_rate(df, loss_col="claim_amt", exposure_col="exposure")
+vehicle_table = build_factor_table(
+    df,
+    factor_col="vehicle_type",
+    loss_col="claim_amt",
+    exposure_col="exposure",
+    base_rate=base_rate,
+)
+
+premium = rate_premium(
+    df,
+    exposure_col="exposure",
+    base_rate=base_rate,
+    factor_tables={"vehicle_type": vehicle_table},
+)
+
+factor = fit_calibration_factor(premium, df["claim_amt"].to_numpy(), target_lr=0.65)
+premium_calibrated = premium * factor
+```

ins_pricing/pricing/__init__.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from .calibration import apply_calibration, fit_calibration_factor
+from .data_quality import detect_leakage, profile_columns, validate_schema
+from .exposure import aggregate_policy_level, build_frequency_severity, compute_exposure
+from .factors import bin_numeric, build_factor_table
+from .monitoring import population_stability_index, psi_report
+from .rate_table import RateTable, apply_factor_tables, compute_base_rate, rate_premium
+
+__all__ = [
+    "apply_calibration",
+    "fit_calibration_factor",
+    "detect_leakage",
+    "profile_columns",
+    "validate_schema",
+    "aggregate_policy_level",
+    "build_frequency_severity",
+    "compute_exposure",
+    "bin_numeric",
+    "build_factor_table",
+    "population_stability_index",
+    "psi_report",
+    "RateTable",
+    "apply_factor_tables",
+    "compute_base_rate",
+    "rate_premium",
+]