datasci-toolkit 0.1.0__tar.gz

datasci_toolkit-0.1.0/.github/workflows/docs.yml

@@ -0,0 +1,48 @@
+name: Build and deploy docs
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        run: pip install uv
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Build docs
+        run: uv run mkdocs build --strict --site-dir _site
+
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: _site
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - uses: actions/deploy-pages@v4
+        id: deployment

datasci_toolkit-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,24 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v5
+
+      - name: Build
+        run: uv build
+
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@release/v1

datasci_toolkit-0.1.0/.gitignore

@@ -0,0 +1,83 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+*.egg
+*.egg-info/
+dist/
+build/
+eggs/
+parts/
+var/
+sdist/
+wheels/
+*.egg-link
+.installed.cfg
+lib/
+lib64/
+
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Jupyter
+.ipynb_checkpoints/
+*.ipynb_checkpoints
+
+# Distribution / packaging
+MANIFEST
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Testing
+.tox/
+.coverage
+.coverage.*
+.cache
+.pytest_cache/
+htmlcov/
+nosetests.xml
+coverage.xml
+*.cover
+
+# Type checking
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.pytype/
+
+# IDEs
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Environment variables
+.env
+.env.*
+
+# Data
+*.csv
+*.parquet
+*.feather
+*.pkl
+*.pickle
+*.h5
+*.hdf5
+data/
+
+# Source originals and extraction notes
+src/
+EXTRACTABLE_COMPONENTS.md
+CLAUDE.md
+examples/__marimo__
+_site/

datasci_toolkit-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

datasci_toolkit-0.1.0/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: datasci-toolkit
+Version: 0.1.0
+Summary: Add your description here
+Requires-Python: >=3.12
+Requires-Dist: anywidget>=0.9.21
+Requires-Dist: lightgbm>=4.6.0
+Requires-Dist: matplotlib>=3.10.8
+Requires-Dist: numpy>=2.4.3
+Requires-Dist: optbinning>=0.21.0
+Requires-Dist: polars>=1.39.3
+Requires-Dist: scikit-learn>=1.8.0
+Requires-Dist: scipy>=1.17.1

datasci_toolkit-0.1.0/README.md

@@ -0,0 +1,67 @@
+# datasci-toolkit
+
+My personal Python toolkit for data science — a clean rewrite of tools I use day-to-day for binary classification, scorecard development, and model validation.
+
+Polars-native, sklearn-compatible, zero external state.
+
+## Modules
+
+| Module | Classes / Functions | Description |
+|---|---|---|
+| `stability` | `PSI`, `ESI`, `StabilityMonitor` | Population and event stability indices |
+| `grouping` | `StabilityGrouping`, `WOETransformer` | Stability-constrained optimal binning and WOE encoding |
+| `metrics` | `gini`, `ks`, `lift`, `iv`, `BootstrapGini`, `feature_power`, `gini_by_period`, `lift_by_period`, `plot_metric_by_period` | Binary classification metrics with period breakdowns |
+| `model_selection` | `AUCStepwiseLogit` | Gini-based stepwise logistic regression |
+| `label_imputation` | `KNNLabelImputer`, `TargetImputer` | KNN imputation for records with missing labels |
+| `bin_editor` | `BinEditor`, `BinEditorWidget` | Headless and interactive bin boundary editor |
+| `variable_clustering` | `CorrVarClus` | Hierarchical correlation clustering for variable reduction |
+
+## Quick start
+
+```python
+import polars as pl
+from datasci_toolkit import StabilityGrouping, AUCStepwiseLogit, CorrVarClus
+
+# 1. Stability-constrained binning
+sg = StabilityGrouping(stability_threshold=0.1).fit(
+    X_train, y_train, t_train=month_train,
+    X_val=X_val, y_val=y_val, t_val=month_val,
+)
+X_woe = sg.transform(X_test)
+
+# 2. Remove correlated features
+cc = CorrVarClus(max_correlation=0.5).fit(X_woe, y_train)
+features = cc.best_features()
+
+# 3. Stepwise selection
+model = AUCStepwiseLogit(max_predictors=10, max_correlation=0.8).fit(
+    X_woe.select(features), y_train,
+    X_val=X_val_woe.select(features), y_val=y_val,
+)
+```
+
+## Install
+
+```bash
+pip install datasci-toolkit
+```
+
+## Documentation
+
+**[detrin.github.io/datasci-toolkit](https://detrin.github.io/datasci-toolkit)**
+
+| Notebook | Topic |
+|---|---|
+| [01 Stability](https://detrin.github.io/datasci-toolkit/tutorials/stability/) | PSI drift detection, StabilityMonitor, ESI |
+| [02 Grouping](https://detrin.github.io/datasci-toolkit/tutorials/grouping/) | StabilityGrouping, WOETransformer |
+| [03 Metrics](https://detrin.github.io/datasci-toolkit/tutorials/metrics/) | Gini, KS, lift, IV, bootstrap CI, period breakdowns |
+| [04 Model selection](https://detrin.github.io/datasci-toolkit/tutorials/model_selection/) | AUCStepwiseLogit, correlation filter, CV mode |
+| [05 Label imputation](https://detrin.github.io/datasci-toolkit/tutorials/label_imputation/) | KNNLabelImputer, TargetImputer |
+| [06 Bin editor](https://detrin.github.io/datasci-toolkit/tutorials/bin_editor/) | BinEditor headless API, BinEditorWidget |
+| [07 Variable clustering](https://detrin.github.io/datasci-toolkit/tutorials/variable_clustering/) | CorrVarClus dendrogram, best_features |
+
+## Stack
+
+- Python 3.12, `polars` — no pandas
+- `scikit-learn` for estimator conventions
+- `matplotlib` for standalone plot functions

datasci_toolkit-0.1.0/datasci_toolkit/__init__.py

@@ -0,0 +1,33 @@
+from datasci_toolkit.bin_editor import BinEditor
+from datasci_toolkit.bin_editor_widget import BinEditorWidget
+from datasci_toolkit.grouping import StabilityGrouping, WOETransformer
+from datasci_toolkit.metrics import BootstrapGini, feature_power, gini, gini_by_period, iv, ks, lift, lift_by_period, plot_metric_by_period
+from datasci_toolkit.model_selection import AUCStepwiseLogit
+from datasci_toolkit.variable_clustering import CorrVarClus
+from datasci_toolkit.label_imputation import KNNLabelImputer, TargetImputer
+from datasci_toolkit.stability import ESI, PSI, StabilityMonitor, plot_psi_comparison, psi_hist
+
+__all__ = [
+    "PSI",
+    "ESI",
+    "StabilityMonitor",
+    "plot_psi_comparison",
+    "psi_hist",
+    "WOETransformer",
+    "StabilityGrouping",
+    "AUCStepwiseLogit",
+    "gini",
+    "ks",
+    "lift",
+    "iv",
+    "BootstrapGini",
+    "feature_power",
+    "TargetImputer",
+    "KNNLabelImputer",
+    "BinEditor",
+    "BinEditorWidget",
+    "CorrVarClus",
+    "gini_by_period",
+    "lift_by_period",
+    "plot_metric_by_period",
+]

datasci_toolkit-0.1.0/datasci_toolkit/bin_editor.py

@@ -0,0 +1,373 @@
+from __future__ import annotations
+
+import copy
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+import numpy as np
+import polars as pl
+
+from datasci_toolkit.grouping import _rsi
+
+_SMOOTH = 0.5
+
+
+class FeatureDtype(str, Enum):
+    NUMERIC = "float"
+    CATEGORICAL = "category"
+
+
+@dataclass(frozen=True)
+class BinStats:
+    counts: np.ndarray
+    event_rates: np.ndarray
+    woe: np.ndarray
+    iv: float
+
+
+@dataclass(frozen=True)
+class TemporalStats:
+    months: list[Any]
+    rsi: float
+    event_rates: list[list[float | None]]
+    pop_shares: list[list[float]]
+
+
+@dataclass
+class FeatureState:
+    feature: str
+    dtype: FeatureDtype
+    n_bins: int
+    bins: list[str] | dict[str, int]
+    counts: list[float]
+    event_rates: list[float | None]
+    woe: list[float]
+    iv: float
+    splits: list[float] | None = None
+    groups: dict[int, list[str]] | None = None
+    temporal: TemporalStats | None = None
+
+
+def _bin_stats(target: np.ndarray, weights: np.ndarray, assignments: np.ndarray, n_bins: int) -> BinStats:
+    total_ev = float((target * weights).sum())
+    total_nev = float(((1.0 - target) * weights).sum())
+    yw = target * weights
+    counts = np.bincount(assignments, weights=weights, minlength=n_bins + 1).astype(float)
+    events = np.bincount(assignments, weights=yw, minlength=n_bins + 1).astype(float)
+    nonevents = counts - events
+    event_rates = np.where(counts > 0, events / counts, np.nan)
+
+    events_per_bin, nonevents_per_bin = events[:n_bins], nonevents[:n_bins]
+    event_dist = (events_per_bin + _SMOOTH) / (total_ev + _SMOOTH * n_bins)
+    nonevent_dist = (nonevents_per_bin + _SMOOTH) / (total_nev + _SMOOTH * n_bins)
+    woe_per_bin = np.log(event_dist / nonevent_dist)
+    iv = float(((event_dist - nonevent_dist) * woe_per_bin).sum())
+
+    nan_event_dist = (events[n_bins] + _SMOOTH) / (total_ev + _SMOOTH)
+    nan_nonevent_dist = (nonevents[n_bins] + _SMOOTH) / (total_nev + _SMOOTH)
+    woe_nan = float(np.log(nan_event_dist / nan_nonevent_dist))
+
+    return BinStats(counts=counts, event_rates=event_rates, woe=np.append(woe_per_bin, woe_nan), iv=iv)
+
+
+def _temporal_stats(
+    target: np.ndarray,
+    weights: np.ndarray,
+    assignments: np.ndarray,
+    n_bins: int,
+    time_periods: np.ndarray,
+    threshold: float,
+) -> TemporalStats:
+    months = np.sort(np.unique(time_periods))
+    event_rate_by_bin: list[list[float | None]] = [[] for _ in range(n_bins)]
+    pop_share_by_bin: list[list[float]] = [[] for _ in range(n_bins)]
+
+    for month in months:
+        mask = time_periods == month
+        stats = _bin_stats(target[mask], weights[mask], assignments[mask], n_bins)
+        total = float(stats.counts[:n_bins].sum()) or 1.0
+        for bin_index in range(n_bins):
+            event_rate = stats.event_rates[bin_index]
+            event_rate_by_bin[bin_index].append(None if np.isnan(event_rate) else round(float(event_rate), 6))
+            pop_share_by_bin[bin_index].append(round(float(stats.counts[bin_index] / total), 6))
+
+    scores_array: list[float] = []
+    rates_array: list[float] = []
+    months_array: list[Any] = []
+    for month_index, month in enumerate(months):
+        for bin_index in range(n_bins):
+            event_rate = event_rate_by_bin[bin_index][month_index]
+            if event_rate is not None:
+                scores_array.append(float(bin_index))
+                rates_array.append(event_rate)
+                months_array.append(month)
+
+    rsi = _rsi(np.array(scores_array), np.array(rates_array), np.array(months_array), threshold) if len(scores_array) > 1 else 1.0
+
+    return TemporalStats(
+        months=months.tolist(),
+        rsi=round(rsi, 4),
+        event_rates=event_rate_by_bin,
+        pop_shares=pop_share_by_bin,
+    )
+
+
+def _num_assign(values: np.ndarray, splits: list[float]) -> np.ndarray:
+    missing_mask = np.isnan(values)
+    assignments = np.digitize(values, splits)
+    assignments[missing_mask] = len(splits) + 1
+    return assignments
+
+
+def _cat_assign(values: np.ndarray, category_bins: dict[str, int]) -> np.ndarray:
+    n_groups = max(category_bins.values()) + 1 if category_bins else 0
+    assignments = np.full(len(values), n_groups, dtype=np.intp)
+    for category, group in category_bins.items():
+        assignments[values == category] = group
+    return assignments
+
+
+def _num_labels(splits: list[float]) -> list[str]:
+    if not splits:
+        return ["-inf to inf", "NaN"]
+    split_strs = [f"{v:.4g}" for v in splits]
+    return [f"-inf to {split_strs[0]}"] + [f"{split_strs[i]} to {split_strs[i+1]}" for i in range(len(split_strs) - 1)] + [f"{split_strs[-1]} to inf", "NaN"]
+
+
+def _num_state(feat: str, splits: list[float], values: np.ndarray, target: np.ndarray, weights: np.ndarray) -> FeatureState:
+    n_bins = len(splits) + 1
+    stats = _bin_stats(target, weights, _num_assign(values, splits), n_bins)
+    return FeatureState(
+        feature=feat,
+        dtype=FeatureDtype.NUMERIC,
+        n_bins=n_bins,
+        splits=list(splits),
+        bins=_num_labels(splits),
+        counts=stats.counts.tolist(),
+        event_rates=[None if np.isnan(v) else round(float(v), 6) for v in stats.event_rates],
+        woe=[round(float(v), 6) for v in stats.woe],
+        iv=round(stats.iv, 6),
+    )
+
+
+def _cat_state(feat: str, category_bins: dict[str, int], values: np.ndarray, target: np.ndarray, weights: np.ndarray) -> FeatureState:
+    n_groups = max(category_bins.values()) + 1 if category_bins else 0
+    stats = _bin_stats(target, weights, _cat_assign(values, category_bins), n_groups)
+    groups: dict[int, list[str]] = {}
+    for cat, grp in category_bins.items():
+        groups.setdefault(grp, []).append(str(cat))
+    return FeatureState(
+        feature=feat,
+        dtype=FeatureDtype.CATEGORICAL,
+        n_bins=n_groups,
+        groups={k: sorted(v) for k, v in groups.items()},
+        bins=dict(category_bins),
+        counts=stats.counts.tolist(),
+        event_rates=[None if np.isnan(v) else round(float(v), 6) for v in stats.event_rates],
+        woe=[round(float(v), 6) for v in stats.woe],
+        iv=round(stats.iv, 6),
+    )
+
+
+class BinEditor:
+    """Headless state machine for editing bin boundaries.
+
+    Works identically in plain Python scripts, notebooks, and agents. All
+    edits are logged per feature with undo support. Call `accept()` to export
+    the final bin specs dict for use with `WOETransformer`.
+
+    Args:
+        bin_specs: Initial bin specifications — a dict produced by
+            `StabilityGrouping.bin_specs_` or built manually.
+        features: Feature DataFrame matching the features in ``bin_specs``.
+        target: Binary target series (0/1 or float).
+        time_periods: Optional time series for temporal stability metrics.
+        weights: Optional sample weight series.
+        stability_threshold: RSI threshold used to flag unstable bins in the
+            state dict (does not block edits).
+
+    Note:
+        All state is accessible via `state(feat)`, which returns a `FeatureState`
+        dataclass with attributes ``bins``, ``n_bins``, ``counts``, ``event_rates``,
+        ``woe``, ``iv``, ``dtype``, ``groups``, and ``temporal``.
+    """
+
+    def __init__(
+        self,
+        bin_specs: dict[str, dict[str, Any]],
+        features: pl.DataFrame,
+        target: pl.Series,
+        time_periods: pl.Series | None = None,
+        weights: pl.Series | None = None,
+        stability_threshold: float = 0.1,
+    ) -> None:
+        self._targets = target.cast(pl.Float64).to_numpy()
+        self._weights = weights.cast(pl.Float64).to_numpy() if weights is not None else np.ones(len(self._targets))
+        self._time: np.ndarray | None = time_periods.to_numpy() if time_periods is not None else None
+        self._threshold = stability_threshold
+        self._x: dict[str, np.ndarray] = {}
+        self._splits: dict[str, list[float]] = {}
+        self._cat_bins: dict[str, dict[str, int]] = {}
+        self._history: dict[str, list[tuple[str, Any]]] = {}
+        self._orig: dict[str, dict[str, Any]] = {}
+
+        for feat, spec in bin_specs.items():
+            if feat not in features.columns:
+                continue
+            self._orig[feat] = spec
+            self._history[feat] = []
+            if spec["dtype"] == FeatureDtype.NUMERIC:
+                self._x[feat] = features[feat].cast(pl.Float64).to_numpy()
+                self._splits[feat] = [float(s) for s in spec["bins"][1:-1] if np.isfinite(s)]
+            else:
+                self._x[feat] = features[feat].cast(pl.Utf8).to_numpy().astype(str)
+                self._cat_bins[feat] = {str(k): int(v) for k, v in spec["bins"].items()}
+
+    def features(self) -> list[str]:
+        return list(self._splits.keys()) + list(self._cat_bins.keys())
+
+    def _base_state(self, feat: str) -> FeatureState:
+        if feat in self._splits:
+            return _num_state(feat, self._splits[feat], self._x[feat], self._targets, self._weights)
+        return _cat_state(feat, self._cat_bins[feat], self._x[feat], self._targets, self._weights)
+
+    def _assignments(self, feat: str) -> np.ndarray:
+        if feat in self._splits:
+            return _num_assign(self._x[feat], self._splits[feat])
+        return _cat_assign(self._x[feat], self._cat_bins[feat])
+
+    def state(self, feat: str) -> FeatureState:
+        s = self._base_state(feat)
+        if self._time is not None:
+            s.temporal = _temporal_stats(
+                self._targets, self._weights, self._assignments(feat), s.n_bins, self._time, self._threshold
+            )
+        return s
+
+    def _push(self, feat: str) -> None:
+        if feat in self._splits:
+            self._history[feat].append(("splits", list(self._splits[feat])))
+        else:
+            self._history[feat].append(("cat", copy.deepcopy(self._cat_bins[feat])))
+
+    def split(self, feat: str, value: float) -> FeatureState:
+        if value in self._splits[feat]:
+            return self.state(feat)
+        self._push(feat)
+        self._splits[feat] = sorted(self._splits[feat] + [value])
+        return self.state(feat)
+
+    def merge(self, feat: str, bin_idx: int) -> FeatureState:
+        if feat in self._splits:
+            splits = self._splits[feat]
+            if bin_idx >= len(splits):
+                return self.state(feat)
+            self._push(feat)
+            self._splits[feat] = [s for i, s in enumerate(splits) if i != bin_idx]
+        else:
+            cat_bins = self._cat_bins[feat]
+            n_groups = max(cat_bins.values()) + 1 if cat_bins else 0
+            if bin_idx >= n_groups - 1:
+                return self.state(feat)
+            self._push(feat)
+            self._cat_bins[feat] = {
+                cat: (bin_idx if grp == bin_idx + 1 else (grp - 1 if grp > bin_idx + 1 else grp))
+                for cat, grp in cat_bins.items()
+            }
+        return self.state(feat)
+
+    def move_boundary(self, feat: str, bin_idx: int, new_value: float) -> FeatureState:
+        splits = self._splits[feat]
+        if bin_idx >= len(splits):
+            return self.state(feat)
+        self._push(feat)
+        new = list(splits)
+        new[bin_idx] = new_value
+        self._splits[feat] = sorted(set(new))
+        return self.state(feat)
+
+    def reset(self, feat: str) -> FeatureState:
+        self._history[feat] = []
+        spec = self._orig[feat]
+        if spec["dtype"] == FeatureDtype.NUMERIC:
+            self._splits[feat] = [float(s) for s in spec["bins"][1:-1] if np.isfinite(s)]
+        else:
+            self._cat_bins[feat] = {str(k): int(v) for k, v in spec["bins"].items()}
+        return self.state(feat)
+
+    def undo(self, feat: str) -> FeatureState:
+        if not self._history[feat]:
+            return self.state(feat)
+        kind, prev = self._history[feat].pop()
+        if kind == "splits":
+            self._splits[feat] = prev
+        else:
+            self._cat_bins[feat] = prev
+        return self.state(feat)
+
+    def history(self, feat: str) -> list[dict[str, Any]]:
+        return [{"type": k, "value": v} for k, v in self._history[feat]]
+
+    def _suggest_num(self, feat: str, n_suggestions: int) -> list[float]:
+        values = self._x[feat]
+        x_valid = values[~np.isnan(values)]
+        if len(x_valid) == 0:
+            return []
+        current = self._splits[feat]
+        span = float(x_valid.max() - x_valid.min())
+        min_gap = span * 0.01
+        candidates = [
+            float(candidate) for candidate in np.unique(np.percentile(x_valid, np.linspace(5, 95, 40)))
+            if all(abs(candidate - split) > min_gap for split in current)
+        ]
+        base_information_value = self._base_state(feat).iv
+        pairs: list[tuple[float, float]] = sorted(
+            [
+                (
+                    _bin_stats(self._targets, self._weights, _num_assign(values, sorted(current + [candidate])), len(current) + 2).iv - base_information_value,
+                    float(candidate),
+                )
+                for candidate in candidates
+            ],
+            reverse=True,
+        )
+        return [v for _, v in pairs[:n_suggestions]]
+
+    def _suggest_cat(self, feat: str, n_suggestions: int) -> list[tuple[int, int]]:
+        category_bins = self._cat_bins[feat]
+        n_groups = max(category_bins.values()) + 1 if category_bins else 0
+        if n_groups <= 1:
+            return []
+        values = self._x[feat]
+        base_information_value = self._base_state(feat).iv
+        pairs: list[tuple[float, tuple[int, int]]] = sorted(
+            [
+                (
+                    base_information_value - _bin_stats(
+                        self._targets, self._weights,
+                        _cat_assign(values, {
+                            category: (bin_idx if group == bin_idx + 1 else (group - 1 if group > bin_idx + 1 else group))
+                            for category, group in category_bins.items()
+                        }),
+                        n_groups - 1,
+                    ).iv,
+                    (bin_idx, bin_idx + 1),
+                )
+                for bin_idx in range(n_groups - 1)
+            ]
+        )
+        return [pair for _, pair in pairs[:n_suggestions]]
+
+    def suggest_splits(self, feat: str, n: int = 5) -> list:  # type: ignore[type-arg]
+        if feat in self._splits:
+            return self._suggest_num(feat, n)
+        return self._suggest_cat(feat, n)
+
+    def accept(self) -> dict[str, dict[str, Any]]:
+        return {feat: self.accept_feature(feat) for feat in self.features()}
+
+    def accept_feature(self, feat: str) -> dict[str, Any]:
+        if feat in self._splits:
+            return {"dtype": "float", "bins": [-np.inf] + self._splits[feat] + [np.inf]}
+        return {"dtype": "category", "bins": dict(self._cat_bins[feat])}