glm-factor-optimizer 0.1.0__py3-none-any.whl

glm_factor_optimizer/__init__.py

@@ -0,0 +1,83 @@
+"""Simple GLM factor-binning and modeling tools."""
+
+from .aggregation import aggregate_rate_table, aggregate_table
+from .bins import apply_spec, category_target_order, make_numeric_bins
+from .core import GLM, RateGLM
+from .diagnostics import find_interactions, pair_diagnostics
+from .factor import FactorBlock
+from .metrics import (
+    calibration,
+    gaussian_deviance,
+    gamma_deviance,
+    lift_table,
+    model_deviance,
+    poisson_deviance,
+    summary,
+    weighted_mae,
+    weighted_rmse,
+)
+from .model import FittedGLM, FittedRateGLM, fit_glm, fit_rate_glm
+from .optimize import (
+    OptimizationResult,
+    optimize_factor,
+    small_bin_size_penalty,
+    small_count_penalty,
+    train_validation_gap_penalty,
+)
+from .penalties import bin_count_penalty, small_target_penalty, unstable_relativity_penalty
+from .runs import RunLogger
+from .sampling import missing_strata, stratified_sample
+from .screening import rank_factors
+from .split import split
+from .study import GLMStudy
+from .validation import by_factor_report, train_validation_comparison, validation_report
+from .workflow import GLMWorkflow, WorkflowResult, run_workflow
+
+optimize_bins = optimize_factor
+
+__all__ = [
+    "FittedRateGLM",
+    "FittedGLM",
+    "FactorBlock",
+    "GLM",
+    "GLMStudy",
+    "GLMWorkflow",
+    "OptimizationResult",
+    "RateGLM",
+    "RunLogger",
+    "aggregate_rate_table",
+    "aggregate_table",
+    "apply_spec",
+    "bin_count_penalty",
+    "calibration",
+    "category_target_order",
+    "by_factor_report",
+    "fit_rate_glm",
+    "fit_glm",
+    "find_interactions",
+    "gaussian_deviance",
+    "gamma_deviance",
+    "lift_table",
+    "make_numeric_bins",
+    "missing_strata",
+    "model_deviance",
+    "optimize_bins",
+    "optimize_factor",
+    "pair_diagnostics",
+    "poisson_deviance",
+    "rank_factors",
+    "split",
+    "small_bin_size_penalty",
+    "small_count_penalty",
+    "small_target_penalty",
+    "summary",
+    "train_validation_gap_penalty",
+    "train_validation_comparison",
+    "unstable_relativity_penalty",
+    "stratified_sample",
+    "validation_report",
+    "weighted_mae",
+    "weighted_rmse",
+    "WorkflowResult",
+    "run_workflow",
+]

glm_factor_optimizer/aggregation.py

@@ -0,0 +1,136 @@
+"""Generic aggregation helpers for model development tables."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+import pandas as pd
+
+AggregationSpec = Mapping[str, tuple[str, str]]
+
+
+def aggregate_table(
+    df: pd.DataFrame,
+    group_by: str | Sequence[str] | None = None,
+    aggregations: AggregationSpec | None = None,
+) -> pd.DataFrame:
+    """Return grouped rows plus optional named aggregations.
+
+    Parameters
+    ----------
+    df:
+        Data to aggregate.
+    group_by:
+        Optional column or columns to group by. When omitted, a single overall
+        row is returned.
+    aggregations:
+        Optional mapping from output column name to ``(source_column,
+        function)``. Supported functions include ``sum``, ``mean``, ``min``,
+        ``max``, ``count``, ``size``, and ``std``.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Grouped table with a ``rows`` column and requested aggregations.
+    """
+
+    group_cols = _as_list(group_by)
+    named = {name: tuple(spec) for name, spec in (aggregations or {}).items()}
+    if group_cols:
+        return (
+            df.groupby(group_cols, dropna=False)
+            .agg(rows=(group_cols[0], "size"), **named)
+            .reset_index()
+        )
+    row: dict[str, Any] = {"rows": int(len(df))}
+    for name, (column, function) in named.items():
+        row[name] = _aggregate_series(df[column], function)
+    return pd.DataFrame([row])
+
+
+def aggregate_rate_table(
+    df: pd.DataFrame,
+    group_by: str | Sequence[str] | None,
+    *,
+    target: str,
+    exposure: str | None = None,
+    weight: str | None = None,
+    prediction: str | None = None,
+    extras: AggregationSpec | None = None,
+) -> pd.DataFrame:
+    """Return grouped observed/predicted totals and level diagnostics.
+
+    Parameters
+    ----------
+    df:
+        Data to aggregate.
+    group_by:
+        Optional column or columns to group by.
+    target:
+        Observed outcome column.
+    exposure:
+        Optional exposure column for exposure-adjusted level columns.
+    weight:
+        Optional weight column to total.
+    prediction:
+        Optional predicted outcome column.
+    extras:
+        Optional extra named aggregations, using the same format as
+        :func:`aggregate_table`.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Grouped observed and predicted totals with mean or exposure-adjusted
+        diagnostics.
+    """
+
+    aggregations: dict[str, tuple[str, str]] = {"actual": (target, "sum")}
+    if exposure is not None:
+        aggregations["exposure"] = (exposure, "sum")
+    if weight is not None:
+        aggregations["weight"] = (weight, "sum")
+    if prediction is not None:
+        aggregations["predicted"] = (prediction, "sum")
+    aggregations.update(extras or {})
+
+    table = aggregate_table(df, group_by, aggregations)
+    if exposure is not None:
+        table["actual_rate"] = table["actual"] / table["exposure"].clip(lower=1e-9)
+        if prediction is not None:
+            table["predicted_rate"] = table["predicted"] / table["exposure"].clip(lower=1e-9)
+    else:
+        table["actual_mean"] = table["actual"] / table["rows"].clip(lower=1)
+        if prediction is not None:
+            table["predicted_mean"] = table["predicted"] / table["rows"].clip(lower=1)
+    if prediction is not None:
+        table["actual_to_predicted"] = table["actual"] / table["predicted"].clip(lower=1e-9)
+    return table
+
+
+def _aggregate_series(series: pd.Series, function: str) -> Any:
+    normalized = function.lower().strip()
+    if normalized == "sum":
+        return series.sum()
+    if normalized in {"mean", "avg"}:
+        return series.mean()
+    if normalized == "min":
+        return series.min()
+    if normalized == "max":
+        return series.max()
+    if normalized == "count":
+        return int(series.count())
+    if normalized == "size":
+        return int(len(series))
+    if normalized == "std":
+        return series.std()
+    raise ValueError(f"Unsupported aggregation function {function!r}.")
+
+
+def _as_list(value: str | Sequence[str] | None) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [value]
+    return list(value)

glm_factor_optimizer/bins.py

@@ -0,0 +1,257 @@
+"""Binning and grouping specs for model factors."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+JsonDict = dict[str, Any]
+
+_MISSING = "__missing__"
+
+
+def _final_edges(raw_edges: list[float], series: pd.Series) -> list[float]:
+    clean = pd.to_numeric(series, errors="coerce").dropna()
+    if clean.empty:
+        raise ValueError("Cannot create bins from an empty numeric column.")
+    edges = sorted({float(value) for value in raw_edges})
+    lower = float(clean.min()) - 1e-9
+    upper = float(clean.max()) + 1e-9
+    if len(edges) < 2:
+        midpoint = float(clean.median())
+        edges = [lower, midpoint, upper]
+    else:
+        edges[0] = min(edges[0], lower)
+        edges[-1] = max(edges[-1], upper)
+    final = [edges[0]]
+    for edge in edges[1:]:
+        if edge > final[-1]:
+            final.append(edge)
+    return final if len(final) >= 2 else [lower, upper]
+
+
+def make_numeric_bins(
+    series: pd.Series,
+    bins: int = 10,
+    column: str | None = None,
+    method: str = "quantile",
+) -> JsonDict:
+    """Create a JSON-serializable numeric binning spec.
+
+    Parameters
+    ----------
+    series:
+        Numeric values used to derive bin edges.
+    bins:
+        Number of bins to request.
+    column:
+        Source column name. When omitted, ``series.name`` is used if present.
+    method:
+        Binning method, currently ``"quantile"`` or ``"uniform"``.
+
+    Returns
+    -------
+    dict
+        Numeric binning spec with edges, labels, source column, and output
+        column.
+    """
+
+    if bins < 1:
+        raise ValueError("bins must be at least 1.")
+    clean = pd.to_numeric(series, errors="coerce").dropna()
+    if method == "quantile":
+        raw_edges = clean.quantile(np.linspace(0.0, 1.0, bins + 1)).tolist()
+    elif method == "uniform":
+        raw_edges = np.linspace(float(clean.min()), float(clean.max()), bins + 1).tolist()
+    else:
+        raise ValueError("method must be 'quantile' or 'uniform'.")
+
+    edges = _final_edges(raw_edges, clean)
+    labels = [f"bin_{index + 1}" for index in range(len(edges) - 1)]
+    source_column = column or series.name or "value"
+    return {
+        "type": "numeric",
+        "column": source_column,
+        "output": f"{source_column}_bin",
+        "method": method,
+        "edges": edges,
+        "labels": labels,
+    }
+
+
+def _category_key(value: object) -> str:
+    if pd.isna(value):
+        return _MISSING
+    if isinstance(value, np.generic):
+        value = value.item()
+    return str(value)
+
+
+def category_target_order(
+    df: pd.DataFrame,
+    factor: str,
+    target: str,
+    exposure: str | None = None,
+    weight: str | None = None,
+) -> pd.DataFrame:
+    """Order categories by observed training target level.
+
+    Parameters
+    ----------
+    df:
+        Training data.
+    factor:
+        Categorical factor column to order.
+    target:
+        Observed outcome column.
+    exposure:
+        Optional exposure column used as the denominator for the level.
+    weight:
+        Optional row-weight column used as the denominator for weighted levels.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Category table sorted by observed level, including measure, actual,
+        level, and credibility columns.
+    """
+
+    columns = [factor, target]
+    if exposure is not None:
+        columns.append(exposure)
+    if weight is not None and weight not in columns:
+        columns.append(weight)
+    work = df[columns].copy()
+    work[factor] = work[factor].map(_category_key)
+    if exposure is not None:
+        table = (
+            work.groupby(factor, dropna=False)
+            .agg(measure=(exposure, "sum"), actual=(target, "sum"))
+            .reset_index()
+        )
+        table["level"] = table["actual"] / table["measure"].clip(lower=1e-9)
+    elif weight is not None:
+        work["_weighted_target"] = work[target] * work[weight]
+        table = (
+            work.groupby(factor, dropna=False)
+            .agg(measure=(weight, "sum"), actual=("_weighted_target", "sum"))
+            .reset_index()
+        )
+        table["level"] = table["actual"] / table["measure"].clip(lower=1e-9)
+    else:
+        table = (
+            work.groupby(factor, dropna=False)
+            .agg(measure=(target, "size"), actual=(target, "sum"))
+            .reset_index()
+        )
+        table["level"] = table["actual"] / table["measure"].clip(lower=1e-9)
+    table["credibility"] = np.sqrt(table["actual"].clip(lower=0.0) + 1.0)
+    return table.sort_values(["level", "measure"]).reset_index(drop=True)
+
+
+def make_categorical_groups(
+    df: pd.DataFrame,
+    factor: str,
+    target: str,
+    exposure: str | None = None,
+    weight: str | None = None,
+    cutpoints: list[int] | None = None,
+) -> JsonDict:
+    """Create a target-ordered categorical grouping spec from training data.
+
+    Parameters
+    ----------
+    df:
+        Training data.
+    factor:
+        Categorical factor column to group.
+    target:
+        Observed outcome column used for ordering categories.
+    exposure:
+        Optional exposure column used as the denominator for category levels.
+    weight:
+        Optional row-weight column used for weighted category levels.
+    cutpoints:
+        Ordered category positions where new groups should start.
+
+    Returns
+    -------
+    dict
+        JSON-serializable categorical grouping spec with order, cutpoints,
+        mapping, labels, and training statistics.
+    """
+
+    cutpoints = cutpoints or []
+    target_order = category_target_order(df, factor, target, exposure=exposure, weight=weight)
+    ordered = [str(value) for value in target_order[factor].tolist()]
+    clean_cutpoints = sorted({int(point) for point in cutpoints if 0 < int(point) < len(ordered)})
+    boundaries = [0, *clean_cutpoints, len(ordered)]
+    mapping: dict[str, str] = {}
+    labels: list[str] = []
+
+    for index in range(len(boundaries) - 1):
+        label = f"group_{index + 1:02d}"
+        labels.append(label)
+        for category in ordered[boundaries[index] : boundaries[index + 1]]:
+            mapping[category] = label
+
+    return {
+        "type": "categorical",
+        "column": factor,
+        "output": f"{factor}_group",
+        "order": ordered,
+        "cutpoints": clean_cutpoints,
+        "mapping": mapping,
+        "labels": labels,
+        "default": "other",
+        "missing": _MISSING,
+        "stats": target_order.rename(columns={factor: "category"}).to_dict("records"),
+    }
+
+
+def apply_spec(
+    df: pd.DataFrame,
+    spec: JsonDict,
+    output: str | None = None,
+) -> pd.DataFrame:
+    """Apply a saved numeric or categorical spec to a dataframe.
+
+    Parameters
+    ----------
+    df:
+        Data containing the raw column referenced by ``spec``.
+    spec:
+        JSON-serializable numeric or categorical spec.
+    output:
+        Optional output column override. When omitted, the spec output is used.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Copy of ``df`` with the transformed output column added.
+    """
+
+    kind = str(spec["type"])
+    column = str(spec["column"])
+    output = output or str(spec.get("output") or f"{column}_{kind}")
+    result = df.copy()
+
+    if kind == "numeric":
+        binned = pd.cut(
+            pd.to_numeric(result[column], errors="coerce"),
+            bins=spec["edges"],
+            labels=spec["labels"],
+            include_lowest=True,
+        )
+        result[output] = binned.astype(object)
+        result[output] = result[output].where(~result[output].isna(), other="missing")
+        return result
+
+    if kind == "categorical":
+        keys = result[column].map(_category_key)
+        result[output] = keys.map(spec["mapping"]).fillna(str(spec.get("default", "other")))
+        return result
+
+    raise ValueError("spec type must be 'numeric' or 'categorical'.")