bossanova 0.1.0.dev24__tar.gz → 0.1.0.dev26__tar.gz

{bossanova-0.1.0.dev24 → bossanova-0.1.0.dev26}/.gitignore

@@ -8,6 +8,7 @@ sdist/
 dev
 *.DS*
 research/
+scratch/
 plans/
 archive/
 .private-journal
@@ -31,6 +32,7 @@ lmer-refactor-notes/
 *.txt
 *.json
 !tests/pyodide/package.json
+!context7.json
 *.png
 
 # Node.js (for Pyodide tests)

{bossanova-0.1.0.dev24 → bossanova-0.1.0.dev26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bossanova
-Version: 0.1.0.dev24
+Version: 0.1.0.dev26
 Summary: Bridging statistical cultures with some jazz
 Author: Eshin Jolly
 License-Expression: MIT

{bossanova-0.1.0.dev24 → bossanova-0.1.0.dev26}/bossanova/internal/compare/compare.py

@@ -60,6 +60,7 @@ def compare(
     seed: int | None = None,
     metric: str = "mse",
     digits: int | None = None,
+    holdout_group: str | None = None,
 ) -> pl.DataFrame:
     """Compare nested statistical models.
 
@@ -71,6 +72,20 @@ def compare(
     - lmer/glmer: Likelihood ratio test
     - cv: Cross-validation with Nadeau-Bengio corrected t-test
 
+    Cross-type comparisons (e.g., glm vs glmer) are supported when models
+    share the same family. The fixed-only model is treated as nested within
+    the mixed model via zero variance components.
+
+    | Model Pair | Hypothesis Test | AIC/BIC | CV |
+    |------------|-----------------|---------|-----|
+    | lm vs lm | F-test | Yes | Yes |
+    | glm vs glm | Deviance | Yes | Yes |
+    | lmer vs lmer | LRT | Yes | Yes |
+    | glmer vs glmer | LRT | Yes | Yes |
+    | lm vs lmer | LRT | Yes | Yes |
+    | glm vs glmer | LRT | Yes | Yes |
+    | Different families | — | Yes | Yes |
+
     Args:
         *models: Two or more fitted model objects.
         method: Comparison method. Options:
@@ -78,7 +93,7 @@ def compare(
             - "f": F-test (for lm)
             - "lrt": Likelihood ratio test (for mixed models)
             - "deviance": Deviance test (for glm)
-            - "cv": Cross-validation comparison (for lm/glm)
+            - "cv": Cross-validation comparison (any model type)
             - "aic": AIC comparison with delta-AIC and Akaike weights
             - "bic": BIC comparison with delta-BIC and Schwarz weights
         sort: If True, sort models by complexity before comparing.
@@ -234,7 +249,8 @@ def compare(
 
     # Infer method if auto
     if method == "auto":
-        method = resolve_method(model_list[0])
+        type_set = {m._model_type for m in model_list}
+        method = resolve_method(model_list[0], model_types=type_set)
 
     # For LRT with refit=True, refit REML models with ML
     if method == "lrt" and refit:
@@ -248,7 +264,13 @@ def compare(
     elif method == "deviance":
         result = compare_deviance(model_list, test=test)
     elif method == "cv":
-        result = compare_cv(model_list, cv=cv, seed=seed, metric=metric)
+        result = compare_cv(
+            model_list,
+            cv=cv,
+            seed=seed,
+            metric=metric,
+            holdout_group=holdout_group,
+        )
     elif method == "aic":
         result = compare_aic(model_list)
     elif method == "bic":

{bossanova-0.1.0.dev24 → bossanova-0.1.0.dev26}/bossanova/internal/compare/cv.py

@@ -3,6 +3,9 @@
 Compares models using k-fold (or LOO) cross-validation with the corrected
 variance estimator from Nadeau & Bengio (2003) to account for overlapping
 training sets.
+
+Supports group-aware splitting via ``holdout_group`` to prevent data leakage
+from random-effects structure.
 """
 
 from typing import Any
@@ -12,11 +15,10 @@ import polars as pl
 from scipy import stats
 
 from bossanova.internal.containers.schemas import Col, ComparisonCv
-from bossanova.internal.maths.rng import RNG
-from bossanova.internal.compare.helpers import get_model_type
-from bossanova.internal.infer.resample.core import (
-    generate_kfold_indices,
-    generate_loo_indices,
+from bossanova.internal.infer.cv import (
+    generate_group_kfold_splits,
+    generate_kfold_splits,
+    resolve_holdout_group_ids,
 )
 
 __all__ = [
@@ -44,8 +46,6 @@ def compute_cv_fold_score(
     Returns:
         Test set error for this fold.
     """
-    model_type = get_model_type(model)
-
     # Get original data
     data = model.data
     formula = model.formula
@@ -57,12 +57,9 @@ def compute_cv_fold_score(
     # Refit on training data using unified model()
     from bossanova.model import model as model_cls
 
-    if model_type in ("lm", "glm"):
-        new_model = model_cls(
-            formula, train_data, family=model.family, link=model.link
-        ).fit()
-    else:
-        raise ValueError(f"CV comparison not supported for model type: {model_type}")
+    new_model = model_cls(
+        formula, train_data, family=model.family, link=model.link
+    ).fit()
 
     # Predict on test data
     new_model.predict(newdata=test_data)
@@ -84,58 +81,112 @@ def compute_cv_fold_score(
         raise ValueError(f"Unknown metric: {metric}. Use 'mse', 'rmse', or 'mae'.")
 
 
+def _resolve_compare_splits(
+    models: list[Any],
+    cv: int | str,
+    seed: int | None,
+    holdout_group: str | None,
+) -> tuple[list[tuple[np.ndarray, np.ndarray]], int]:
+    """Resolve fold splits for model comparison.
+
+    Checks all models for RE structure and resolves group-aware splitting.
+    For cross-type comparisons (lm vs lmer), the mixed model's grouping
+    governs splits for all models.
+
+    Args:
+        models: List of fitted models.
+        cv: Number of folds or ``"loo"``.
+        seed: Random seed.
+        holdout_group: User-specified grouping column, or None.
+
+    Returns:
+        Tuple of (splits list, effective integer k).
+    """
+    import warnings
+
+    n = models[0]._bundle.n
+
+    # Find the first model with RE structure (for cross-type comparisons)
+    group_ids = None
+    for m in models:
+        group_ids = resolve_holdout_group_ids(holdout_group, m.data, m._spec, m._bundle)
+        if group_ids is not None:
+            break
+
+    if group_ids is not None:
+        n_groups = len(np.unique(group_ids))
+
+        if cv == "loo":
+            splits = generate_group_kfold_splits(group_ids, k=n_groups, seed=seed)
+            return splits, n_groups
+
+        k = int(cv)
+        if k > n_groups:
+            warnings.warn(
+                f"k={k} exceeds number of groups ({n_groups}); using k={n_groups}",
+                stacklevel=4,
+            )
+            k = n_groups
+        splits = generate_group_kfold_splits(group_ids, k=k, seed=seed)
+        return splits, k
+
+    # No group structure — naive splits
+    if cv == "loo":
+        from bossanova.internal.infer.resample.core import generate_loo_indices
+
+        return generate_loo_indices(n), n
+
+    k = int(cv)
+    return generate_kfold_splits(n, k, seed=seed), k
+
+
 def compare_cv(
     models: list[Any],
     cv: int | str = 5,
     seed: int | None = None,
     metric: str = "mse",
+    holdout_group: str | None = None,
 ) -> pl.DataFrame:
     """Compare models using cross-validation with Nadeau-Bengio corrected t-test.
 
     Uses the corrected variance estimator from Nadeau & Bengio (2003) to account
     for the non-independence of CV folds (overlapping training sets).
 
+    When models have random effects, uses group-aware fold splitting to prevent
+    data leakage. For crossed random effects, ``holdout_group`` must be specified.
+
     Args:
-        models: List of fitted models (lm or glm) to compare.
+        models: List of fitted models to compare.
         cv: Number of folds, or "loo" for leave-one-out.
         seed: Random seed for reproducible fold splits.
         metric: Error metric ("mse", "rmse", "mae").
+        holdout_group: Column name to use for group-aware fold splitting.
+            Required for crossed random effects. Can reference any column
+            in the model's data, even if not in the formula.
 
     Returns:
-        DataFrame with columns:
-        - model: Formula string
-        - PRE: Proportional reduction in error
-        - t_stat: Nadeau-Bengio corrected t-statistic
-        - cv_score: Mean CV score (error)
-        - cv_se: Standard error of CV score
-        - diff: Difference from reference model (first model)
-        - diff_se: Corrected standard error of difference
-        - p_value: Two-sided p-value
+        DataFrame with columns: model, PRE, t_stat, cv_score, cv_se,
+        diff, diff_se, p_value.
 
     Notes:
         The Nadeau-Bengio correction accounts for the fact that k-fold CV
         training sets overlap by approximately (k-2)/(k-1) fraction.
 
-        Corrected variance: var_corrected = var(diff) * (1/k + n_test/n_train)
-        t-statistic: t = mean(diff) / sqrt(var_corrected)
-        p-value: 2 * (1 - t.cdf(|t|, df=k-1))
+        With group-aware splits, fold sizes may be unequal. The correction
+        uses the mean n_test/n_train ratio across folds.
 
     References:
         Nadeau & Bengio (2003) "Inference for the Generalization Error"
     """
     n_models = len(models)
-    n = models[0]._bundle.n
 
     # Generate shared fold splits for fair comparison
-    rng = RNG.from_seed(seed)
-    if cv == "loo":
-        k = n  # LOO has n folds
-        splits = generate_loo_indices(n)
-    else:
-        k = int(cv)
-        splits = generate_kfold_indices(rng, n, k, shuffle=True)
-    n_train = len(splits[0][0])
-    n_test = len(splits[0][1])
+    splits, k = _resolve_compare_splits(models, cv, seed, holdout_group)
+
+    # Compute mean n_test/n_train ratio for Nadeau-Bengio correction
+    # (handles unequal fold sizes from group splits)
+    ratios = [len(test) / len(train) for train, test in splits]
+    mean_ratio = float(np.mean(ratios))
 
     # Collect per-fold scores for each model
     fold_scores: dict[int, list[float]] = {i: [] for i in range(n_models)}
@@ -183,9 +234,8 @@ def compare_cv(
         mean_diff = np.mean(diff_vals)
         var_diff = np.var(diff_vals, ddof=1)
 
-        # Nadeau-Bengio correction
-        # var_corrected = var(diff) * (1/k + n_test/n_train)
-        var_corrected = var_diff * (1 / k + n_test / n_train)
+        # Nadeau-Bengio correction with mean ratio for unequal folds
+        var_corrected = var_diff * (1 / k + mean_ratio)
         se_corrected = np.sqrt(var_corrected)
 
         # t-statistic and p-value

{bossanova-0.1.0.dev24 → bossanova-0.1.0.dev26}/bossanova/internal/compare/helpers.py

@@ -10,6 +10,7 @@ from typing import Any
 
 __all__ = [
     "check_nested",
+    "get_model_family_group",
     "get_model_type",
     "resolve_method",
     "sort_models_by_complexity",
@@ -34,6 +35,24 @@ def get_model_type(model: Any) -> str:
     return model._model_type
 
 
+def get_model_family_group(model: Any) -> str:
+    """Get the family group for cross-type compatibility checking.
+
+    Models in the same family group can be compared via LRT because
+    the fixed-only model is nested within the mixed model (zero variance
+    components). Models in different family groups have incompatible
+    likelihoods and cannot be compared via hypothesis tests.
+
+    Args:
+        model: A fitted model instance.
+
+    Returns:
+        Family group string: 'gaussian' for lm/lmer, or the family name
+        (e.g. 'binomial', 'poisson', 'gamma') for glm/glmer.
+    """
+    return model._spec.family
+
+
 # =============================================================================
 # Validation and Sorting Helpers
 # =============================================================================
@@ -73,12 +92,31 @@ def validate_models(
         if not getattr(m, "_is_fitted", False):
             m.fit()
 
-    # Check same type
+    # Check type compatibility
     model_types = [get_model_type(m) for m in models]
     if len(set(model_types)) > 1:
-        raise ValueError(
-            f"All models must be the same type. Got: {', '.join(model_types)}"
+        # Cross-type comparisons are allowed for:
+        # - lm + lmer (both Gaussian) via LRT
+        # - glm + glmer (same family) via LRT
+        # - Any combination via AIC/BIC/CV (non-hypothesis methods)
+        #
+        # Disallowed cross-type pairs (different likelihoods):
+        # - lm + glm, lm + glmer, glm + lmer, lmer + glmer
+        type_set = set(model_types)
+        family_groups = {get_model_family_group(m) for m in models}
+
+        is_valid_cross_type = len(family_groups) == 1 and (
+            type_set <= {"lm", "lmer"} or type_set <= {"glm", "glmer"}
         )
+        is_ic_or_cv = method in ("aic", "bic", "cv")
+
+        if not is_valid_cross_type and not is_ic_or_cv:
+            raise ValueError(
+                f"Cannot compare models of types {', '.join(model_types)} "
+                f"with method='{method}'. Cross-type hypothesis tests require "
+                "compatible families (lm+lmer or glm+glmer with same family). "
+                "Use method='aic', 'bic', or 'cv' for non-nested comparison."
+            )
 
     # Check same response
     responses = [m._spec.response_var for m in models]
@@ -95,10 +133,13 @@ def validate_models(
         )
 
     # For LRT, check that all models use ML estimation (unless refit=True)
-    model_type = model_types[0]
-    inferred_method = method if method != "auto" else resolve_method(models[0])
+    type_set = set(model_types)
+    inferred_method = (
+        method if method != "auto" else resolve_method(models[0], model_types=type_set)
+    )
+    has_mixed = any(t in ("lmer", "glmer") for t in model_types)
 
-    if inferred_method == "lrt" and model_type in ("lmer", "glmer") and not refit:
+    if inferred_method == "lrt" and has_mixed and not refit:
         methods = [m._spec.method for m in models]
         reml_models = [m for m, meth in zip(models, methods) if meth == "reml"]
         if reml_models:
@@ -147,23 +188,30 @@ def sort_models_by_complexity(models: tuple[Any, ...]) -> list[Any]:
     return sorted(models, key=get_total_params)
 
 
-def resolve_method(model: Any) -> str:
+def resolve_method(model: Any, *, model_types: set[str] | None = None) -> str:
     """Infer the appropriate comparison method for a model type.
 
-    Reads ``model._model_type`` to determine the model type and maps it
-    to the corresponding comparison method.
+    For same-type comparisons, reads ``model._model_type`` to determine
+    the method. For cross-type comparisons (lm+lmer or glm+glmer),
+    returns ``"lrt"`` since the fixed-only model is nested within the
+    mixed model via zero variance components.
 
     Args:
         model: A fitted model object with a ``_model_type`` attribute
             (one of ``"lm"``, ``"glm"``, ``"lmer"``, ``"glmer"``).
+        model_types: Set of all model types being compared. When provided,
+            enables cross-type method resolution.
 
     Returns:
-        Comparison method: ``"f"`` for lm, ``"lrt"`` for lmer/glmer,
-        ``"deviance"`` for glm.
+        Comparison method: ``"f"`` for lm, ``"lrt"`` for lmer/glmer
+        (and cross-type lm+lmer or glm+glmer), ``"deviance"`` for glm.
 
     Raises:
         ValueError: If the model type cannot be mapped to a comparison method.
     """
+    # Cross-type: lm+lmer or glm+glmer → always LRT
+    if model_types is not None and len(model_types) > 1:
+        return "lrt"
 
     if model._model_type == "lm":
         return "f"

{bossanova-0.1.0.dev24 → bossanova-0.1.0.dev26}/bossanova/internal/compare/lrt_compare.py

@@ -16,7 +16,8 @@ from bossanova.internal.containers.schemas import (
 from bossanova.internal.maths.config import is_singular
 from bossanova.internal.maths.inference import compute_aic as compute_aic
 from bossanova.internal.maths.inference import compute_bic as _compute_bic
-from bossanova.internal.compare.helpers import check_nested, get_model_type
+from bossanova.internal.maths.family import ESTIMATED_DISPERSION_FAMILIES
+from bossanova.internal.compare.helpers import check_nested
 
 __all__ = [
     "compare_lrt",
@@ -25,23 +26,28 @@ __all__ = [
 
 
 def get_n_params(model: Any) -> int:
-    """Get total number of parameters for a mixed model.
+    """Get total number of estimated parameters for LRT comparison.
+
+    Handles all model types (lm, glm, lmer, glmer) for cross-type
+    LRT comparisons where a fixed-only model is nested within a mixed
+    model via zero variance components.
+
+    Parameter counting follows R's ``logLik()`` convention:
+
+    - Fixed effects (p)
+    - Variance parameters (len(theta), mixed models only)
+    - Dispersion/sigma (+1 for Gaussian and other estimated-dispersion families)
 
     Args:
-        model: A fitted lmer or glmer model.
+        model: A fitted model instance.
 
     Returns:
-        Total parameter count: fixed effects + variance parameters + sigma (lmer only).
+        Total parameter count.
     """
     n_fixed = model._bundle.p
-    n_theta = len(model._fit.theta)
-
-    # lmer has sigma as additional parameter, glmer does not
-    model_type = get_model_type(model)
-    if model_type == "lmer":
-        return n_fixed + n_theta + 1
-    else:  # glmer
-        return n_fixed + n_theta
+    n_theta = len(model._fit.theta) if model._fit.theta is not None else 0
+    n_disp = 1 if model._spec.family in ESTIMATED_DISPERSION_FAMILIES else 0
+    return n_fixed + n_theta + n_disp
 
 
 def compare_lrt(models: list[Any]) -> pl.DataFrame:

{bossanova-0.1.0.dev24 → bossanova-0.1.0.dev26}/bossanova/internal/containers/__init__.py

@@ -41,6 +41,7 @@ from bossanova.internal.containers.structs import (
     MathDisplay,
     MeeState,
     ModelSpec,
+    PredictionConfig,
     PredictionState,
     ProfileState,
     REInfo,
@@ -106,6 +107,7 @@ __all__ = [
     "MathDisplay",
     "MeeState",
     "ModelSpec",
+    "PredictionConfig",
     "PredictionState",
     "ProfileState",
     "REInfo",