spforge 0.8.4__py3-none-any.whl → 0.8.18__py3-none-any.whl

spforge/autopipeline.py

@@ -195,6 +195,40 @@ def lgbm_in_root(root) -> bool:
     return any(_is_lightgbm_estimator(obj) for obj in _walk_objects(root))
 
 
+def _get_importance_estimator(estimator) -> tuple[Any, str] | None:
+    """Recursively find innermost estimator with feature_importances_ or coef_."""
+    if hasattr(estimator, "feature_importances_"):
+        inner = _get_importance_estimator_inner(estimator)
+        if inner is not None:
+            return inner
+        return (estimator, "feature_importances_")
+
+    if hasattr(estimator, "coef_"):
+        inner = _get_importance_estimator_inner(estimator)
+        if inner is not None:
+            return inner
+        return (estimator, "coef_")
+
+    return _get_importance_estimator_inner(estimator)
+
+
+def _get_importance_estimator_inner(estimator) -> tuple[Any, str] | None:
+    """Check wrapped estimators for importance attributes."""
+    # Check estimator_ (sklearn fitted wrapper convention)
+    if hasattr(estimator, "estimator_") and estimator.estimator_ is not None:
+        result = _get_importance_estimator(estimator.estimator_)
+        if result is not None:
+            return result
+
+    # Check _est (GroupByEstimator convention)
+    if hasattr(estimator, "_est") and estimator._est is not None:
+        result = _get_importance_estimator(estimator._est)
+        if result is not None:
+            return result
+
+    return None
+
+
 class AutoPipeline(BaseEstimator):
     def __init__(
         self,
@@ -202,6 +236,7 @@ class AutoPipeline(BaseEstimator):
         estimator_features: list[str],
         predictor_transformers: list[PredictorTransformer] | None = None,
         granularity: list[str] | None = None,
+        aggregation_weight: str | None = None,
         filters: list[Filter] | None = None,
         scale_features: bool = False,
         categorical_handling: CategoricalHandling = "auto",
@@ -216,6 +251,7 @@ class AutoPipeline(BaseEstimator):
         self.estimator_features = estimator_features
         self.feature_names = estimator_features  # Internal compat
         self.granularity = granularity or []
+        self.aggregation_weight = aggregation_weight
         self.predictor_transformers = predictor_transformers
         self.estimator = estimator
         self.filters = filters or []
@@ -230,6 +266,7 @@ class AutoPipeline(BaseEstimator):
         self.numeric_features = numeric_features
         self.remainder = remainder
         self._cat_feats = []
+        self._filter_feature_names: list[str] = []
 
         # Auto-compute context features
         self.context_feature_names = self._compute_context_features()
@@ -242,11 +279,12 @@ class AutoPipeline(BaseEstimator):
         self._resolved_categorical_handling: CategoricalHandling | None = None
 
     def _compute_context_features(self) -> list[str]:
-        """Auto-compute context features from estimator, granularity, and filters.
+        """Auto-compute context features from estimator and granularity.
 
         Note: Context from predictor_transformers is tracked separately in
         context_predictor_transformer_feature_names and is dropped before
-        the final estimator.
+        the final estimator. Filter columns are tracked separately and are
+        dropped before the final estimator.
         """
         from spforge.transformers._base import PredictorTransformer
 
@@ -290,9 +328,15 @@ class AutoPipeline(BaseEstimator):
         # Add granularity columns
         context.extend(self.granularity)
 
+        # Add aggregation weight column
+        if self.aggregation_weight:
+            context.append(self.aggregation_weight)
+
         # Add filter columns
+        self._filter_feature_names = []
         for f in self.filters:
-            context.append(f.column_name)
+            if f.column_name not in self._filter_feature_names:
+                self._filter_feature_names.append(f.column_name)
 
         # Dedupe while preserving order, excluding estimator_features
         seen = set()
@@ -454,7 +498,11 @@ class AutoPipeline(BaseEstimator):
         pre = PreprocessorToDataFrame(pre_raw)
 
         est = (
-            GroupByEstimator(self.estimator, granularity=[f"{c}" for c in self.granularity])
+            GroupByEstimator(
+                self.estimator,
+                granularity=[f"{c}" for c in self.granularity],
+                aggregation_weight=self.aggregation_weight,
+            )
             if do_groupby
             else self.estimator
         )
@@ -506,8 +554,10 @@ class AutoPipeline(BaseEstimator):
             prev_transformer_feats_out.extend(feats_out)
 
         # Use FunctionTransformer with global function for serializability
+        drop_filter_cols = set(self._filter_feature_names)
+        drop_cols = drop_ctx_set | drop_filter_cols
         final = FunctionTransformer(
-            _drop_columns_transformer, validate=False, kw_args={"drop_cols": drop_ctx_set}
+            _drop_columns_transformer, validate=False, kw_args={"drop_cols": drop_cols}
         )
         steps.append(("final", final))
 
@@ -538,6 +588,7 @@ class AutoPipeline(BaseEstimator):
                 self.feature_names
                 + self.context_feature_names
                 + self.context_predictor_transformer_feature_names
+                + self._filter_feature_names
                 + self.granularity
             )
         )
@@ -626,4 +677,117 @@ class AutoPipeline(BaseEstimator):
             if ctx not in all_features:
                 all_features.append(ctx)
 
+        # Add filter columns (needed for fit-time filtering)
+        for col in self._filter_feature_names:
+            if col not in all_features:
+                all_features.append(col)
+
         return all_features
+
+    def _get_estimator_feature_names(self) -> list[str]:
+        """Get feature names as seen by the final estimator after all transformations."""
+        pre_out = list(self.sklearn_pipeline.named_steps["pre"].get_feature_names_out())
+
+        # Remove context columns dropped by "final" step
+        final_step = self.sklearn_pipeline.named_steps["final"]
+        drop_cols = final_step.kw_args.get("drop_cols", set()) if final_step.kw_args else set()
+        features = [f for f in pre_out if f not in drop_cols]
+
+        # Remove granularity columns (dropped by GroupByEstimator)
+        granularity_set = set(self.granularity)
+        features = [f for f in features if f not in granularity_set]
+
+        # Remove context features (used by wrapper estimators, not inner model)
+        context_set = set(self.context_feature_names)
+        features = [f for f in features if f not in context_set]
+
+        # Remove filter columns (used only for fit-time filtering)
+        filter_set = set(self._filter_feature_names)
+        features = [f for f in features if f not in filter_set]
+
+        return features
+
+    def _resolve_importance_feature_names(self, estimator, n_features: int) -> list[str]:
+        names = None
+        if hasattr(estimator, "feature_names_in_") and estimator.feature_names_in_ is not None:
+            names = list(estimator.feature_names_in_)
+        elif hasattr(estimator, "feature_name_") and estimator.feature_name_ is not None:
+            names = list(estimator.feature_name_)
+        elif hasattr(estimator, "feature_names_") and estimator.feature_names_ is not None:
+            names = list(estimator.feature_names_)
+        if names is None:
+            names = self._get_estimator_feature_names()
+        if len(names) != n_features:
+            raise ValueError(
+                f"Feature names length ({len(names)}) does not match importances length ({n_features})."
+            )
+        return names
+
+    @property
+    def feature_importances_(self) -> pd.DataFrame:
+        """Get feature importances from the fitted estimator.
+
+        Returns a DataFrame with columns ["feature", "importance"] sorted by
+        absolute importance descending. Works with tree-based models
+        (feature_importances_) and linear models (coef_).
+        """
+        if self.sklearn_pipeline is None:
+            raise RuntimeError("Pipeline not fitted. Call fit() first.")
+
+        est = self.sklearn_pipeline.named_steps["est"]
+        result = _get_importance_estimator(est)
+
+        if result is None:
+            raise RuntimeError(
+                "Estimator does not support feature importances. "
+                "Requires feature_importances_ or coef_ attribute."
+            )
+
+        inner_est, attr_name = result
+        raw = getattr(inner_est, attr_name)
+
+        if attr_name == "coef_":
+            # Linear models: use absolute value of coefficients
+            if raw.ndim == 2:
+                # Multi-class: average absolute values across classes
+                importances = np.abs(raw).mean(axis=0)
+            else:
+                importances = np.abs(raw)
+        else:
+            importances = raw
+
+        feature_names = self._get_estimator_feature_names()
+
+        df = pd.DataFrame({"feature": feature_names, "importance": importances})
+        df = df.sort_values("importance", ascending=False, key=abs).reset_index(drop=True)
+        return df
+
+    @property
+    def feature_importance_names(self) -> dict[str, float]:
+        """Map deepest estimator feature names to importances."""
+        if self.sklearn_pipeline is None:
+            raise RuntimeError("Pipeline not fitted. Call fit() first.")
+
+        est = self.sklearn_pipeline.named_steps["est"]
+        result = _get_importance_estimator(est)
+
+        if result is None:
+            raise RuntimeError(
+                "Estimator does not support feature importances. "
+                "Requires feature_importances_ or coef_ attribute."
+            )
+
+        inner_est, attr_name = result
+        raw = getattr(inner_est, attr_name)
+
+        if attr_name == "coef_":
+            if raw.ndim == 2:
+                importances = np.abs(raw).mean(axis=0)
+            else:
+                importances = np.abs(raw)
+        else:
+            importances = raw
+
+        importances = np.asarray(importances)
+        feature_names = self._resolve_importance_feature_names(inner_est, len(importances))
+        return dict(zip(feature_names, importances.tolist()))

spforge/estimator/_group_by_estimator.py

@@ -10,10 +10,16 @@ from spforge.transformers._other_transformer import GroupByReducer
 
 
 class GroupByEstimator(BaseEstimator):
-    def __init__(self, estimator: Any, granularity: list[str] | None = None):
+    def __init__(
+        self,
+        estimator: Any,
+        granularity: list[str] | None = None,
+        aggregation_weight: str | None = None,
+    ):
         self.estimator = estimator
         self.granularity = granularity or []
-        self._reducer = GroupByReducer(self.granularity)
+        self.aggregation_weight = aggregation_weight
+        self._reducer = GroupByReducer(self.granularity, aggregation_weight=aggregation_weight)
         self._est = None
 
     def __sklearn_is_fitted__(self):
@@ -22,7 +28,9 @@ class GroupByEstimator(BaseEstimator):
     @nw.narwhalify
     def fit(self, X: IntoFrameT, y: Any, sample_weight: np.ndarray | None = None):
         X = X.to_pandas()
-        self._reducer = GroupByReducer(self.granularity)
+        # Backwards compatibility: old pickled objects may not have aggregation_weight
+        agg_weight = getattr(self, "aggregation_weight", None)
+        self._reducer = GroupByReducer(self.granularity, aggregation_weight=agg_weight)
         X_red = nw.from_native(self._reducer.fit_transform(X))
         y_red, sw_red = self._reducer.reduce_y(X, y, sample_weight=sample_weight)
 

spforge/features_generator_pipeline.py

@@ -120,7 +120,8 @@ class FeatureGeneratorPipeline(FeatureGenerator):
 
         for transformer in self.feature_generators:
             pre_row_count = len(df)
-            df = nw.from_native(transformer.fit_transform(df, column_names=column_names))
+            native_df = df.to_native()
+            df = nw.from_native(transformer.fit_transform(native_df, column_names=column_names))
             assert len(df) == pre_row_count
             for f in transformer.features_out:
                 if f in expected_feats_added:
@@ -151,7 +152,8 @@ class FeatureGeneratorPipeline(FeatureGenerator):
 
         for transformer in self.feature_generators:
             pre_row_count = len(df)
-            df = nw.from_native(transformer.transform(df))
+            native_df = df.to_native()
+            df = nw.from_native(transformer.transform(native_df))
             assert len(df) == pre_row_count
             for f in transformer.features_out:
                 if f in expected_feats_added:
@@ -181,9 +183,11 @@ class FeatureGeneratorPipeline(FeatureGenerator):
         for transformer in self.feature_generators:
             pre_row_count = len(df)
             if hasattr(transformer, "future_transform") and callable(transformer.future_transform):
-                df = nw.from_native(transformer.future_transform(df))
+                native_df = df.to_native()
+                df = nw.from_native(transformer.future_transform(native_df))
             else:
-                df = nw.from_native(transformer.transform(df))
+                native_df = df.to_native()
+                df = nw.from_native(transformer.transform(native_df))
             assert len(df) == pre_row_count
             for f in transformer.features_out:
                 if f in expected_feats_added:

spforge/hyperparameter_tuning/__init__.py

@@ -1,9 +1,15 @@
 from spforge.hyperparameter_tuning._default_search_spaces import (
+    get_default_estimator_search_space,
+    get_default_lgbm_search_space,
+    get_default_negative_binomial_search_space,
+    get_default_normal_distribution_search_space,
     get_default_player_rating_search_space,
     get_default_search_space,
+    get_default_student_t_search_space,
     get_default_team_rating_search_space,
 )
 from spforge.hyperparameter_tuning._tuner import (
+    EstimatorHyperparameterTuner,
     OptunaResult,
     ParamSpec,
     RatingHyperparameterTuner,
@@ -11,9 +17,15 @@ from spforge.hyperparameter_tuning._tuner import (
 
 __all__ = [
     "RatingHyperparameterTuner",
+    "EstimatorHyperparameterTuner",
     "ParamSpec",
     "OptunaResult",
+    "get_default_estimator_search_space",
+    "get_default_lgbm_search_space",
+    "get_default_negative_binomial_search_space",
+    "get_default_normal_distribution_search_space",
     "get_default_player_rating_search_space",
     "get_default_team_rating_search_space",
+    "get_default_student_t_search_space",
     "get_default_search_space",
 ]

spforge/hyperparameter_tuning/_default_search_spaces.py

@@ -1,12 +1,133 @@
 from spforge.hyperparameter_tuning._tuner import ParamSpec
 from spforge.ratings import PlayerRatingGenerator, TeamRatingGenerator
+from spforge.distributions import (
+    NegativeBinomialEstimator,
+    NormalDistributionPredictor,
+    StudentTDistributionEstimator,
+)
+
+
+def _is_lightgbm_estimator(obj: object) -> bool:
+    mod = (getattr(type(obj), "__module__", "") or "").lower()
+    name = type(obj).__name__
+    if "lightgbm" in mod:
+        return True
+    return bool(name.startswith("LGBM"))
+
+
+def get_default_lgbm_search_space() -> dict[str, ParamSpec]:
+    return {
+        "n_estimators": ParamSpec(
+            param_type="int",
+            low=50,
+            high=800,
+            log=True,
+        ),
+        "num_leaves": ParamSpec(
+            param_type="int",
+            low=16,
+            high=256,
+            log=True,
+        ),
+        "max_depth": ParamSpec(
+            param_type="int",
+            low=3,
+            high=12,
+        ),
+        "min_child_samples": ParamSpec(
+            param_type="int",
+            low=10,
+            high=200,
+            log=True,
+        ),
+        "subsample": ParamSpec(
+            param_type="float",
+            low=0.6,
+            high=1.0,
+        ),
+        "subsample_freq": ParamSpec(
+            param_type="int",
+            low=1,
+            high=7,
+        ),
+        "reg_alpha": ParamSpec(
+            param_type="float",
+            low=1e-8,
+            high=10.0,
+            log=True,
+        ),
+        "reg_lambda": ParamSpec(
+            param_type="float",
+            low=1e-8,
+            high=10.0,
+            log=True,
+        ),
+    }
+
+
+def get_default_negative_binomial_search_space() -> dict[str, ParamSpec]:
+    return {
+        "predicted_r_weight": ParamSpec(
+            param_type="float",
+            low=0.0,
+            high=1.0,
+        ),
+        "r_rolling_mean_window": ParamSpec(
+            param_type="int",
+            low=10,
+            high=120,
+        ),
+        "predicted_r_iterations": ParamSpec(
+            param_type="int",
+            low=2,
+            high=12,
+        ),
+    }
+
+
+def get_default_normal_distribution_search_space() -> dict[str, ParamSpec]:
+    return {
+        "sigma": ParamSpec(
+            param_type="float",
+            low=0.5,
+            high=30.0,
+            log=True,
+        ),
+    }
+
+
+def get_default_student_t_search_space() -> dict[str, ParamSpec]:
+    return {
+        "df": ParamSpec(
+            param_type="float",
+            low=3.0,
+            high=30.0,
+            log=True,
+        ),
+        "min_sigma": ParamSpec(
+            param_type="float",
+            low=0.5,
+            high=10.0,
+            log=True,
+        ),
+        "sigma_bins": ParamSpec(
+            param_type="int",
+            low=4,
+            high=12,
+        ),
+        "min_bin_rows": ParamSpec(
+            param_type="int",
+            low=10,
+            high=100,
+        ),
+    }
 
 
 def get_default_player_rating_search_space() -> dict[str, ParamSpec]:
     """
     Default search space for PlayerRatingGenerator.
 
-    Focuses on 5-8 core parameters that have the most impact on performance.
+    Focuses on core parameters that have the most impact on performance.
 
     Returns:
         Dictionary mapping parameter names to ParamSpec objects
@@ -46,6 +167,31 @@ def get_default_player_rating_search_space() -> dict[str, ParamSpec]:
             param_type="categorical",
             choices=["difference", "mean", "ignore_opponent"],
         ),
+        "start_league_quantile": ParamSpec(
+            param_type="float",
+            low=0.05,
+            high=0.5,
+        ),
+        "start_min_count_for_percentiles": ParamSpec(
+            param_type="int",
+            low=40,
+            high=500,
+        ),
+        "start_team_rating_subtract": ParamSpec(
+            param_type="float",
+            low=0.0,
+            high=200.0,
+        ),
+        "start_team_weight": ParamSpec(
+            param_type="float",
+            low=0.0,
+            high=1.0,
+        ),
+        "start_min_match_count_team_rating": ParamSpec(
+            param_type="int",
+            low=1,
+            high=10,
+        ),
     }
 
 
@@ -120,3 +266,15 @@ def get_default_search_space(
             f"Unsupported rating generator type: {type(rating_generator)}. "
             "Expected PlayerRatingGenerator or TeamRatingGenerator."
         )
+
+
+def get_default_estimator_search_space(estimator: object) -> dict[str, ParamSpec]:
+    if _is_lightgbm_estimator(estimator):
+        return get_default_lgbm_search_space()
+    if isinstance(estimator, NegativeBinomialEstimator):
+        return get_default_negative_binomial_search_space()
+    if isinstance(estimator, NormalDistributionPredictor):
+        return get_default_normal_distribution_search_space()
+    if isinstance(estimator, StudentTDistributionEstimator):
+        return get_default_student_t_search_space()
+    return {}