spforge 0.8.8__py3-none-any.whl → 0.8.19__py3-none-any.whl

spforge/autopipeline.py

@@ -195,6 +195,40 @@ def lgbm_in_root(root) -> bool:
     return any(_is_lightgbm_estimator(obj) for obj in _walk_objects(root))
 
 
+def _get_importance_estimator(estimator) -> tuple[Any, str] | None:
+    """Recursively find innermost estimator with feature_importances_ or coef_."""
+    if hasattr(estimator, "feature_importances_"):
+        inner = _get_importance_estimator_inner(estimator)
+        if inner is not None:
+            return inner
+        return (estimator, "feature_importances_")
+
+    if hasattr(estimator, "coef_"):
+        inner = _get_importance_estimator_inner(estimator)
+        if inner is not None:
+            return inner
+        return (estimator, "coef_")
+
+    return _get_importance_estimator_inner(estimator)
+
+
+def _get_importance_estimator_inner(estimator) -> tuple[Any, str] | None:
+    """Check wrapped estimators for importance attributes."""
+    # Check estimator_ (sklearn fitted wrapper convention)
+    if hasattr(estimator, "estimator_") and estimator.estimator_ is not None:
+        result = _get_importance_estimator(estimator.estimator_)
+        if result is not None:
+            return result
+
+    # Check _est (GroupByEstimator convention)
+    if hasattr(estimator, "_est") and estimator._est is not None:
+        result = _get_importance_estimator(estimator._est)
+        if result is not None:
+            return result
+
+    return None
+
+
 class AutoPipeline(BaseEstimator):
     def __init__(
         self,
@@ -202,6 +236,7 @@ class AutoPipeline(BaseEstimator):
         estimator_features: list[str],
         predictor_transformers: list[PredictorTransformer] | None = None,
         granularity: list[str] | None = None,
+        aggregation_weight: str | None = None,
         filters: list[Filter] | None = None,
         scale_features: bool = False,
         categorical_handling: CategoricalHandling = "auto",
@@ -216,6 +251,7 @@ class AutoPipeline(BaseEstimator):
         self.estimator_features = estimator_features
         self.feature_names = estimator_features  # Internal compat
         self.granularity = granularity or []
+        self.aggregation_weight = aggregation_weight
         self.predictor_transformers = predictor_transformers
         self.estimator = estimator
         self.filters = filters or []
@@ -230,6 +266,7 @@ class AutoPipeline(BaseEstimator):
         self.numeric_features = numeric_features
         self.remainder = remainder
         self._cat_feats = []
+        self._filter_feature_names: list[str] = []
 
         # Auto-compute context features
         self.context_feature_names = self._compute_context_features()
@@ -242,11 +279,12 @@ class AutoPipeline(BaseEstimator):
         self._resolved_categorical_handling: CategoricalHandling | None = None
 
     def _compute_context_features(self) -> list[str]:
-        """Auto-compute context features from estimator, granularity, and filters.
+        """Auto-compute context features from estimator and granularity.
 
         Note: Context from predictor_transformers is tracked separately in
         context_predictor_transformer_feature_names and is dropped before
-        the final estimator.
+        the final estimator. Filter columns are tracked separately and are
+        dropped before the final estimator.
         """
         from spforge.transformers._base import PredictorTransformer
 
@@ -290,9 +328,15 @@ class AutoPipeline(BaseEstimator):
         # Add granularity columns
         context.extend(self.granularity)
 
+        # Add aggregation weight column
+        if self.aggregation_weight:
+            context.append(self.aggregation_weight)
+
         # Add filter columns
+        self._filter_feature_names = []
         for f in self.filters:
-            context.append(f.column_name)
+            if f.column_name not in self._filter_feature_names:
+                self._filter_feature_names.append(f.column_name)
 
         # Dedupe while preserving order, excluding estimator_features
         seen = set()
@@ -454,7 +498,11 @@ class AutoPipeline(BaseEstimator):
         pre = PreprocessorToDataFrame(pre_raw)
 
         est = (
-            GroupByEstimator(self.estimator, granularity=[f"{c}" for c in self.granularity])
+            GroupByEstimator(
+                self.estimator,
+                granularity=[f"{c}" for c in self.granularity],
+                aggregation_weight=self.aggregation_weight,
+            )
             if do_groupby
             else self.estimator
         )
@@ -506,8 +554,10 @@ class AutoPipeline(BaseEstimator):
             prev_transformer_feats_out.extend(feats_out)
 
         # Use FunctionTransformer with global function for serializability
+        drop_filter_cols = set(self._filter_feature_names)
+        drop_cols = drop_ctx_set | drop_filter_cols
         final = FunctionTransformer(
-            _drop_columns_transformer, validate=False, kw_args={"drop_cols": drop_ctx_set}
+            _drop_columns_transformer, validate=False, kw_args={"drop_cols": drop_cols}
         )
         steps.append(("final", final))
 
@@ -538,6 +588,7 @@ class AutoPipeline(BaseEstimator):
                 self.feature_names
                 + self.context_feature_names
                 + self.context_predictor_transformer_feature_names
+                + self._filter_feature_names
                 + self.granularity
             )
         )
@@ -626,4 +677,117 @@ class AutoPipeline(BaseEstimator):
             if ctx not in all_features:
                 all_features.append(ctx)
 
+        # Add filter columns (needed for fit-time filtering)
+        for col in self._filter_feature_names:
+            if col not in all_features:
+                all_features.append(col)
+
         return all_features
+
+    def _get_estimator_feature_names(self) -> list[str]:
+        """Get feature names as seen by the final estimator after all transformations."""
+        pre_out = list(self.sklearn_pipeline.named_steps["pre"].get_feature_names_out())
+
+        # Remove context columns dropped by "final" step
+        final_step = self.sklearn_pipeline.named_steps["final"]
+        drop_cols = final_step.kw_args.get("drop_cols", set()) if final_step.kw_args else set()
+        features = [f for f in pre_out if f not in drop_cols]
+
+        # Remove granularity columns (dropped by GroupByEstimator)
+        granularity_set = set(self.granularity)
+        features = [f for f in features if f not in granularity_set]
+
+        # Remove context features (used by wrapper estimators, not inner model)
+        context_set = set(self.context_feature_names)
+        features = [f for f in features if f not in context_set]
+
+        # Remove filter columns (used only for fit-time filtering)
+        filter_set = set(self._filter_feature_names)
+        features = [f for f in features if f not in filter_set]
+
+        return features
+
+    def _resolve_importance_feature_names(self, estimator, n_features: int) -> list[str]:
+        names = None
+        if hasattr(estimator, "feature_names_in_") and estimator.feature_names_in_ is not None:
+            names = list(estimator.feature_names_in_)
+        elif hasattr(estimator, "feature_name_") and estimator.feature_name_ is not None:
+            names = list(estimator.feature_name_)
+        elif hasattr(estimator, "feature_names_") and estimator.feature_names_ is not None:
+            names = list(estimator.feature_names_)
+        if names is None:
+            names = self._get_estimator_feature_names()
+        if len(names) != n_features:
+            raise ValueError(
+                f"Feature names length ({len(names)}) does not match importances length ({n_features})."
+            )
+        return names
+
+    @property
+    def feature_importances_(self) -> pd.DataFrame:
+        """Get feature importances from the fitted estimator.
+
+        Returns a DataFrame with columns ["feature", "importance"] sorted by
+        absolute importance descending. Works with tree-based models
+        (feature_importances_) and linear models (coef_).
+        """
+        if self.sklearn_pipeline is None:
+            raise RuntimeError("Pipeline not fitted. Call fit() first.")
+
+        est = self.sklearn_pipeline.named_steps["est"]
+        result = _get_importance_estimator(est)
+
+        if result is None:
+            raise RuntimeError(
+                "Estimator does not support feature importances. "
+                "Requires feature_importances_ or coef_ attribute."
+            )
+
+        inner_est, attr_name = result
+        raw = getattr(inner_est, attr_name)
+
+        if attr_name == "coef_":
+            # Linear models: use absolute value of coefficients
+            if raw.ndim == 2:
+                # Multi-class: average absolute values across classes
+                importances = np.abs(raw).mean(axis=0)
+            else:
+                importances = np.abs(raw)
+        else:
+            importances = raw
+
+        feature_names = self._get_estimator_feature_names()
+
+        df = pd.DataFrame({"feature": feature_names, "importance": importances})
+        df = df.sort_values("importance", ascending=False, key=abs).reset_index(drop=True)
+        return df
+
+    @property
+    def feature_importance_names(self) -> dict[str, float]:
+        """Map deepest estimator feature names to importances."""
+        if self.sklearn_pipeline is None:
+            raise RuntimeError("Pipeline not fitted. Call fit() first.")
+
+        est = self.sklearn_pipeline.named_steps["est"]
+        result = _get_importance_estimator(est)
+
+        if result is None:
+            raise RuntimeError(
+                "Estimator does not support feature importances. "
+                "Requires feature_importances_ or coef_ attribute."
+            )
+
+        inner_est, attr_name = result
+        raw = getattr(inner_est, attr_name)
+
+        if attr_name == "coef_":
+            if raw.ndim == 2:
+                importances = np.abs(raw).mean(axis=0)
+            else:
+                importances = np.abs(raw)
+        else:
+            importances = raw
+
+        importances = np.asarray(importances)
+        feature_names = self._resolve_importance_feature_names(inner_est, len(importances))
+        return dict(zip(feature_names, importances.tolist()))

spforge/estimator/_group_by_estimator.py

@@ -10,10 +10,16 @@ from spforge.transformers._other_transformer import GroupByReducer
 
 
 class GroupByEstimator(BaseEstimator):
-    def __init__(self, estimator: Any, granularity: list[str] | None = None):
+    def __init__(
+        self,
+        estimator: Any,
+        granularity: list[str] | None = None,
+        aggregation_weight: str | None = None,
+    ):
         self.estimator = estimator
         self.granularity = granularity or []
-        self._reducer = GroupByReducer(self.granularity)
+        self.aggregation_weight = aggregation_weight
+        self._reducer = GroupByReducer(self.granularity, aggregation_weight=aggregation_weight)
         self._est = None
 
     def __sklearn_is_fitted__(self):
@@ -22,7 +28,9 @@ class GroupByEstimator(BaseEstimator):
     @nw.narwhalify
     def fit(self, X: IntoFrameT, y: Any, sample_weight: np.ndarray | None = None):
         X = X.to_pandas()
-        self._reducer = GroupByReducer(self.granularity)
+        # Backwards compatibility: old pickled objects may not have aggregation_weight
+        agg_weight = getattr(self, "aggregation_weight", None)
+        self._reducer = GroupByReducer(self.granularity, aggregation_weight=agg_weight)
         X_red = nw.from_native(self._reducer.fit_transform(X))
         y_red, sw_red = self._reducer.reduce_y(X, y, sample_weight=sample_weight)
 

spforge/hyperparameter_tuning/__init__.py

@@ -7,6 +7,7 @@ from spforge.hyperparameter_tuning._default_search_spaces import (
     get_default_search_space,
     get_default_student_t_search_space,
     get_default_team_rating_search_space,
+    get_full_player_rating_search_space,
 )
 from spforge.hyperparameter_tuning._tuner import (
     EstimatorHyperparameterTuner,
@@ -28,4 +29,5 @@ __all__ = [
     "get_default_team_rating_search_space",
     "get_default_student_t_search_space",
     "get_default_search_space",
+    "get_full_player_rating_search_space",
 ]

spforge/hyperparameter_tuning/_default_search_spaces.py

@@ -128,6 +128,7 @@ def get_default_player_rating_search_space() -> dict[str, ParamSpec]:
     Default search space for PlayerRatingGenerator.
 
     Focuses on core parameters that have the most impact on performance.
+    Excludes performance_predictor and team-based start rating params.
 
     Returns:
         Dictionary mapping parameter names to ParamSpec objects
@@ -163,10 +164,6 @@ def get_default_player_rating_search_space() -> dict[str, ParamSpec]:
         "use_off_def_split": ParamSpec(
             param_type="bool",
         ),
-        "performance_predictor": ParamSpec(
-            param_type="categorical",
-            choices=["difference", "mean", "ignore_opponent"],
-        ),
         "start_league_quantile": ParamSpec(
             param_type="float",
             low=0.05,
@@ -177,24 +174,46 @@ def get_default_player_rating_search_space() -> dict[str, ParamSpec]:
             low=40,
             high=500,
         ),
-        "start_team_rating_subtract": ParamSpec(
-            param_type="float",
-            low=0.0,
-            high=200.0,
-        ),
-        "start_team_weight": ParamSpec(
-            param_type="float",
-            low=0.0,
-            high=1.0,
-        ),
-        "start_min_match_count_team_rating": ParamSpec(
-            param_type="int",
-            low=1,
-            high=10,
-        ),
     }
 
 
+def get_full_player_rating_search_space() -> dict[str, ParamSpec]:
+    """
+    Full search space for PlayerRatingGenerator including all tunable parameters.
+
+    Includes performance_predictor and team-based start rating parameters.
+    Use this when you want to tune all parameters.
+
+    Returns:
+        Dictionary mapping parameter names to ParamSpec objects
+    """
+    base = get_default_player_rating_search_space()
+    base.update(
+        {
+            "performance_predictor": ParamSpec(
+                param_type="categorical",
+                choices=["difference", "mean", "ignore_opponent"],
+            ),
+            "start_team_rating_subtract": ParamSpec(
+                param_type="float",
+                low=0.0,
+                high=200.0,
+            ),
+            "start_team_weight": ParamSpec(
+                param_type="float",
+                low=0.0,
+                high=1.0,
+            ),
+            "start_min_match_count_team_rating": ParamSpec(
+                param_type="int",
+                low=1,
+                high=10,
+            ),
+        }
+    )
+    return base
+
+
 def get_default_team_rating_search_space() -> dict[str, ParamSpec]:
     """
     Default search space for TeamRatingGenerator.
@@ -235,10 +254,6 @@ def get_default_team_rating_search_space() -> dict[str, ParamSpec]:
         "use_off_def_split": ParamSpec(
             param_type="bool",
         ),
-        "performance_predictor": ParamSpec(
-            param_type="categorical",
-            choices=["difference", "mean", "ignore_opponent"],
-        ),
     }
 
 

spforge/hyperparameter_tuning/_tuner.py

@@ -91,6 +91,9 @@ class RatingHyperparameterTuner:
         scorer: BaseScorer,
         direction: Literal["minimize", "maximize"],
         param_search_space: dict[str, ParamSpec] | None = None,
+        param_ranges: dict[str, tuple[float | int, float | int]] | None = None,
+        exclude_params: list[str] | None = None,
+        fixed_params: dict[str, Any] | None = None,
         n_trials: int = 50,
         n_jobs: int = 1,
         storage: str | None = None,
@@ -109,6 +112,14 @@ class RatingHyperparameterTuner:
             scorer: Scorer for evaluation (must have score(df) -> float | dict)
             direction: "minimize" or "maximize"
             param_search_space: Custom search space (merges with defaults if provided)
+            param_ranges: Easy range override for float/int params. Maps param name to
+                (low, high) tuple. Preserves param_type and log scale from defaults.
+                Example: {"confidence_weight": (0.2, 1.0)}
+            exclude_params: List of param names to exclude from tuning entirely.
+                Example: ["performance_predictor", "use_off_def_split"]
+            fixed_params: Parameters to fix at specific values (not tuned).
+                These values are applied to the rating generator each trial.
+                Example: {"performance_predictor": "mean"}
             n_trials: Number of optimization trials
             n_jobs: Number of parallel jobs (1 = sequential)
             storage: Optuna storage URL (e.g., "sqlite:///optuna.db") for persistence
@@ -123,6 +134,9 @@ class RatingHyperparameterTuner:
         self.scorer = scorer
         self.direction = direction
         self.custom_search_space = param_search_space
+        self.param_ranges = param_ranges
+        self.exclude_params = exclude_params or []
+        self.fixed_params = fixed_params or {}
         self.n_trials = n_trials
         self.n_jobs = n_jobs
         self.storage = storage
@@ -196,6 +210,9 @@ class RatingHyperparameterTuner:
         try:
             copied_gen = copy.deepcopy(self.rating_generator)
 
+            for param_name, param_value in self.fixed_params.items():
+                setattr(copied_gen, param_name, param_value)
+
             trial_params = self._suggest_params(trial, search_space)
 
             for param_name, param_value in trial_params.items():
@@ -243,18 +260,54 @@ class RatingHyperparameterTuner:
         defaults: dict[str, ParamSpec],
     ) -> dict[str, ParamSpec]:
         """
-        Merge custom search space with defaults (custom takes precedence).
+        Merge custom search space with defaults.
+
+        Priority order (highest to lowest):
+        1. exclude_params - removes param entirely
+        2. fixed_params - removes from search (applied separately)
+        3. custom (param_search_space) - full ParamSpec override
+        4. param_ranges - updates only low/high bounds
+        5. defaults - base search space
 
         Args:
             custom: Custom search space (may be None)
             defaults: Default search space
 
         Returns:
-            Merged search space
+            Merged search space (excludes fixed_params, those are applied separately)
         """
         merged = defaults.copy()
+
+        if self.param_ranges:
+            for param_name, (low, high) in self.param_ranges.items():
+                if param_name not in merged:
+                    raise ValueError(
+                        f"param_ranges contains unknown parameter: '{param_name}'. "
+                        f"Available parameters: {list(merged.keys())}"
+                    )
+                existing = merged[param_name]
+                if existing.param_type not in ("float", "int"):
+                    raise ValueError(
+                        f"param_ranges can only override float/int parameters. "
+                        f"'{param_name}' is {existing.param_type}."
+                    )
+                merged[param_name] = ParamSpec(
+                    param_type=existing.param_type,
+                    low=low,
+                    high=high,
+                    log=existing.log,
+                    step=existing.step,
+                )
+
         if custom:
             merged.update(custom)
+
+        for param_name in self.exclude_params:
+            merged.pop(param_name, None)
+
+        for param_name in self.fixed_params:
+            merged.pop(param_name, None)
+
         return merged
 
     @staticmethod

spforge/performance_transformers/_performance_manager.py

@@ -250,8 +250,6 @@ class PerformanceWeightsManager(PerformanceManager):
                 )
             )
 
-        sum_weight = sum([w.weight for w in self.weights])
-
         for column_weight in self.weights:
             weight_col = f"weight__{column_weight.name}"
             feature_col = column_weight.name
@@ -261,14 +259,14 @@ class PerformanceWeightsManager(PerformanceManager):
                 df = df.with_columns(
                     (
                         nw.col(tmp_out_performance_colum_name)
-                        + (nw.col(weight_col) / sum_weight * (1 - nw.col(feature_name)))
+                        + (nw.col(weight_col) * (1 - nw.col(feature_name)))
                     ).alias(tmp_out_performance_colum_name)
                 )
             else:
                 df = df.with_columns(
                     (
                         nw.col(tmp_out_performance_colum_name)
-                        + (nw.col(weight_col) / sum_weight * nw.col(feature_name))
+                        + (nw.col(weight_col) * nw.col(feature_name))
                     ).alias(tmp_out_performance_colum_name)
                 )