spforge 0.8.4__tar.gz → 0.8.5__tar.gz

{spforge-0.8.4/spforge.egg-info → spforge-0.8.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spforge
-Version: 0.8.4
+Version: 0.8.5
 Summary: A flexible framework for generating features, ratings, and building machine learning or other models for training and inference on sports data.
 Author-email: Mathias Holmstrøm <mathiasholmstom@gmail.com>
 License: See LICENSE file
@@ -17,7 +17,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy>=1.23.4
 Requires-Dist: optuna>=3.4.0
-Requires-Dist: pandas>=2.0.0
+Requires-Dist: pandas<3.0.0,>=2.0.0
 Requires-Dist: pendulum>=1.0.0
 Requires-Dist: scikit-learn>=1.4.0
 Requires-Dist: lightgbm>=4.0.0

{spforge-0.8.4 → spforge-0.8.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spforge"
-version = "0.8.4"
+version = "0.8.5"
 description = "A flexible framework for generating features, ratings, and building machine learning or other models for training and inference on sports data."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -23,7 +23,7 @@ classifiers = [
 dependencies = [
   "numpy>=1.23.4",
   "optuna>=3.4.0",
-  "pandas>=2.0.0",
+  "pandas>=2.0.0,<3.0.0",
   "pendulum>=1.0.0",
   "scikit-learn >=1.4.0",
   "lightgbm>=4.0.0",

{spforge-0.8.4 → spforge-0.8.5}/spforge/__init__.py

@@ -2,6 +2,7 @@ from .autopipeline import AutoPipeline as AutoPipeline
 from .data_structures import ColumnNames as ColumnNames, GameColumnNames as GameColumnNames
 from .features_generator_pipeline import FeatureGeneratorPipeline as FeatureGeneratorPipeline
 from .hyperparameter_tuning import (
+    EstimatorHyperparameterTuner as EstimatorHyperparameterTuner,
     OptunaResult as OptunaResult,
     ParamSpec as ParamSpec,
     RatingHyperparameterTuner as RatingHyperparameterTuner,

{spforge-0.8.4 → spforge-0.8.5}/spforge/hyperparameter_tuning/__init__.py

@@ -1,9 +1,15 @@
 from spforge.hyperparameter_tuning._default_search_spaces import (
+    get_default_estimator_search_space,
+    get_default_lgbm_search_space,
+    get_default_negative_binomial_search_space,
+    get_default_normal_distribution_search_space,
     get_default_player_rating_search_space,
     get_default_search_space,
+    get_default_student_t_search_space,
     get_default_team_rating_search_space,
 )
 from spforge.hyperparameter_tuning._tuner import (
+    EstimatorHyperparameterTuner,
     OptunaResult,
     ParamSpec,
     RatingHyperparameterTuner,
@@ -11,9 +17,15 @@ from spforge.hyperparameter_tuning._tuner import (
 
 __all__ = [
     "RatingHyperparameterTuner",
+    "EstimatorHyperparameterTuner",
     "ParamSpec",
     "OptunaResult",
+    "get_default_estimator_search_space",
+    "get_default_lgbm_search_space",
+    "get_default_negative_binomial_search_space",
+    "get_default_normal_distribution_search_space",
     "get_default_player_rating_search_space",
     "get_default_team_rating_search_space",
+    "get_default_student_t_search_space",
     "get_default_search_space",
 ]

{spforge-0.8.4 → spforge-0.8.5}/spforge/hyperparameter_tuning/_default_search_spaces.py

@@ -1,5 +1,126 @@
 from spforge.hyperparameter_tuning._tuner import ParamSpec
 from spforge.ratings import PlayerRatingGenerator, TeamRatingGenerator
+from spforge.distributions import (
+    NegativeBinomialEstimator,
+    NormalDistributionPredictor,
+    StudentTDistributionEstimator,
+)
+
+
+def _is_lightgbm_estimator(obj: object) -> bool:
+    mod = (getattr(type(obj), "__module__", "") or "").lower()
+    name = type(obj).__name__
+    if "lightgbm" in mod:
+        return True
+    return bool(name.startswith("LGBM"))
+
+
+def get_default_lgbm_search_space() -> dict[str, ParamSpec]:
+    return {
+        "n_estimators": ParamSpec(
+            param_type="int",
+            low=50,
+            high=800,
+            log=True,
+        ),
+        "num_leaves": ParamSpec(
+            param_type="int",
+            low=16,
+            high=256,
+            log=True,
+        ),
+        "max_depth": ParamSpec(
+            param_type="int",
+            low=3,
+            high=12,
+        ),
+        "min_child_samples": ParamSpec(
+            param_type="int",
+            low=10,
+            high=200,
+            log=True,
+        ),
+        "subsample": ParamSpec(
+            param_type="float",
+            low=0.6,
+            high=1.0,
+        ),
+        "subsample_freq": ParamSpec(
+            param_type="int",
+            low=1,
+            high=7,
+        ),
+        "reg_alpha": ParamSpec(
+            param_type="float",
+            low=1e-8,
+            high=10.0,
+            log=True,
+        ),
+        "reg_lambda": ParamSpec(
+            param_type="float",
+            low=1e-8,
+            high=10.0,
+            log=True,
+        ),
+    }
+
+
+def get_default_negative_binomial_search_space() -> dict[str, ParamSpec]:
+    return {
+        "predicted_r_weight": ParamSpec(
+            param_type="float",
+            low=0.0,
+            high=1.0,
+        ),
+        "r_rolling_mean_window": ParamSpec(
+            param_type="int",
+            low=10,
+            high=120,
+        ),
+        "predicted_r_iterations": ParamSpec(
+            param_type="int",
+            low=2,
+            high=12,
+        ),
+    }
+
+
+def get_default_normal_distribution_search_space() -> dict[str, ParamSpec]:
+    return {
+        "sigma": ParamSpec(
+            param_type="float",
+            low=0.5,
+            high=30.0,
+            log=True,
+        ),
+    }
+
+
+def get_default_student_t_search_space() -> dict[str, ParamSpec]:
+    return {
+        "df": ParamSpec(
+            param_type="float",
+            low=3.0,
+            high=30.0,
+            log=True,
+        ),
+        "min_sigma": ParamSpec(
+            param_type="float",
+            low=0.5,
+            high=10.0,
+            log=True,
+        ),
+        "sigma_bins": ParamSpec(
+            param_type="int",
+            low=4,
+            high=12,
+        ),
+        "min_bin_rows": ParamSpec(
+            param_type="int",
+            low=10,
+            high=100,
+        ),
+    }
 
 
 def get_default_player_rating_search_space() -> dict[str, ParamSpec]:
@@ -120,3 +241,15 @@ def get_default_search_space(
             f"Unsupported rating generator type: {type(rating_generator)}. "
             "Expected PlayerRatingGenerator or TeamRatingGenerator."
         )
+
+
+def get_default_estimator_search_space(estimator: object) -> dict[str, ParamSpec]:
+    if _is_lightgbm_estimator(estimator):
+        return get_default_lgbm_search_space()
+    if isinstance(estimator, NegativeBinomialEstimator):
+        return get_default_negative_binomial_search_space()
+    if isinstance(estimator, NormalDistributionPredictor):
+        return get_default_normal_distribution_search_space()
+    if isinstance(estimator, StudentTDistributionEstimator):
+        return get_default_student_t_search_space()
+    return {}

{spforge-0.8.4 → spforge-0.8.5}/spforge/hyperparameter_tuning/_tuner.py

@@ -45,6 +45,8 @@ class ParamSpec:
         elif self.param_type == "int":
             if self.low is None or self.high is None:
                 raise ValueError(f"int parameter '{name}' requires low and high bounds")
+            if self.step is None:
+                return trial.suggest_int(name, int(self.low), int(self.high))
             return trial.suggest_int(name, int(self.low), int(self.high), step=self.step)
         elif self.param_type == "categorical":
             if self.choices is None:
@@ -272,3 +274,193 @@ class RatingHyperparameterTuner:
                 raise ValueError("Scorer returned invalid values in dict")
             return float(np.mean(values))
         return float(score)
+
+
+def _is_estimator(obj: object) -> bool:
+    return hasattr(obj, "get_params") and hasattr(obj, "set_params")
+
+
+def _get_leaf_estimator_paths(estimator: Any) -> dict[str, Any]:
+    if not _is_estimator(estimator):
+        raise ValueError("estimator must implement get_params and set_params")
+
+    params = estimator.get_params(deep=True)
+    estimator_keys = [k for k, v in params.items() if _is_estimator(v)]
+
+    if not estimator_keys:
+        return {"": estimator}
+
+    leaves: list[str] = []
+    for key in estimator_keys:
+        if not any(other != key and other.startswith(f"{key}__") for other in estimator_keys):
+            leaves.append(key)
+
+    return {key: params[key] for key in sorted(leaves)}
+
+
+def _build_search_space_for_targets(
+    targets: dict[str, dict[str, ParamSpec]],
+) -> dict[str, ParamSpec]:
+    search_space: dict[str, ParamSpec] = {}
+    for path, params in targets.items():
+        for param_name, param_spec in params.items():
+            full_name = f"{path}__{param_name}" if path else param_name
+            if full_name in search_space:
+                raise ValueError(f"Duplicate parameter name detected: {full_name}")
+            search_space[full_name] = param_spec
+    return search_space
+
+
+def _enqueue_predicted_r_weight_zero(study: optuna.Study, search_space: dict[str, ParamSpec]):
+    zero_params: dict[str, float] = {}
+    for name, spec in search_space.items():
+        if not name.endswith("predicted_r_weight"):
+            continue
+        if spec.param_type not in {"float", "int"}:
+            continue
+        if spec.low is None or spec.high is None:
+            continue
+        if spec.low <= 0 <= spec.high:
+            zero_params[name] = 0.0
+
+    if zero_params:
+        study.enqueue_trial(zero_params)
+
+
+class EstimatorHyperparameterTuner:
+    """
+    Hyperparameter tuner for sklearn-compatible estimators.
+
+    Supports nested estimators and can target deepest leaf estimators.
+    """
+
+    def __init__(
+        self,
+        estimator: Any,
+        cross_validator: MatchKFoldCrossValidator,
+        scorer: BaseScorer,
+        direction: Literal["minimize", "maximize"],
+        param_search_space: dict[str, ParamSpec] | None = None,
+        param_targets: dict[str, dict[str, ParamSpec]] | None = None,
+        n_trials: int = 50,
+        n_jobs: int = 1,
+        storage: str | None = None,
+        study_name: str | None = None,
+        timeout: float | None = None,
+        show_progress_bar: bool = True,
+        sampler: optuna.samplers.BaseSampler | None = None,
+        pruner: optuna.pruners.BasePruner | None = None,
+    ):
+        self.estimator = estimator
+        self.cross_validator = cross_validator
+        self.scorer = scorer
+        self.direction = direction
+        self.param_search_space = param_search_space
+        self.param_targets = param_targets
+        self.n_trials = n_trials
+        self.n_jobs = n_jobs
+        self.storage = storage
+        self.study_name = study_name
+        self.timeout = timeout
+        self.show_progress_bar = show_progress_bar
+        self.sampler = sampler
+        self.pruner = pruner
+
+        if direction not in ["minimize", "maximize"]:
+            raise ValueError(f"direction must be 'minimize' or 'maximize', got: {direction}")
+
+        if storage is not None and study_name is None:
+            raise ValueError("study_name is required when using storage")
+
+        if param_search_space is not None and param_targets is not None:
+            raise ValueError("param_search_space and param_targets cannot both be provided")
+
+    def optimize(self, df: IntoFrameT) -> OptunaResult:
+        from spforge.hyperparameter_tuning._default_search_spaces import (
+            get_default_estimator_search_space,
+        )
+
+        leaf_estimators = _get_leaf_estimator_paths(self.estimator)
+        default_targets = {
+            path: get_default_estimator_search_space(est)
+            for path, est in leaf_estimators.items()
+        }
+        default_targets = {path: space for path, space in default_targets.items() if space}
+
+        if self.param_targets is not None:
+            unknown = set(self.param_targets) - set(leaf_estimators)
+            if unknown:
+                raise ValueError(f"param_targets contains unknown estimator paths: {unknown}")
+            targets = self.param_targets
+        elif self.param_search_space is not None:
+            targets = {path: self.param_search_space for path in leaf_estimators}
+        elif default_targets:
+            targets = default_targets
+        else:
+            raise ValueError(
+                "param_search_space is required when no default search space is available"
+            )
+
+        search_space = _build_search_space_for_targets(targets)
+        if not search_space:
+            raise ValueError("Resolved search space is empty")
+
+        study = optuna.create_study(
+            direction=self.direction,
+            sampler=self.sampler,
+            pruner=self.pruner,
+            storage=self.storage,
+            study_name=self.study_name,
+            load_if_exists=True if self.storage else False,
+        )
+
+        _enqueue_predicted_r_weight_zero(study, search_space)
+
+        study.optimize(
+            lambda trial: self._objective(trial, df, search_space),
+            n_trials=self.n_trials,
+            n_jobs=self.n_jobs,
+            timeout=self.timeout,
+            show_progress_bar=self.show_progress_bar,
+        )
+
+        return OptunaResult(
+            best_params=study.best_params,
+            best_value=study.best_value,
+            best_trial=study.best_trial,
+            study=study,
+        )
+
+    def _objective(
+        self, trial: optuna.Trial, df: IntoFrameT, search_space: dict[str, ParamSpec]
+    ) -> float:
+        try:
+            trial_params = self._suggest_params(trial, search_space)
+
+            copied_estimator = copy.deepcopy(self.estimator)
+            copied_estimator.set_params(**trial_params)
+
+            cv = copy.deepcopy(self.cross_validator)
+            cv.estimator = copied_estimator
+
+            validation_df = cv.generate_validation_df(df)
+            score = self.scorer.score(validation_df)
+            score_value = RatingHyperparameterTuner._aggregate_score(score)
+
+            if math.isnan(score_value) or math.isinf(score_value):
+                logger.warning(f"Trial {trial.number} returned invalid score: {score_value}")
+                return float("inf") if self.direction == "minimize" else float("-inf")
+
+            return score_value
+
+        except Exception as e:
+            logger.warning(f"Trial {trial.number} failed with error: {e}")
+            return float("inf") if self.direction == "minimize" else float("-inf")
+
+    def _suggest_params(
+        self, trial: optuna.Trial, search_space: dict[str, ParamSpec]
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {}
+        for param_name, param_spec in search_space.items():
+            params[param_name] = param_spec.suggest(trial, param_name)
+        return params

{spforge-0.8.4 → spforge-0.8.5/spforge.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spforge
-Version: 0.8.4
+Version: 0.8.5
 Summary: A flexible framework for generating features, ratings, and building machine learning or other models for training and inference on sports data.
 Author-email: Mathias Holmstrøm <mathiasholmstom@gmail.com>
 License: See LICENSE file
@@ -17,7 +17,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy>=1.23.4
 Requires-Dist: optuna>=3.4.0
-Requires-Dist: pandas>=2.0.0
+Requires-Dist: pandas<3.0.0,>=2.0.0
 Requires-Dist: pendulum>=1.0.0
 Requires-Dist: scikit-learn>=1.4.0
 Requires-Dist: lightgbm>=4.0.0

{spforge-0.8.4 → spforge-0.8.5}/spforge.egg-info/SOURCES.txt

@@ -84,6 +84,7 @@ tests/test_autopipeline_context.py
 tests/test_feature_generator_pipeline.py
 tests/cross_validator/test_cross_validator.py
 tests/distributions/test_distribution.py
+tests/end_to_end/test_estimator_hyperparameter_tuning.py
 tests/end_to_end/test_lol_player_kills.py
 tests/end_to_end/test_nba_player_points.py
 tests/end_to_end/test_nba_player_ratings_hyperparameter_tuning.py
@@ -95,6 +96,7 @@ tests/feature_generator/test_rolling_against_opponent.py
 tests/feature_generator/test_rolling_mean_binary.py
 tests/feature_generator/test_rolling_mean_days.py
 tests/feature_generator/test_rolling_window.py
+tests/hyperparameter_tuning/test_estimator_tuner.py
 tests/hyperparameter_tuning/test_rating_tuner.py
 tests/performance_transformers/test_performance_manager.py
 tests/performance_transformers/test_performances_transformers.py

{spforge-0.8.4 → spforge-0.8.5}/spforge.egg-info/requires.txt

@@ -1,6 +1,6 @@
 numpy>=1.23.4
 optuna>=3.4.0
-pandas>=2.0.0
+pandas<3.0.0,>=2.0.0
 pendulum>=1.0.0
 scikit-learn>=1.4.0
 lightgbm>=4.0.0

spforge-0.8.5/tests/end_to_end/test_estimator_hyperparameter_tuning.py

@@ -0,0 +1,85 @@
+import polars as pl
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import mean_absolute_error
+
+from examples import get_sub_sample_nba_data
+from spforge import AutoPipeline, ColumnNames, EstimatorHyperparameterTuner, ParamSpec
+from spforge.cross_validator import MatchKFoldCrossValidator
+from spforge.scorer import SklearnScorer
+
+
+def test_nba_estimator_hyperparameter_tuning__workflow_completes():
+    df = get_sub_sample_nba_data(as_polars=True, as_pandas=False)
+    column_names = ColumnNames(
+        team_id="team_id",
+        match_id="game_id",
+        start_date="start_date",
+        player_id="player_id",
+        participation_weight="minutes_ratio",
+    )
+
+    df = df.sort(
+        [
+            column_names.start_date,
+            column_names.match_id,
+            column_names.team_id,
+            column_names.player_id,
+        ]
+    )
+
+    df = df.with_columns(
+        [
+            (pl.col("minutes") / pl.col("minutes").sum().over("game_id")).alias(
+                "minutes_ratio"
+            ),
+            (pl.col("points") > pl.lit(10)).cast(pl.Int64).alias("points_over_10"),
+        ]
+    )
+
+    estimator = AutoPipeline(
+        estimator=LogisticRegression(max_iter=200),
+        estimator_features=["minutes", "minutes_ratio"],
+    )
+
+    cv = MatchKFoldCrossValidator(
+        match_id_column_name=column_names.match_id,
+        date_column_name=column_names.start_date,
+        target_column="points_over_10",
+        estimator=estimator,
+        prediction_column_name="points_pred",
+        n_splits=2,
+        features=estimator.required_features,
+    )
+
+    scorer = SklearnScorer(
+        scorer_function=mean_absolute_error,
+        pred_column="points_pred",
+        target="points_over_10",
+        validation_column="is_validation",
+    )
+
+    tuner = EstimatorHyperparameterTuner(
+        estimator=estimator,
+        cross_validator=cv,
+        scorer=scorer,
+        direction="minimize",
+        param_search_space={
+            "C": ParamSpec(
+                param_type="float",
+                low=0.1,
+                high=2.0,
+                log=True,
+            ),
+        },
+        n_trials=3,
+        show_progress_bar=False,
+    )
+
+    result = tuner.optimize(df)
+
+    assert result.best_params is not None
+    assert isinstance(result.best_params, dict)
+    assert "estimator__C" in result.best_params
+    assert isinstance(result.best_value, float)
+    assert result.best_trial is not None
+    assert result.study is not None

spforge-0.8.5/tests/hyperparameter_tuning/test_estimator_tuner.py

@@ -0,0 +1,167 @@
+import numpy as np
+import pandas as pd
+import pytest
+from sklearn.base import BaseEstimator
+from sklearn.linear_model import LogisticRegression
+
+from spforge import EstimatorHyperparameterTuner, ParamSpec
+from spforge.cross_validator import MatchKFoldCrossValidator
+from spforge.estimator import SkLearnEnhancerEstimator
+from spforge.scorer import MeanBiasScorer
+
+
+class FakeLGBMClassifier(BaseEstimator):
+    __module__ = "lightgbm.sklearn"
+
+    def __init__(
+        self,
+        n_estimators: int = 100,
+        num_leaves: int = 31,
+        max_depth: int = 5,
+        min_child_samples: int = 20,
+        subsample: float = 1.0,
+        subsample_freq: int = 1,
+        reg_alpha: float = 0.0,
+        reg_lambda: float = 0.0,
+    ):
+        self.n_estimators = n_estimators
+        self.num_leaves = num_leaves
+        self.max_depth = max_depth
+        self.min_child_samples = min_child_samples
+        self.subsample = subsample
+        self.subsample_freq = subsample_freq
+        self.reg_alpha = reg_alpha
+        self.reg_lambda = reg_lambda
+
+    def fit(self, X, y):
+        self.classes_ = np.unique(y)
+        return self
+
+    def predict_proba(self, X):
+        n = len(X)
+        if len(self.classes_) < 2:
+            return np.ones((n, 1))
+        return np.tile([0.4, 0.6], (n, 1))
+
+    def predict(self, X):
+        n = len(X)
+        if len(self.classes_) == 1:
+            return np.full(n, self.classes_[0])
+        proba = self.predict_proba(X)
+        idx = np.argmax(proba, axis=1)
+        return np.array(self.classes_)[idx]
+
+
+@pytest.fixture
+def sample_df():
+    dates = pd.date_range("2024-01-01", periods=12, freq="D")
+    rows = []
+    for i, date in enumerate(dates):
+        rows.append(
+            {
+                "mid": f"M{i // 2}",
+                "date": date,
+                "x1": float(i),
+                "y": 1 if i % 2 == 0 else 0,
+            }
+        )
+    return pd.DataFrame(rows)
+
+
+@pytest.fixture
+def scorer():
+    return MeanBiasScorer(
+        pred_column="y_pred",
+        target="y",
+        validation_column="is_validation",
+    )
+
+
+def test_estimator_tuner_requires_search_space(sample_df, scorer):
+    estimator = LogisticRegression()
+
+    cv = MatchKFoldCrossValidator(
+        match_id_column_name="mid",
+        date_column_name="date",
+        target_column="y",
+        estimator=estimator,
+        prediction_column_name="y_pred",
+        n_splits=2,
+        features=["x1"],
+    )
+
+    tuner = EstimatorHyperparameterTuner(
+        estimator=estimator,
+        cross_validator=cv,
+        scorer=scorer,
+        direction="minimize",
+        n_trials=2,
+        show_progress_bar=False,
+    )
+
+    with pytest.raises(ValueError, match="param_search_space is required"):
+        tuner.optimize(sample_df)
+
+
+def test_estimator_tuner_custom_search_space(sample_df, scorer):
+    estimator = SkLearnEnhancerEstimator(estimator=LogisticRegression())
+
+    cv = MatchKFoldCrossValidator(
+        match_id_column_name="mid",
+        date_column_name="date",
+        target_column="y",
+        estimator=estimator,
+        prediction_column_name="y_pred",
+        n_splits=2,
+        features=["x1"],
+    )
+
+    tuner = EstimatorHyperparameterTuner(
+        estimator=estimator,
+        cross_validator=cv,
+        scorer=scorer,
+        direction="minimize",
+        param_search_space={
+            "C": ParamSpec(
+                param_type="float",
+                low=0.1,
+                high=2.0,
+                log=True,
+            )
+        },
+        n_trials=2,
+        show_progress_bar=False,
+    )
+
+    result = tuner.optimize(sample_df)
+
+    assert "estimator__C" in result.best_params
+    assert isinstance(result.best_value, float)
+
+
+def test_estimator_tuner_lgbm_defaults(sample_df, scorer):
+    estimator = FakeLGBMClassifier()
+
+    cv = MatchKFoldCrossValidator(
+        match_id_column_name="mid",
+        date_column_name="date",
+        target_column="y",
+        estimator=estimator,
+        prediction_column_name="y_pred",
+        n_splits=2,
+        features=["x1"],
+    )
+
+    tuner = EstimatorHyperparameterTuner(
+        estimator=estimator,
+        cross_validator=cv,
+        scorer=scorer,
+        direction="minimize",
+        n_trials=2,
+        show_progress_bar=False,
+    )
+
+    result = tuner.optimize(sample_df)
+
+    assert "n_estimators" in result.best_params
+    assert isinstance(result.best_value, float)