wavetrainer 0.1.11__tar.gz → 0.1.13__tar.gz

{wavetrainer-0.1.11/wavetrainer.egg-info → wavetrainer-0.1.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wavetrainer
-Version: 0.1.11
+Version: 0.1.13
 Summary: A library for automatically finding the optimal model within feature and hyperparameter space.
 Home-page: https://github.com/8W9aG/wavetrainer
 Author: Will Sackfield
@@ -30,6 +30,7 @@ Requires-Dist: tabpfn_extensions>=0.0.4
 Requires-Dist: hyperopt>=0.2.7
 Requires-Dist: pycaleva>=0.8.2
 Requires-Dist: lightgbm>=4.6.0
+Requires-Dist: kaleido>=0.2.1
 
 # wavetrainer
 
@@ -66,6 +67,7 @@ Python 3.11.6:
 - [hyperopt](https://github.com/hyperopt/hyperopt)
 - [pycaleva](https://github.com/MartinWeigl/pycaleva)
 - [lightgbm](https://github.com/microsoft/LightGBM)
+- [kaleido](https://github.com/plotly/Kaleido)
 
 ## Raison D'être :thought_balloon:
 

{wavetrainer-0.1.11 → wavetrainer-0.1.13}/README.md

@@ -33,6 +33,7 @@ Python 3.11.6:
 - [hyperopt](https://github.com/hyperopt/hyperopt)
 - [pycaleva](https://github.com/MartinWeigl/pycaleva)
 - [lightgbm](https://github.com/microsoft/LightGBM)
+- [kaleido](https://github.com/plotly/Kaleido)
 
 ## Raison D'être :thought_balloon:
 

wavetrainer-0.1.11/wavetrainer.egg-info/requires.txt → wavetrainer-0.1.13/requirements.txt

@@ -17,3 +17,4 @@ tabpfn_extensions>=0.0.4
 hyperopt>=0.2.7
 pycaleva>=0.8.2
 lightgbm>=4.6.0
+kaleido>=0.2.1

{wavetrainer-0.1.11 → wavetrainer-0.1.13}/setup.py

@@ -23,7 +23,7 @@ def install_requires() -> typing.List[str]:
 
 setup(
     name='wavetrainer',
-    version='0.1.11',
+    version='0.1.13',
     description='A library for automatically finding the optimal model within feature and hyperparameter space.',
     long_description=long_description,
     long_description_content_type='text/markdown',

{wavetrainer-0.1.11 → wavetrainer-0.1.13}/wavetrainer/__init__.py

@@ -2,5 +2,5 @@
 
 from .create import create
 
-__VERSION__ = "0.1.11"
+__VERSION__ = "0.1.13"
 __all__ = ("create",)

{wavetrainer-0.1.11 → wavetrainer-0.1.13}/wavetrainer/model/lightgbm/lightgbm_model.py

@@ -148,7 +148,7 @@ class LightGBMModel(Model):
 
         eval_set = None
         callbacks = []
-        if eval_x is None or eval_y is None:
+        if eval_x is not None and eval_y is not None:
             eval_set = [(eval_x, eval_y.to_numpy().flatten())]  # type: ignore
             callbacks = [
                 lgb.early_stopping(stopping_rounds=early_stopping_rounds),

{wavetrainer-0.1.11 → wavetrainer-0.1.13}/wavetrainer/model/model_router.py

@@ -35,13 +35,18 @@ class ModelRouter(Model):
     _model: Model | None
     _false_positive_reduction_steps: int | None
 
-    def __init__(self, allowed_models: set[str] | None) -> None:
+    def __init__(
+        self,
+        allowed_models: set[str] | None,
+        max_false_positive_reduction_steps: int | None,
+    ) -> None:
         super().__init__()
         self._model = None
         self._false_positive_reduction_steps = None
         self._allowed_models = (
             allowed_models if allowed_models is not None else set(_MODELS.keys())
         )
+        self._max_false_positive_reduction_steps = max_false_positive_reduction_steps
 
     @classmethod
     def name(cls) -> str:
@@ -93,7 +98,11 @@ class ModelRouter(Model):
         self, trial: optuna.Trial | optuna.trial.FrozenTrial, df: pd.DataFrame
     ) -> None:
         self._false_positive_reduction_steps = trial.suggest_int(
-            _FALSE_POSITIVE_REDUCTION_STEPS_KEY, 0, 5
+            _FALSE_POSITIVE_REDUCTION_STEPS_KEY,
+            0,
+            5
+            if self._max_false_positive_reduction_steps is None
+            else self._max_false_positive_reduction_steps,
         )
         model_name = trial.suggest_categorical(
             "model",

{wavetrainer-0.1.11 → wavetrainer-0.1.13}/wavetrainer/reducer/combined_reducer.py

@@ -2,7 +2,6 @@
 
 # pylint: disable=line-too-long
 import json
-import logging
 import os
 import time
 from typing import Self
@@ -129,6 +128,6 @@ class CombinedReducer(Reducer):
             try:
                 df = reducer.transform(df)
             except ValueError as exc:
-                logging.warning("Failed to reduce %s", reducer.name())
+                print("Failed to reduce %s", reducer.name())
                 raise exc
         return df

{wavetrainer-0.1.11 → wavetrainer-0.1.13}/wavetrainer/trainer.py

@@ -1,5 +1,6 @@
 """The trainer class."""
 
+# pylint: disable=line-too-long
 import datetime
 import functools
 import json
@@ -12,12 +13,14 @@ from typing import Self
 import optuna
 import pandas as pd
 import tqdm
-from sklearn.metrics import f1_score, r2_score  # type: ignore
+from sklearn.metrics import f1_score  # type: ignore
+from sklearn.metrics import (accuracy_score, brier_score_loss, log_loss,
+                             precision_score, r2_score, recall_score)
 
 from .calibrator.calibrator_router import CalibratorRouter
 from .exceptions import WavetrainException
 from .fit import Fit
-from .model.model import PREDICTION_COLUMN
+from .model.model import PREDICTION_COLUMN, PROBABILITY_COLUMN_PREFIX
 from .model.model_router import ModelRouter
 from .model_type import ModelType, determine_model_type
 from .reducer.combined_reducer import CombinedReducer
@@ -38,6 +41,7 @@ _TEST_SIZE_KEY = "test_size"
 _VALIDATION_SIZE_KEY = "validation_size"
 _IDX_USR_ATTR_KEY = "idx"
 _DT_COLUMN_KEY = "dt_column"
+_MAX_FALSE_POSITIVE_REDUCTION_STEPS_KEY = "max_false_positive_reduction_steps"
 _BAD_OUTPUT = -1000.0
 
 
@@ -48,6 +52,11 @@ def _assign_bin(timestamp, bins: list[datetime.datetime]) -> int:
     return len(bins) - 2  # Assign to last bin if at the end
 
 
+def _best_trial(study: optuna.Study) -> optuna.trial.FrozenTrial:
+    best_brier = min(study.best_trials, key=lambda t: t.values[1])
+    return best_brier
+
+
 class Trainer(Fit):
     """A class for training and predicting from an array of data."""
 
@@ -65,6 +74,7 @@ class Trainer(Fit):
         cutoff_dt: datetime.datetime | None = None,
         embedding_cols: list[list[str]] | None = None,
         allowed_models: set[str] | None = None,
+        max_false_positive_reduction_steps: int | None = None,
     ):
         tqdm.tqdm.pandas()
 
@@ -115,6 +125,10 @@ class Trainer(Fit):
                         )
                 if dt_column is None:
                     dt_column = params[_DT_COLUMN_KEY]
+                if max_false_positive_reduction_steps is None:
+                    max_false_positive_reduction_steps = params.get(
+                        _MAX_FALSE_POSITIVE_REDUCTION_STEPS_KEY
+                    )
         else:
             with open(params_file, "w", encoding="utf8") as handle:
                 validation_size_value = None
@@ -145,6 +159,7 @@ class Trainer(Fit):
                         _TEST_SIZE_KEY: test_size_value,
                         _VALIDATION_SIZE_KEY: validation_size_value,
                         _DT_COLUMN_KEY: dt_column,
+                        _MAX_FALSE_POSITIVE_REDUCTION_STEPS_KEY: max_false_positive_reduction_steps,
                     },
                     handle,
                 )
@@ -157,6 +172,7 @@ class Trainer(Fit):
         self._cutoff_dt = cutoff_dt
         self.embedding_cols = embedding_cols
         self._allowed_models = allowed_models
+        self._max_false_positive_reduction_steps = max_false_positive_reduction_steps
 
     def _provide_study(self, column: str) -> optuna.Study:
         storage_name = f"sqlite:///{self._folder}/{column}/{_STUDYDB_FILENAME}"
@@ -170,7 +186,10 @@ class Trainer(Fit):
             storage=storage_name,
             load_if_exists=True,
             sampler=restored_sampler,
-            direction=optuna.study.StudyDirection.MAXIMIZE,
+            directions=[
+                optuna.study.StudyDirection.MAXIMIZE,
+                optuna.study.StudyDirection.MINIMIZE,
+            ],
         )
 
     def fit(
@@ -210,7 +229,7 @@ class Trainer(Fit):
                 save: bool,
                 split_idx: datetime.datetime,
                 no_evaluation: bool,
-            ) -> float:
+            ) -> tuple[float, float]:
                 print(f"Beginning trial for: {split_idx.isoformat()}")
                 trial.set_user_attr(_IDX_USR_ATTR_KEY, split_idx.isoformat())
                 folder = os.path.join(
@@ -246,7 +265,7 @@ class Trainer(Fit):
                         if new_folder:
                             os.removedirs(folder)
                         logging.warning("Y train only contains 1 unique datapoint.")
-                        return _BAD_OUTPUT
+                        return _BAD_OUTPUT, -_BAD_OUTPUT
                     print(f"Windowing took {time.time() - start_windower}")
 
                     # Perform common reductions
@@ -267,7 +286,9 @@ class Trainer(Fit):
                     print(f"Row weights took {time.time() - start_row_weights}")
 
                     # Create model
-                    model = ModelRouter(self._allowed_models)
+                    model = ModelRouter(
+                        self._allowed_models, self._max_false_positive_reduction_steps
+                    )
                     model.set_options(trial, x)
 
                     # Train
@@ -311,10 +332,29 @@ class Trainer(Fit):
                     )
                     cal_pred[PREDICTION_COLUMN] = y_pred[PREDICTION_COLUMN]
                     output = 0.0
+                    loss = 0.0
                     if determine_model_type(y_series) == ModelType.REGRESSION:
                         output = float(r2_score(y_test, y_pred[[PREDICTION_COLUMN]]))
+                        print(f"R2: {output}")
                     else:
                         output = float(f1_score(y_test, y_pred[[PREDICTION_COLUMN]]))
+                        print(f"F1: {output}")
+                        prob_col = PROBABILITY_COLUMN_PREFIX + str(1)
+                        if prob_col in y_pred.columns.values.tolist():
+                            loss = float(brier_score_loss(y_test, y_pred[[prob_col]]))
+                            print(f"Brier: {loss}")
+                            print(
+                                f"Log Loss: {float(log_loss(y_test.astype(float), y_pred[[prob_col]]))}"
+                            )
+                        print(
+                            f"Accuracy: {float(accuracy_score(y_test, y_pred[[PREDICTION_COLUMN]]))}"
+                        )
+                        print(
+                            f"Precision: {float(precision_score(y_test, y_pred[[PREDICTION_COLUMN]]))}"
+                        )
+                        print(
+                            f"Recall: {float(recall_score(y_test, y_pred[[PREDICTION_COLUMN]]))}"
+                        )
 
                     if save:
                         windower.save(folder, trial)
@@ -332,13 +372,13 @@ class Trainer(Fit):
                                 handle,
                             )
 
-                    return output
+                    return output, loss
                 except WavetrainException as exc:
                     print(str(exc))
                     logging.warning(str(exc))
                     if new_folder:
                         os.removedirs(folder)
-                    return _BAD_OUTPUT
+                    return _BAD_OUTPUT, -_BAD_OUTPUT
 
             start_validation_index = (
                 dt_index.to_list()[-int(len(dt_index) * self._validation_size) - 1]
@@ -359,7 +399,7 @@ class Trainer(Fit):
                 ].to_list()[0]
             )
 
-            def test_objective(trial: optuna.Trial) -> float:
+            def test_objective(trial: optuna.Trial) -> tuple[float, float]:
                 return _fit(
                     trial,
                     test_df,
@@ -382,7 +422,8 @@ class Trainer(Fit):
                     else self._max_train_timeout.total_seconds(),
                 )
             while (
-                study.best_trial.value is None or study.best_trial.value == _BAD_OUTPUT
+                _best_trial(study).values is None
+                or _best_trial(study).values == (_BAD_OUTPUT, -_BAD_OUTPUT)
             ) and len(study.trials) < 1000:
                 logging.info("Performing extra train")
                 study.optimize(
@@ -420,7 +461,7 @@ class Trainer(Fit):
                 if found:
                     last_processed_dt = test_dt
                     _fit(
-                        study.best_trial,
+                        _best_trial(study),
                         test_df.copy(),
                         test_series,
                         True,
@@ -441,7 +482,7 @@ class Trainer(Fit):
 
                     def validate_objctive(
                         trial: optuna.Trial, idx: datetime.datetime, series: pd.Series
-                    ) -> float:
+                    ) -> tuple[float, float]:
                         return _fit(trial, test_df.copy(), series, False, idx, False)
 
                     study.optimize(
@@ -457,10 +498,36 @@ class Trainer(Fit):
                     break
 
                 _fit(
-                    study.best_trial, test_df.copy(), test_series, True, test_idx, True
+                    _best_trial(study),
+                    test_df.copy(),
+                    test_series,
+                    True,
+                    test_idx,
+                    True,
                 )
                 last_processed_dt = test_idx
 
+            target_names = ["F1", "Brier"]
+            fig = optuna.visualization.plot_pareto_front(
+                study, target_names=target_names
+            )
+            fig.write_image(
+                os.path.join(column_dir, "pareto_frontier.png"),
+                format="png",
+                width=800,
+                height=600,
+            )
+            for target_name in target_names:
+                fig = optuna.visualization.plot_param_importances(
+                    study, target=lambda t: t.values[0], target_name=target_name
+                )
+                fig.write_image(
+                    os.path.join(column_dir, f"{target_name}_frontier.png"),
+                    format="png",
+                    width=800,
+                    height=600,
+                )
+
         if isinstance(y, pd.Series):
             _fit_column(y)
         else:
@@ -519,7 +586,7 @@ class Trainer(Fit):
                 reducer = CombinedReducer(self.embedding_cols)
                 reducer.load(folder)
 
-                model = ModelRouter(None)
+                model = ModelRouter(None, None)
                 model.load(folder)
 
                 selector = Selector(model)
@@ -572,7 +639,7 @@ class Trainer(Fit):
                 if not os.path.isdir(date_path):
                     continue
                 try:
-                    model = ModelRouter(None)
+                    model = ModelRouter(None, None)
                     model.load(date_path)
                     feature_importances[date_str] = model.feature_importances
                 except FileNotFoundError as exc:

{wavetrainer-0.1.11 → wavetrainer-0.1.13/wavetrainer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wavetrainer
-Version: 0.1.11
+Version: 0.1.13
 Summary: A library for automatically finding the optimal model within feature and hyperparameter space.
 Home-page: https://github.com/8W9aG/wavetrainer
 Author: Will Sackfield
@@ -30,6 +30,7 @@ Requires-Dist: tabpfn_extensions>=0.0.4
 Requires-Dist: hyperopt>=0.2.7
 Requires-Dist: pycaleva>=0.8.2
 Requires-Dist: lightgbm>=4.6.0
+Requires-Dist: kaleido>=0.2.1
 
 # wavetrainer
 
@@ -66,6 +67,7 @@ Python 3.11.6:
 - [hyperopt](https://github.com/hyperopt/hyperopt)
 - [pycaleva](https://github.com/MartinWeigl/pycaleva)
 - [lightgbm](https://github.com/microsoft/LightGBM)
+- [kaleido](https://github.com/plotly/Kaleido)
 
 ## Raison D'être :thought_balloon:
 

wavetrainer-0.1.11/requirements.txt → wavetrainer-0.1.13/wavetrainer.egg-info/requires.txt

@@ -16,4 +16,5 @@ jax>=0.6.1
 tabpfn_extensions>=0.0.4
 hyperopt>=0.2.7
 pycaleva>=0.8.2
-lightgbm>=4.6.0
+lightgbm>=4.6.0
+kaleido>=0.2.1