BayesHalvingSearchCV 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bayeshalvingsearchcv-0.1.0/LICENSE +29 -0
- bayeshalvingsearchcv-0.1.0/PKG-INFO +141 -0
- bayeshalvingsearchcv-0.1.0/README.md +109 -0
- bayeshalvingsearchcv-0.1.0/pyproject.toml +53 -0
- bayeshalvingsearchcv-0.1.0/setup.cfg +4 -0
- bayeshalvingsearchcv-0.1.0/src/BayesHalvingSearchCV.egg-info/PKG-INFO +141 -0
- bayeshalvingsearchcv-0.1.0/src/BayesHalvingSearchCV.egg-info/SOURCES.txt +26 -0
- bayeshalvingsearchcv-0.1.0/src/BayesHalvingSearchCV.egg-info/dependency_links.txt +1 -0
- bayeshalvingsearchcv-0.1.0/src/BayesHalvingSearchCV.egg-info/requires.txt +11 -0
- bayeshalvingsearchcv-0.1.0/src/BayesHalvingSearchCV.egg-info/top_level.txt +1 -0
- bayeshalvingsearchcv-0.1.0/src/bayes_halving_search_cv/__init__.py +15 -0
- bayeshalvingsearchcv-0.1.0/src/bayes_halving_search_cv/_bayes.py +509 -0
- bayeshalvingsearchcv-0.1.0/src/bayes_halving_search_cv/_climber.py +307 -0
- bayeshalvingsearchcv-0.1.0/src/bayes_halving_search_cv/_engine.py +113 -0
- bayeshalvingsearchcv-0.1.0/src/bayes_halving_search_cv/_fidelity.py +81 -0
- bayeshalvingsearchcv-0.1.0/src/bayes_halving_search_cv/_gp.py +185 -0
- bayeshalvingsearchcv-0.1.0/src/bayes_halving_search_cv/_sampling.py +155 -0
- bayeshalvingsearchcv-0.1.0/src/bayes_halving_search_cv/_search.py +465 -0
- bayeshalvingsearchcv-0.1.0/src/bayes_halving_search_cv/_space.py +157 -0
- bayeshalvingsearchcv-0.1.0/src/bayes_halving_search_cv/_starts.py +54 -0
- bayeshalvingsearchcv-0.1.0/tests/test_bayes.py +175 -0
- bayeshalvingsearchcv-0.1.0/tests/test_climber.py +218 -0
- bayeshalvingsearchcv-0.1.0/tests/test_estimator.py +235 -0
- bayeshalvingsearchcv-0.1.0/tests/test_fidelity.py +76 -0
- bayeshalvingsearchcv-0.1.0/tests/test_gp.py +96 -0
- bayeshalvingsearchcv-0.1.0/tests/test_sampling.py +78 -0
- bayeshalvingsearchcv-0.1.0/tests/test_sklearn_compat.py +25 -0
- bayeshalvingsearchcv-0.1.0/tests/test_starts.py +56 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Rodrigo Pinto Coelho
|
|
4
|
+
All rights reserved.
|
|
5
|
+
|
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
|
8
|
+
|
|
9
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
10
|
+
list of conditions and the following disclaimer.
|
|
11
|
+
|
|
12
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
|
14
|
+
and/or other materials provided with the distribution.
|
|
15
|
+
|
|
16
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
17
|
+
contributors may be used to endorse or promote products derived from
|
|
18
|
+
this software without specific prior written permission.
|
|
19
|
+
|
|
20
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
21
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
22
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
23
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
24
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
25
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
26
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
27
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
28
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: BayesHalvingSearchCV
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: scikit-learn hyperparameter search with multi-fidelity data growth and scatter-search multi-start: a from-scratch Bayesian (GP + Expected Improvement) search, BayesHalvingSearchCV, plus a Hooke-Jeeves pattern search, PatternSearchCV
|
|
5
|
+
Author: Rodrigo Pinto Coelho
|
|
6
|
+
License-Expression: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://github.com/rpcoelho17/PatternSearch_HyperParam_Opt__V2
|
|
8
|
+
Project-URL: Repository, https://github.com/rpcoelho17/PatternSearch_HyperParam_Opt__V2
|
|
9
|
+
Keywords: scikit-learn,hyperparameter-optimization,bayesian-optimization,pattern-search,multi-fidelity,gaussian-process,cross-validation,model-selection
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: numpy>=1.24
|
|
23
|
+
Requires-Dist: scipy>=1.10
|
|
24
|
+
Requires-Dist: scikit-learn<1.10,>=1.6
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: pytest; extra == "test"
|
|
27
|
+
Provides-Extra: docs
|
|
28
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
29
|
+
Requires-Dist: numpydoc; extra == "docs"
|
|
30
|
+
Requires-Dist: pydata-sphinx-theme; extra == "docs"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# BayesHalvingSearchCV & PatternSearchCV
|
|
34
|
+
|
|
35
|
+
During my post-graduate studies in Data Science at USI Switzerland, I had to hyperparameter-tune
|
|
36
|
+
models on genuinely large datasets, and I went looking for a way to find good
|
|
37
|
+
hyperparameters quickly instead of paying full price for every fit.
|
|
38
|
+
Inspired by MATLAB's Global Optimization Toolbox pattern search for discrete
|
|
39
|
+
problems, I built a first version of what became `PatternSearchCV` — and even
|
|
40
|
+
that first version already beat Bayesian search on real benchmarks, simply
|
|
41
|
+
because it deduplicated repeated evaluations, something the Bayesian search
|
|
42
|
+
tools I compared it against didn't do; that alone saved real fits on large
|
|
43
|
+
datasets. I later added a "halving" mechanism on top of that base pattern
|
|
44
|
+
search — growing how much data each evaluation sees as the search's own
|
|
45
|
+
trajectory shows it converging (measured by the shrinking distance between
|
|
46
|
+
successive improving moves), instead of committing to full data from the
|
|
47
|
+
start — and that further improved the gains over the deduplication-only
|
|
48
|
+
version. When I eventually benchmarked properly against Optuna, the picture
|
|
49
|
+
became precise: Optuna's Bayesian search found the optimum in fewer raw
|
|
50
|
+
trials, but every one of those trials evaluated 100% of the data — my
|
|
51
|
+
advantage was coming from deduplication, the multi-fidelity data growth, and
|
|
52
|
+
the halving logic together, not from having fewer trials. So I adapted that
|
|
53
|
+
same multi-fidelity infrastructure onto a from-scratch Bayesian search of my
|
|
54
|
+
own, giving `BayesHalvingSearchCV` Optuna-quality trial efficiency without
|
|
55
|
+
paying full price for every trial.
|
|
56
|
+
|
|
57
|
+
**Both estimators implement scikit-learn's standard search-CV interface and
|
|
58
|
+
work with any scikit-learn-compatible estimator** — they drop into existing
|
|
59
|
+
pipelines exactly the way `GridSearchCV` or `RandomizedSearchCV` do. Our own
|
|
60
|
+
benchmarks use `ExtraTreesRegressor` specifically because it's fast and
|
|
61
|
+
performs well on the dataset we tested against, not because either algorithm
|
|
62
|
+
is limited to it.
|
|
63
|
+
|
|
64
|
+
**Best measured results** (523K-row retail regression benchmark; full
|
|
65
|
+
methodology and every run logged in [`EXPERIMENTS.md`](EXPERIMENTS.md)):
|
|
66
|
+
`BayesHalvingSearchCV` reached the benchmark's historical optimum at
|
|
67
|
+
**3.125 full-fit equivalents** (Experiment 17); `PatternSearchCV` reached the
|
|
68
|
+
same optimum at **5.04 full-fit equivalents** (Experiment 11). For
|
|
69
|
+
comparison, Optuna's own GP-based sampler needed **15.00 full-fit
|
|
70
|
+
equivalents** — every one of its trials at 100% of the data — to reach a
|
|
71
|
+
comparable answer. That's a **4.8× reduction** in compute for
|
|
72
|
+
`BayesHalvingSearchCV` and a **2.98× reduction** for `PatternSearchCV`,
|
|
73
|
+
relative to Optuna, for the same answer.
|
|
74
|
+
|
|
75
|
+
GPU acceleration is available the same way it is for any scikit-learn
|
|
76
|
+
estimator: enable it in `estimator`'s own settings. See
|
|
77
|
+
[`API_REFERENCE.md`](API_REFERENCE.md) for the full explanation.
|
|
78
|
+
|
|
79
|
+
Both estimators share one **multi-fidelity "bullseye" data-growth mechanism**
|
|
80
|
+
and one **scatter-search multi-start layer**:
|
|
81
|
+
|
|
82
|
+
- `PatternSearchCV` — classic Hooke-Jeeves pattern search (1961), adapted to
|
|
83
|
+
grow its data budget as it converges.
|
|
84
|
+
- `BayesHalvingSearchCV` — a from-scratch Gaussian Process + Expected
|
|
85
|
+
Improvement Bayesian search, on the exact same multi-fidelity
|
|
86
|
+
infrastructure, with **zero additional dependencies** (no Optuna, no
|
|
87
|
+
torch — just `numpy`, `scipy`, `scikit-learn`, already required by
|
|
88
|
+
`PatternSearchCV`).
|
|
89
|
+
|
|
90
|
+
Both estimators start every search on a small, representative subsample of
|
|
91
|
+
the training data and only pay full price once their own search trajectory
|
|
92
|
+
shows them converging on an optimum — every reported result is still
|
|
93
|
+
confirmed on 100% of the data before it's trusted.
|
|
94
|
+
|
|
95
|
+
## Quick start
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from bayes_halving_search_cv import PatternSearchCV, BayesHalvingSearchCV
|
|
99
|
+
from sklearn.model_selection import TimeSeriesSplit
|
|
100
|
+
|
|
101
|
+
param_grid = {"max_depth": [3, 5, 7, 9, 12, 16], "min_samples_leaf": [1, 2, 4, 8]}
|
|
102
|
+
|
|
103
|
+
search = PatternSearchCV(
|
|
104
|
+
estimator, param_grid,
|
|
105
|
+
cv=TimeSeriesSplit(n_splits=5),
|
|
106
|
+
scoring="neg_mean_absolute_error",
|
|
107
|
+
n_starts=4, # scatter-search multi-start
|
|
108
|
+
subsample="stratified", # transition sampling for time-series data
|
|
109
|
+
random_state=0,
|
|
110
|
+
)
|
|
111
|
+
search.fit(X, y)
|
|
112
|
+
search.best_params_ # chosen ONLY from full-data evaluations
|
|
113
|
+
search.local_optima_ # the map: every distinct optimum found
|
|
114
|
+
search.cv_results_ # every point evaluated, and its score
|
|
115
|
+
search.search_history_ # every confirmed-improving move across every start
|
|
116
|
+
|
|
117
|
+
# or the Bayesian search, on the exact same multi-fidelity infrastructure:
|
|
118
|
+
search = BayesHalvingSearchCV(estimator, param_grid, cv=TimeSeriesSplit(5),
|
|
119
|
+
scoring="neg_mean_absolute_error", random_state=0)
|
|
120
|
+
search.fit(X, y)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
For the full parameter reference and worked examples — how to specify a
|
|
124
|
+
search space, the data ladder, `contraction="eager"`'s cost/risk trade-off,
|
|
125
|
+
and why `subsample="stratified"` matters for time series — see
|
|
126
|
+
[`API_REFERENCE.md`](API_REFERENCE.md). For the full design rationale behind
|
|
127
|
+
every default, see [`PatternSearchCV_SPEC.md`](PatternSearchCV_SPEC.md) and
|
|
128
|
+
[`BAYESHALVINGSearchCV_SPEC.md`](BAYESHALVINGSearchCV_SPEC.md).
|
|
129
|
+
|
|
130
|
+
## Development
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
python -m venv .venv
|
|
134
|
+
.venv/Scripts/pip install -e .[test]
|
|
135
|
+
.venv/Scripts/python -m pytest
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Logging: the package logs every algorithmic decision (moves, contractions,
|
|
139
|
+
ring calibrations and crossings, data climbs, merges, cache statistics) to the
|
|
140
|
+
`SearchCV` logger. `verbose=1` attaches a stream handler at INFO,
|
|
141
|
+
`verbose=2` at DEBUG.
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# BayesHalvingSearchCV & PatternSearchCV
|
|
2
|
+
|
|
3
|
+
During my post-graduate studies in Data Science at USI Switzerland, I had to hyperparameter-tune
|
|
4
|
+
models on genuinely large datasets, and I went looking for a way to find good
|
|
5
|
+
hyperparameters quickly instead of paying full price for every fit.
|
|
6
|
+
Inspired by MATLAB's Global Optimization Toolbox pattern search for discrete
|
|
7
|
+
problems, I built a first version of what became `PatternSearchCV` — and even
|
|
8
|
+
that first version already beat Bayesian search on real benchmarks, simply
|
|
9
|
+
because it deduplicated repeated evaluations, something the Bayesian search
|
|
10
|
+
tools I compared it against didn't do; that alone saved real fits on large
|
|
11
|
+
datasets. I later added a "halving" mechanism on top of that base pattern
|
|
12
|
+
search — growing how much data each evaluation sees as the search's own
|
|
13
|
+
trajectory shows it converging (measured by the shrinking distance between
|
|
14
|
+
successive improving moves), instead of committing to full data from the
|
|
15
|
+
start — and that further improved the gains over the deduplication-only
|
|
16
|
+
version. When I eventually benchmarked properly against Optuna, the picture
|
|
17
|
+
became precise: Optuna's Bayesian search found the optimum in fewer raw
|
|
18
|
+
trials, but every one of those trials evaluated 100% of the data — my
|
|
19
|
+
advantage was coming from deduplication, the multi-fidelity data growth, and
|
|
20
|
+
the halving logic together, not from having fewer trials. So I adapted that
|
|
21
|
+
same multi-fidelity infrastructure onto a from-scratch Bayesian search of my
|
|
22
|
+
own, giving `BayesHalvingSearchCV` Optuna-quality trial efficiency without
|
|
23
|
+
paying full price for every trial.
|
|
24
|
+
|
|
25
|
+
**Both estimators implement scikit-learn's standard search-CV interface and
|
|
26
|
+
work with any scikit-learn-compatible estimator** — they drop into existing
|
|
27
|
+
pipelines exactly the way `GridSearchCV` or `RandomizedSearchCV` do. Our own
|
|
28
|
+
benchmarks use `ExtraTreesRegressor` specifically because it's fast and
|
|
29
|
+
performs well on the dataset we tested against, not because either algorithm
|
|
30
|
+
is limited to it.
|
|
31
|
+
|
|
32
|
+
**Best measured results** (523K-row retail regression benchmark; full
|
|
33
|
+
methodology and every run logged in [`EXPERIMENTS.md`](EXPERIMENTS.md)):
|
|
34
|
+
`BayesHalvingSearchCV` reached the benchmark's historical optimum at
|
|
35
|
+
**3.125 full-fit equivalents** (Experiment 17); `PatternSearchCV` reached the
|
|
36
|
+
same optimum at **5.04 full-fit equivalents** (Experiment 11). For
|
|
37
|
+
comparison, Optuna's own GP-based sampler needed **15.00 full-fit
|
|
38
|
+
equivalents** — every one of its trials at 100% of the data — to reach a
|
|
39
|
+
comparable answer. That's a **4.8× reduction** in compute for
|
|
40
|
+
`BayesHalvingSearchCV` and a **2.98× reduction** for `PatternSearchCV`,
|
|
41
|
+
relative to Optuna, for the same answer.
|
|
42
|
+
|
|
43
|
+
GPU acceleration is available the same way it is for any scikit-learn
|
|
44
|
+
estimator: enable it in `estimator`'s own settings. See
|
|
45
|
+
[`API_REFERENCE.md`](API_REFERENCE.md) for the full explanation.
|
|
46
|
+
|
|
47
|
+
Both estimators share one **multi-fidelity "bullseye" data-growth mechanism**
|
|
48
|
+
and one **scatter-search multi-start layer**:
|
|
49
|
+
|
|
50
|
+
- `PatternSearchCV` — classic Hooke-Jeeves pattern search (1961), adapted to
|
|
51
|
+
grow its data budget as it converges.
|
|
52
|
+
- `BayesHalvingSearchCV` — a from-scratch Gaussian Process + Expected
|
|
53
|
+
Improvement Bayesian search, on the exact same multi-fidelity
|
|
54
|
+
infrastructure, with **zero additional dependencies** (no Optuna, no
|
|
55
|
+
torch — just `numpy`, `scipy`, `scikit-learn`, already required by
|
|
56
|
+
`PatternSearchCV`).
|
|
57
|
+
|
|
58
|
+
Both estimators start every search on a small, representative subsample of
|
|
59
|
+
the training data and only pay full price once their own search trajectory
|
|
60
|
+
shows them converging on an optimum — every reported result is still
|
|
61
|
+
confirmed on 100% of the data before it's trusted.
|
|
62
|
+
|
|
63
|
+
## Quick start
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from bayes_halving_search_cv import PatternSearchCV, BayesHalvingSearchCV
|
|
67
|
+
from sklearn.model_selection import TimeSeriesSplit
|
|
68
|
+
|
|
69
|
+
param_grid = {"max_depth": [3, 5, 7, 9, 12, 16], "min_samples_leaf": [1, 2, 4, 8]}
|
|
70
|
+
|
|
71
|
+
search = PatternSearchCV(
|
|
72
|
+
estimator, param_grid,
|
|
73
|
+
cv=TimeSeriesSplit(n_splits=5),
|
|
74
|
+
scoring="neg_mean_absolute_error",
|
|
75
|
+
n_starts=4, # scatter-search multi-start
|
|
76
|
+
subsample="stratified", # transition sampling for time-series data
|
|
77
|
+
random_state=0,
|
|
78
|
+
)
|
|
79
|
+
search.fit(X, y)
|
|
80
|
+
search.best_params_ # chosen ONLY from full-data evaluations
|
|
81
|
+
search.local_optima_ # the map: every distinct optimum found
|
|
82
|
+
search.cv_results_ # every point evaluated, and its score
|
|
83
|
+
search.search_history_ # every confirmed-improving move across every start
|
|
84
|
+
|
|
85
|
+
# or the Bayesian search, on the exact same multi-fidelity infrastructure:
|
|
86
|
+
search = BayesHalvingSearchCV(estimator, param_grid, cv=TimeSeriesSplit(5),
|
|
87
|
+
scoring="neg_mean_absolute_error", random_state=0)
|
|
88
|
+
search.fit(X, y)
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
For the full parameter reference and worked examples — how to specify a
|
|
92
|
+
search space, the data ladder, `contraction="eager"`'s cost/risk trade-off,
|
|
93
|
+
and why `subsample="stratified"` matters for time series — see
|
|
94
|
+
[`API_REFERENCE.md`](API_REFERENCE.md). For the full design rationale behind
|
|
95
|
+
every default, see [`PatternSearchCV_SPEC.md`](PatternSearchCV_SPEC.md) and
|
|
96
|
+
[`BAYESHALVINGSearchCV_SPEC.md`](BAYESHALVINGSearchCV_SPEC.md).
|
|
97
|
+
|
|
98
|
+
## Development
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
python -m venv .venv
|
|
102
|
+
.venv/Scripts/pip install -e .[test]
|
|
103
|
+
.venv/Scripts/python -m pytest
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Logging: the package logs every algorithmic decision (moves, contractions,
|
|
107
|
+
ring calibrations and crossings, data climbs, merges, cache statistics) to the
|
|
108
|
+
`SearchCV` logger. `verbose=1` attaches a stream handler at INFO,
|
|
109
|
+
`verbose=2` at DEBUG.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "BayesHalvingSearchCV"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "scikit-learn hyperparameter search with multi-fidelity data growth and scatter-search multi-start: a from-scratch Bayesian (GP + Expected Improvement) search, BayesHalvingSearchCV, plus a Hooke-Jeeves pattern search, PatternSearchCV"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "BSD-3-Clause"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Rodrigo Pinto Coelho" }]
|
|
14
|
+
keywords = [
|
|
15
|
+
"scikit-learn",
|
|
16
|
+
"hyperparameter-optimization",
|
|
17
|
+
"bayesian-optimization",
|
|
18
|
+
"pattern-search",
|
|
19
|
+
"multi-fidelity",
|
|
20
|
+
"gaussian-process",
|
|
21
|
+
"cross-validation",
|
|
22
|
+
"model-selection",
|
|
23
|
+
]
|
|
24
|
+
classifiers = [
|
|
25
|
+
"Development Status :: 4 - Beta",
|
|
26
|
+
"Intended Audience :: Science/Research",
|
|
27
|
+
"Intended Audience :: Developers",
|
|
28
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
29
|
+
"Programming Language :: Python :: 3",
|
|
30
|
+
"Programming Language :: Python :: 3.10",
|
|
31
|
+
"Programming Language :: Python :: 3.11",
|
|
32
|
+
"Programming Language :: Python :: 3.12",
|
|
33
|
+
"Operating System :: OS Independent",
|
|
34
|
+
]
|
|
35
|
+
dependencies = [
|
|
36
|
+
"numpy>=1.24",
|
|
37
|
+
"scipy>=1.10",
|
|
38
|
+
"scikit-learn>=1.6,<1.10",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
[project.urls]
|
|
42
|
+
Homepage = "https://github.com/rpcoelho17/PatternSearch_HyperParam_Opt__V2"
|
|
43
|
+
Repository = "https://github.com/rpcoelho17/PatternSearch_HyperParam_Opt__V2"
|
|
44
|
+
|
|
45
|
+
[project.optional-dependencies]
|
|
46
|
+
test = ["pytest"]
|
|
47
|
+
docs = ["sphinx", "numpydoc", "pydata-sphinx-theme"]
|
|
48
|
+
|
|
49
|
+
[tool.setuptools.packages.find]
|
|
50
|
+
where = ["src"]
|
|
51
|
+
|
|
52
|
+
[tool.pytest.ini_options]
|
|
53
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: BayesHalvingSearchCV
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: scikit-learn hyperparameter search with multi-fidelity data growth and scatter-search multi-start: a from-scratch Bayesian (GP + Expected Improvement) search, BayesHalvingSearchCV, plus a Hooke-Jeeves pattern search, PatternSearchCV
|
|
5
|
+
Author: Rodrigo Pinto Coelho
|
|
6
|
+
License-Expression: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://github.com/rpcoelho17/PatternSearch_HyperParam_Opt__V2
|
|
8
|
+
Project-URL: Repository, https://github.com/rpcoelho17/PatternSearch_HyperParam_Opt__V2
|
|
9
|
+
Keywords: scikit-learn,hyperparameter-optimization,bayesian-optimization,pattern-search,multi-fidelity,gaussian-process,cross-validation,model-selection
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: numpy>=1.24
|
|
23
|
+
Requires-Dist: scipy>=1.10
|
|
24
|
+
Requires-Dist: scikit-learn<1.10,>=1.6
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: pytest; extra == "test"
|
|
27
|
+
Provides-Extra: docs
|
|
28
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
29
|
+
Requires-Dist: numpydoc; extra == "docs"
|
|
30
|
+
Requires-Dist: pydata-sphinx-theme; extra == "docs"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# BayesHalvingSearchCV & PatternSearchCV
|
|
34
|
+
|
|
35
|
+
During my post-graduate studies in Data Science at USI Switzerland, I had to hyperparameter-tune
|
|
36
|
+
models on genuinely large datasets, and I went looking for a way to find good
|
|
37
|
+
hyperparameters quickly instead of paying full price for every fit.
|
|
38
|
+
Inspired by MATLAB's Global Optimization Toolbox pattern search for discrete
|
|
39
|
+
problems, I built a first version of what became `PatternSearchCV` — and even
|
|
40
|
+
that first version already beat Bayesian search on real benchmarks, simply
|
|
41
|
+
because it deduplicated repeated evaluations, something the Bayesian search
|
|
42
|
+
tools I compared it against didn't do; that alone saved real fits on large
|
|
43
|
+
datasets. I later added a "halving" mechanism on top of that base pattern
|
|
44
|
+
search — growing how much data each evaluation sees as the search's own
|
|
45
|
+
trajectory shows it converging (measured by the shrinking distance between
|
|
46
|
+
successive improving moves), instead of committing to full data from the
|
|
47
|
+
start — and that further improved the gains over the deduplication-only
|
|
48
|
+
version. When I eventually benchmarked properly against Optuna, the picture
|
|
49
|
+
became precise: Optuna's Bayesian search found the optimum in fewer raw
|
|
50
|
+
trials, but every one of those trials evaluated 100% of the data — my
|
|
51
|
+
advantage was coming from deduplication, the multi-fidelity data growth, and
|
|
52
|
+
the halving logic together, not from having fewer trials. So I adapted that
|
|
53
|
+
same multi-fidelity infrastructure onto a from-scratch Bayesian search of my
|
|
54
|
+
own, giving `BayesHalvingSearchCV` Optuna-quality trial efficiency without
|
|
55
|
+
paying full price for every trial.
|
|
56
|
+
|
|
57
|
+
**Both estimators implement scikit-learn's standard search-CV interface and
|
|
58
|
+
work with any scikit-learn-compatible estimator** — they drop into existing
|
|
59
|
+
pipelines exactly the way `GridSearchCV` or `RandomizedSearchCV` do. Our own
|
|
60
|
+
benchmarks use `ExtraTreesRegressor` specifically because it's fast and
|
|
61
|
+
performs well on the dataset we tested against, not because either algorithm
|
|
62
|
+
is limited to it.
|
|
63
|
+
|
|
64
|
+
**Best measured results** (523K-row retail regression benchmark; full
|
|
65
|
+
methodology and every run logged in [`EXPERIMENTS.md`](EXPERIMENTS.md)):
|
|
66
|
+
`BayesHalvingSearchCV` reached the benchmark's historical optimum at
|
|
67
|
+
**3.125 full-fit equivalents** (Experiment 17); `PatternSearchCV` reached the
|
|
68
|
+
same optimum at **5.04 full-fit equivalents** (Experiment 11). For
|
|
69
|
+
comparison, Optuna's own GP-based sampler needed **15.00 full-fit
|
|
70
|
+
equivalents** — every one of its trials at 100% of the data — to reach a
|
|
71
|
+
comparable answer. That's a **4.8× reduction** in compute for
|
|
72
|
+
`BayesHalvingSearchCV` and a **2.98× reduction** for `PatternSearchCV`,
|
|
73
|
+
relative to Optuna, for the same answer.
|
|
74
|
+
|
|
75
|
+
GPU acceleration is available the same way it is for any scikit-learn
|
|
76
|
+
estimator: enable it in `estimator`'s own settings. See
|
|
77
|
+
[`API_REFERENCE.md`](API_REFERENCE.md) for the full explanation.
|
|
78
|
+
|
|
79
|
+
Both estimators share one **multi-fidelity "bullseye" data-growth mechanism**
|
|
80
|
+
and one **scatter-search multi-start layer**:
|
|
81
|
+
|
|
82
|
+
- `PatternSearchCV` — classic Hooke-Jeeves pattern search (1961), adapted to
|
|
83
|
+
grow its data budget as it converges.
|
|
84
|
+
- `BayesHalvingSearchCV` — a from-scratch Gaussian Process + Expected
|
|
85
|
+
Improvement Bayesian search, on the exact same multi-fidelity
|
|
86
|
+
infrastructure, with **zero additional dependencies** (no Optuna, no
|
|
87
|
+
torch — just `numpy`, `scipy`, `scikit-learn`, already required by
|
|
88
|
+
`PatternSearchCV`).
|
|
89
|
+
|
|
90
|
+
Both estimators start every search on a small, representative subsample of
|
|
91
|
+
the training data and only pay full price once their own search trajectory
|
|
92
|
+
shows them converging on an optimum — every reported result is still
|
|
93
|
+
confirmed on 100% of the data before it's trusted.
|
|
94
|
+
|
|
95
|
+
## Quick start
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from bayes_halving_search_cv import PatternSearchCV, BayesHalvingSearchCV
|
|
99
|
+
from sklearn.model_selection import TimeSeriesSplit
|
|
100
|
+
|
|
101
|
+
param_grid = {"max_depth": [3, 5, 7, 9, 12, 16], "min_samples_leaf": [1, 2, 4, 8]}
|
|
102
|
+
|
|
103
|
+
search = PatternSearchCV(
|
|
104
|
+
estimator, param_grid,
|
|
105
|
+
cv=TimeSeriesSplit(n_splits=5),
|
|
106
|
+
scoring="neg_mean_absolute_error",
|
|
107
|
+
n_starts=4, # scatter-search multi-start
|
|
108
|
+
subsample="stratified", # transition sampling for time-series data
|
|
109
|
+
random_state=0,
|
|
110
|
+
)
|
|
111
|
+
search.fit(X, y)
|
|
112
|
+
search.best_params_ # chosen ONLY from full-data evaluations
|
|
113
|
+
search.local_optima_ # the map: every distinct optimum found
|
|
114
|
+
search.cv_results_ # every point evaluated, and its score
|
|
115
|
+
search.search_history_ # every confirmed-improving move across every start
|
|
116
|
+
|
|
117
|
+
# or the Bayesian search, on the exact same multi-fidelity infrastructure:
|
|
118
|
+
search = BayesHalvingSearchCV(estimator, param_grid, cv=TimeSeriesSplit(5),
|
|
119
|
+
scoring="neg_mean_absolute_error", random_state=0)
|
|
120
|
+
search.fit(X, y)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
For the full parameter reference and worked examples — how to specify a
|
|
124
|
+
search space, the data ladder, `contraction="eager"`'s cost/risk trade-off,
|
|
125
|
+
and why `subsample="stratified"` matters for time series — see
|
|
126
|
+
[`API_REFERENCE.md`](API_REFERENCE.md). For the full design rationale behind
|
|
127
|
+
every default, see [`PatternSearchCV_SPEC.md`](PatternSearchCV_SPEC.md) and
|
|
128
|
+
[`BAYESHALVINGSearchCV_SPEC.md`](BAYESHALVINGSearchCV_SPEC.md).
|
|
129
|
+
|
|
130
|
+
## Development
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
python -m venv .venv
|
|
134
|
+
.venv/Scripts/pip install -e .[test]
|
|
135
|
+
.venv/Scripts/python -m pytest
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Logging: the package logs every algorithmic decision (moves, contractions,
|
|
139
|
+
ring calibrations and crossings, data climbs, merges, cache statistics) to the
|
|
140
|
+
`SearchCV` logger. `verbose=1` attaches a stream handler at INFO,
|
|
141
|
+
`verbose=2` at DEBUG.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/BayesHalvingSearchCV.egg-info/PKG-INFO
|
|
5
|
+
src/BayesHalvingSearchCV.egg-info/SOURCES.txt
|
|
6
|
+
src/BayesHalvingSearchCV.egg-info/dependency_links.txt
|
|
7
|
+
src/BayesHalvingSearchCV.egg-info/requires.txt
|
|
8
|
+
src/BayesHalvingSearchCV.egg-info/top_level.txt
|
|
9
|
+
src/bayes_halving_search_cv/__init__.py
|
|
10
|
+
src/bayes_halving_search_cv/_bayes.py
|
|
11
|
+
src/bayes_halving_search_cv/_climber.py
|
|
12
|
+
src/bayes_halving_search_cv/_engine.py
|
|
13
|
+
src/bayes_halving_search_cv/_fidelity.py
|
|
14
|
+
src/bayes_halving_search_cv/_gp.py
|
|
15
|
+
src/bayes_halving_search_cv/_sampling.py
|
|
16
|
+
src/bayes_halving_search_cv/_search.py
|
|
17
|
+
src/bayes_halving_search_cv/_space.py
|
|
18
|
+
src/bayes_halving_search_cv/_starts.py
|
|
19
|
+
tests/test_bayes.py
|
|
20
|
+
tests/test_climber.py
|
|
21
|
+
tests/test_estimator.py
|
|
22
|
+
tests/test_fidelity.py
|
|
23
|
+
tests/test_gp.py
|
|
24
|
+
tests/test_sampling.py
|
|
25
|
+
tests/test_sklearn_compat.py
|
|
26
|
+
tests/test_starts.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
bayes_halving_search_cv
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""bayes-halving-search-cv: scikit-learn hyperparameter search with bullseye
|
|
2
|
+
multi-fidelity data growth and scatter-search multi-start. Provides
|
|
3
|
+
BayesHalvingSearchCV (a from-scratch GP + Expected Improvement Bayesian
|
|
4
|
+
search) and PatternSearchCV (Hooke-Jeeves pattern search)."""
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
|
|
8
|
+
from ._bayes import BayesHalvingSearchCV
|
|
9
|
+
from ._search import PatternSearchCV
|
|
10
|
+
from ._space import Dimension, Space
|
|
11
|
+
|
|
12
|
+
__all__ = ["PatternSearchCV", "BayesHalvingSearchCV", "Space", "Dimension"]
|
|
13
|
+
__version__ = "0.1.0"
|
|
14
|
+
|
|
15
|
+
logging.getLogger("SearchCV").addHandler(logging.NullHandler())
|