fitfleet 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fitfleet-0.2.0/LICENSE +21 -0
- fitfleet-0.2.0/PKG-INFO +342 -0
- fitfleet-0.2.0/README.md +307 -0
- fitfleet-0.2.0/fitfleet/__init__.py +12 -0
- fitfleet-0.2.0/fitfleet/__main__.py +6 -0
- fitfleet-0.2.0/fitfleet/base.py +738 -0
- fitfleet-0.2.0/fitfleet/classification.py +30 -0
- fitfleet-0.2.0/fitfleet/cli.py +194 -0
- fitfleet-0.2.0/fitfleet/config.py +599 -0
- fitfleet-0.2.0/fitfleet/encoding.py +240 -0
- fitfleet-0.2.0/fitfleet/feature_research.py +702 -0
- fitfleet-0.2.0/fitfleet/features.py +156 -0
- fitfleet-0.2.0/fitfleet/model.py +127 -0
- fitfleet-0.2.0/fitfleet/regression.py +15 -0
- fitfleet-0.2.0/fitfleet/results/__init__.py +0 -0
- fitfleet-0.2.0/fitfleet/results/base.py +239 -0
- fitfleet-0.2.0/fitfleet/results/classification.py +182 -0
- fitfleet-0.2.0/fitfleet/results/experiment.py +377 -0
- fitfleet-0.2.0/fitfleet/results/regression.py +146 -0
- fitfleet-0.2.0/fitfleet/schema.py +346 -0
- fitfleet-0.2.0/fitfleet/search.py +287 -0
- fitfleet-0.2.0/fitfleet/target.py +55 -0
- fitfleet-0.2.0/fitfleet/tracking/__init__.py +18 -0
- fitfleet-0.2.0/fitfleet/tracking/core.py +860 -0
- fitfleet-0.2.0/fitfleet/tracking/log.py +534 -0
- fitfleet-0.2.0/fitfleet/tracking/outputs.py +292 -0
- fitfleet-0.2.0/fitfleet/tracking/tasks.py +72 -0
- fitfleet-0.2.0/fitfleet/train_data.py +159 -0
- fitfleet-0.2.0/fitfleet/utils.py +364 -0
- fitfleet-0.2.0/fitfleet/visualization/__init__.py +18 -0
- fitfleet-0.2.0/fitfleet/visualization/background.py +66 -0
- fitfleet-0.2.0/fitfleet/visualization/context.py +185 -0
- fitfleet-0.2.0/fitfleet/visualization/core.py +337 -0
- fitfleet-0.2.0/fitfleet/visualization/pdf.py +348 -0
- fitfleet-0.2.0/fitfleet/visualization/plots/__init__.py +5 -0
- fitfleet-0.2.0/fitfleet/visualization/plots/_common.py +106 -0
- fitfleet-0.2.0/fitfleet/visualization/plots/classification.py +387 -0
- fitfleet-0.2.0/fitfleet/visualization/plots/comparison.py +369 -0
- fitfleet-0.2.0/fitfleet/visualization/plots/features.py +85 -0
- fitfleet-0.2.0/fitfleet/visualization/plots/regression.py +262 -0
- fitfleet-0.2.0/fitfleet/visualization/style.py +121 -0
- fitfleet-0.2.0/fitfleet/workers.py +171 -0
- fitfleet-0.2.0/fitfleet.egg-info/PKG-INFO +342 -0
- fitfleet-0.2.0/fitfleet.egg-info/SOURCES.txt +71 -0
- fitfleet-0.2.0/fitfleet.egg-info/dependency_links.txt +1 -0
- fitfleet-0.2.0/fitfleet.egg-info/entry_points.txt +2 -0
- fitfleet-0.2.0/fitfleet.egg-info/requires.txt +13 -0
- fitfleet-0.2.0/fitfleet.egg-info/top_level.txt +1 -0
- fitfleet-0.2.0/pyproject.toml +54 -0
- fitfleet-0.2.0/setup.cfg +4 -0
- fitfleet-0.2.0/tests/test_api.py +25 -0
- fitfleet-0.2.0/tests/test_base_results.py +83 -0
- fitfleet-0.2.0/tests/test_classification.py +208 -0
- fitfleet-0.2.0/tests/test_classification_results.py +202 -0
- fitfleet-0.2.0/tests/test_cli.py +274 -0
- fitfleet-0.2.0/tests/test_experiment_report.py +124 -0
- fitfleet-0.2.0/tests/test_feature_research.py +367 -0
- fitfleet-0.2.0/tests/test_features.py +175 -0
- fitfleet-0.2.0/tests/test_fixes.py +165 -0
- fitfleet-0.2.0/tests/test_load.py +271 -0
- fitfleet-0.2.0/tests/test_mixed_types.py +252 -0
- fitfleet-0.2.0/tests/test_mixed_types_phase2.py +196 -0
- fitfleet-0.2.0/tests/test_mixed_types_phase3.py +202 -0
- fitfleet-0.2.0/tests/test_model.py +90 -0
- fitfleet-0.2.0/tests/test_regression.py +64 -0
- fitfleet-0.2.0/tests/test_regression_results.py +141 -0
- fitfleet-0.2.0/tests/test_search.py +340 -0
- fitfleet-0.2.0/tests/test_shared_memory.py +220 -0
- fitfleet-0.2.0/tests/test_target.py +64 -0
- fitfleet-0.2.0/tests/test_tracking.py +809 -0
- fitfleet-0.2.0/tests/test_training.py +183 -0
- fitfleet-0.2.0/tests/test_visualization.py +560 -0
- fitfleet-0.2.0/tests/test_worker_optimizations.py +38 -0
fitfleet-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 shaizae
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
fitfleet-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fitfleet
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Run many scikit-learn models x feature subsets x hyper-parameters in parallel and keep the best.
|
|
5
|
+
Author: shaizae
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/shaizae/fitfleet
|
|
8
|
+
Project-URL: Issues, https://github.com/shaizae/fitfleet/issues
|
|
9
|
+
Keywords: machine-learning,scikit-learn,hyperparameter-search,feature-selection,multiprocessing
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: numpy>=2.0
|
|
23
|
+
Requires-Dist: pandas>=2.2
|
|
24
|
+
Requires-Dist: scikit-learn>=1.6
|
|
25
|
+
Requires-Dist: joblib>=1.4
|
|
26
|
+
Requires-Dist: threadpoolctl>=3.5
|
|
27
|
+
Requires-Dist: tqdm>=4.66
|
|
28
|
+
Requires-Dist: matplotlib>=3.9
|
|
29
|
+
Requires-Dist: reportlab>=4.2
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
32
|
+
Requires-Dist: pytest-cov>=5; extra == "dev"
|
|
33
|
+
Requires-Dist: psutil>=6; extra == "dev"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
# fitfleet
|
|
37
|
+
|
|
38
|
+
A scikit-learn package for running many tests in multiprocessing with minimal garbage creation:
|
|
39
|
+
every model x feature subset x hyper-parameter combination, scored on the same splits, best ones refitted.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
pip install fitfleet # from PyPI
|
|
45
|
+
pip install -e . # the library, from a clone
|
|
46
|
+
pip install -e ".[dev]" # + test tools
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
import fitfleet
|
|
51
|
+
from fitfleet import Classification, FilteringCriteriaClassification
|
|
52
|
+
|
|
53
|
+
with Classification() as run:
|
|
54
|
+
run.set(X, y, [RandomForestClassifier(), SVC()])
|
|
55
|
+
run.features.standard_scaler()
|
|
56
|
+
run.k_folds(5)
|
|
57
|
+
best = run.filter_by(FilteringCriteriaClassification.accuracy)
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Mixed data: categories, dates, text and missing values
|
|
61
|
+
|
|
62
|
+
Features don't have to be numbers. Text and pandas `category` columns, yes/no columns, dates, free text and
|
|
63
|
+
missing values are handled without any preparation:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
with Classification() as run:
|
|
67
|
+
run.set(X, y, [LogisticRegression(), RandomForestClassifier(), HistGradientBoostingClassifier()],
|
|
68
|
+
ordinal={"size": ["S", "M", "L"]}) # optional: kinds are detected, declare to override
|
|
69
|
+
run.features.standard_scaler() # applied to number columns only
|
|
70
|
+
run.k_folds(5)
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
| Column kind | Detected from | How models get it |
|
|
74
|
+
|---|---|---|
|
|
75
|
+
| numeric, boolean | numbers, `bool` | as is (yes/no as 0/1) |
|
|
76
|
+
| categorical | text with few words, pandas `category` | one-hot for linear / distance models (target encoding above `max_onehot_categories`, default 40, categories); integer codes for tree models; native categories for HistGradientBoosting |
|
|
77
|
+
| ordinal | declared `ordinal={col: [lowest, ..., highest]}`, ordered pandas `category` | its position in the order, as a number |
|
|
78
|
+
| datetime | datetime dtype, or declared `datetime=[...]` | year, month, day, weekday, day of year, hour, days since 1970 |
|
|
79
|
+
| text | strings of 3+ words on average with many distinct values, or declared `text=[...]` | 30 numbers per column (character TF-IDF + SVD) |
|
|
80
|
+
| missing values | NaN / None | kept for models that accept NaN (trees, HistGradientBoosting, LightGBM, XGBoost); otherwise filled with the median plus a "was missing" column |
|
|
81
|
+
|
|
82
|
+
- Every encoder, imputer and scaler is fitted on the training rows of each split only. The category list and
|
|
83
|
+
the text embedding are learned from all rows before splitting; neither looks at the target.
|
|
84
|
+
- Feature subsets (brute force, forward / backward selection, `feature_selection`) choose *original* columns:
|
|
85
|
+
a categorical or text column is kept or dropped as a whole. Purely numeric tables without gaps behave as before.
|
|
86
|
+
- Text columns where almost every row has its own value (ids, keys) are left out with a warning
|
|
87
|
+
(`drop_ids=False`, or declaring the column, keeps them). Date columns trigger a warning about shuffled folds.
|
|
88
|
+
- `run.categorical_encoding = "onehot" | "ordinal" | "target"` forces one encoding for every model. To compare
|
|
89
|
+
encodings on the same folds, search them like a hyper-parameter:
|
|
90
|
+
`run.set_hyper_parameters({"categorical_encoding": ["onehot", "target"]})`.
|
|
91
|
+
- A saved model takes raw rows: text, unseen categories, new dates and missing values are encoded the same way.
|
|
92
|
+
- polars DataFrames are accepted wherever pandas ones are.
|
|
93
|
+
- In a config file, the optional `[columns]` section sets `categorical`, `numeric`, `ordinal`, `datetime`, `text`,
|
|
94
|
+
`drop`, `drop_ids`, `encoding` and `max_onehot` (see `fitfleet init`).
|
|
95
|
+
|
|
96
|
+
## Command line
|
|
97
|
+
|
|
98
|
+
```
|
|
99
|
+
fitfleet init experiment.toml # example config with every option explained
|
|
100
|
+
fitfleet run experiment.toml --check # validate the config and the data, no training
|
|
101
|
+
fitfleet run experiment.toml # train every configuration
|
|
102
|
+
fitfleet predict runs/<date_time>/1_<Model>/model.joblib new.csv -o predictions.csv --proba
|
|
103
|
+
fitfleet log runs/<date_time> # what passed, failed or was skipped, where and why
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
`python -m fitfleet ...` works too. A config names the data file, target column and task, the models (any
|
|
107
|
+
scikit-learn class name, or an import path like `xgboost.XGBClassifier`), a hyper-parameter grid, preprocessing,
|
|
108
|
+
optional feature selection and the validation. Each run writes a new `runs/<date_time>/` folder with
|
|
109
|
+
`leaderboard.csv` (every configuration, best first; ties go to fewer features), a copy of the config, and the
|
|
110
|
+
top `keep` models, each in its own folder with `model.joblib` and a PDF report.
|
|
111
|
+
|
|
112
|
+
## Execution tracking
|
|
113
|
+
|
|
114
|
+
Every stage of a run is recorded: what ran, where, when it started and finished, whether it PASSED,
|
|
115
|
+
FAILED, was SKIPPED or CANCELLED and, for a failure, a plain-language reason, the exception type, the
|
|
116
|
+
location (`file.py:line in function`, inside the worker process for training tasks) and the full traceback.
|
|
117
|
+
`fitfleet run` prints one line per stage and an execution summary at the end (`-v` adds tracebacks), and
|
|
118
|
+
the run folder gets `execution_<id>.log` (developer log), `.jsonl` (live JSON events), `.json` and
|
|
119
|
+
`_summary.txt`; the experiment PDF gets an Execution Summary section.
|
|
120
|
+
|
|
121
|
+
```
|
|
122
|
+
fitfleet log runs/<date_time> # summary + stage tree
|
|
123
|
+
fitfleet log runs/<date_time> --status fail # failed stages with reason and location
|
|
124
|
+
fitfleet log runs/<date_time> --expand "Train and validate" --depth 2
|
|
125
|
+
fitfleet log runs/<date_time> --details "Load data" # everything about one stage, with its traceback
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Filters: `--status`, `--stage`, `--module`, `--level`, `--since` / `--until`, `--error-type`,
|
|
129
|
+
`--execution-id`; `--json` for scripts. In Python, a run has three log properties, each
|
|
130
|
+
with the summary and the stage tree: `run.execution_log` (text), `run.save_pdf_log` and
|
|
131
|
+
`run.save_json_log` (written into `log_dir`, set in the constructor:
|
|
132
|
+
`Classification(log_dir="reports")`). `save_pdf_report()` adds the log to its PDF and writes the
|
|
133
|
+
log PDF next to it.
|
|
134
|
+
See [docs/execution_tracking.md](docs/execution_tracking.md).
|
|
135
|
+
|
|
136
|
+
## Search smarter than every combination
|
|
137
|
+
|
|
138
|
+
The default tries every model x feature subset x hyper-parameter combination (grid). For large grids:
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
run.search("random", n_iter=200) # 200 combinations drawn at random
|
|
142
|
+
run.search("halving", factor=3) # successive halving (below)
|
|
143
|
+
run.search("halving", n_iter=500, min_rows=200) # halving on a random sample of the grid
|
|
144
|
+
run.forward_selection(max_features=10, tol=0.002) # greedy feature subsets instead of brute force
|
|
145
|
+
run.backward_elimination(min_features=3)
|
|
146
|
+
run.k_folds(5)
|
|
147
|
+
run.search_log # what each round trained
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
- **halving**: every combination is trained on a small share of the rows, the best 1/`factor` go on to
|
|
151
|
+
`factor` x more rows, and the last round, on every row, is an ordinary k-fold / train-test run. Only the
|
|
152
|
+
last round's results are returned, so they compare fairly. Not available with leave-one-out.
|
|
153
|
+
- **forward / backward**: for each model x hyper-parameter combination, add (remove) the one feature that
|
|
154
|
+
helps most, until a step gains no more than `tol`. About n²/2 subsets instead of 2ⁿ, so it works with many
|
|
155
|
+
columns. Every subset trained is in the results. Uses the default grid search.
|
|
156
|
+
`tol` only decides when to stop: the last step's tries (one feature more, which did not gain more than
|
|
157
|
+
`tol`) are in the results too, and one of them can still top the leaderboard by a tiny, chance margin.
|
|
158
|
+
To get the subset the search settled on, take the best of the step before (`run.search_log` lists the
|
|
159
|
+
steps).
|
|
160
|
+
- Rounds are compared by `select_by` (default accuracy / r2): `run.search("halving", select_by=...)`.
|
|
161
|
+
|
|
162
|
+
In a config file: `[search] strategy = "halving"` (with `n_iter`, `factor`, `min_rows`), and
|
|
163
|
+
`[feature_selection] method = "forward"` or `"backward"` (with `max_features`, `min_features`, `tol`).
|
|
164
|
+
`select_by` from `[run]` is used for both.
|
|
165
|
+
|
|
166
|
+
## Experiment report
|
|
167
|
+
|
|
168
|
+
After a run, summarize every configuration it trained (classification or regression):
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
run.k_folds(5)
|
|
172
|
+
report = run.experiment_report() # ranked by accuracy / r2; or experiment_report("f1"), ("mae"), ...
|
|
173
|
+
print(report) # overview, best configuration, per-model summary, top 10
|
|
174
|
+
report.leaderboard # DataFrame: every configuration, best first
|
|
175
|
+
report.summary # DataFrame: per model - configurations, best / mean / std / worst, ...
|
|
176
|
+
report.best # the best configuration's row
|
|
177
|
+
report.save_csv("out") # out/leaderboard.csv + out/summary.csv
|
|
178
|
+
report.save_pdf_report("out") # out/experiment_report_<date>.pdf
|
|
179
|
+
report.roc_curves() # classification: {model: (fpr, tpr, auc)}, best config per model
|
|
180
|
+
report.plot_roc_curves()
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
`fitfleet run` writes `summary.csv` and the experiment PDF into the run folder too.
|
|
184
|
+
|
|
185
|
+
### Plots and the PDF reports
|
|
186
|
+
|
|
187
|
+
`save_pdf_report` draws every plot that applies to the run and puts all of them in the PDF. The sections are:
|
|
188
|
+
|
|
189
|
+
- an executive summary, the configuration and a dataset overview
|
|
190
|
+
- the model ranking and the spread of scores over configurations
|
|
191
|
+
- **classification**: metrics against the majority-class baseline, a confusion matrix per model (counts and
|
|
192
|
+
row-normalized), per-class precision / recall / F1, ROC and precision-recall curves per model and per
|
|
193
|
+
class, and a calibration diagram (binary)
|
|
194
|
+
- **regression**: R² / MAE / RMSE, actual vs predicted, residuals vs predicted, the residual distribution and
|
|
195
|
+
a normal Q-Q plot, error by target range, and the error distribution across models
|
|
196
|
+
- per-fold scores (k-fold), hyper-parameter sensitivity and two-parameter heatmaps, and score against the
|
|
197
|
+
number of features
|
|
198
|
+
- runtime per configuration and score against cost (with the Pareto front)
|
|
199
|
+
- feature importance for refitted models
|
|
200
|
+
|
|
201
|
+
Each plot comes with an explanation and the values it shows. A last section lists every plot that was
|
|
202
|
+
skipped (not applicable, or not measured, such as memory) or failed.
|
|
203
|
+
|
|
204
|
+
**Plotting can never stop an experiment.** Each plot is drawn in its own error boundary
|
|
205
|
+
(SUCCESS / SKIPPED / FAILED, logged on the `fitfleet.visualization` logger). `fitfleet run` saves the
|
|
206
|
+
models first and writes the PDFs on a background thread. A failing plot or PDF is logged as a warning, and
|
|
207
|
+
the run still finishes with its leaderboard and models.
|
|
208
|
+
|
|
209
|
+
```python
|
|
210
|
+
viz = report.visualize() # every plot, without a PDF; never raises
|
|
211
|
+
print(viz.summary()) # "21 generated, 3 skipped, 0 failed"
|
|
212
|
+
viz.save_images("out/plots") # one PNG per generated plot
|
|
213
|
+
viz.failed, viz.skipped # records with the reason / traceback
|
|
214
|
+
best[0].save_pdf_report("out") # per-model report (confusion, ROC, residuals, importance, ...)
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
The research behind the choice of plots, the architecture, and how to add a plot type are in
|
|
218
|
+
[docs/visualization.md](docs/visualization.md).
|
|
219
|
+
|
|
220
|
+
## Feature research
|
|
221
|
+
|
|
222
|
+
A stand-alone class (no `Classification` / `Regression` run needed) to understand the features before
|
|
223
|
+
choosing them. The task is guessed from `y` (or pass `task="classification"` / `"regression"`).
|
|
224
|
+
|
|
225
|
+
```python
|
|
226
|
+
from fitfleet import FeatureResearch
|
|
227
|
+
|
|
228
|
+
research = FeatureResearch(X, y, cv=5)
|
|
229
|
+
research.missing_report() # missing values per column
|
|
230
|
+
research.constant_features(0.99) # constant / quasi-constant columns
|
|
231
|
+
research.cardinality() # distinct values; flags id-like columns
|
|
232
|
+
research.distribution_summary() # skew, kurtosis, % IQR outliers
|
|
233
|
+
research.univariate_scores() # F-test (+ Pearson / Spearman for regression)
|
|
234
|
+
research.mutual_information() # also catches non-linear relations
|
|
235
|
+
research.single_feature_score() # CV score of a model on each feature alone
|
|
236
|
+
research.leakage_check(0.98) # features that alone score suspiciously high
|
|
237
|
+
research.correlation_matrix() # feature x feature (Spearman by default)
|
|
238
|
+
research.high_correlation_pairs(0.9) # `drop` = the less informative of each pair
|
|
239
|
+
research.vif() # multicollinearity
|
|
240
|
+
research.model_importance(model) # feature_importances_ / |coef_|
|
|
241
|
+
research.permutation_importance(model) # score drop on held-out folds, any model
|
|
242
|
+
research.random_probe(model) # does each feature beat pure-noise columns?
|
|
243
|
+
research.report() # one row per feature, every score + mean rank
|
|
244
|
+
X = X.drop(columns=research.recommend(probe=True).index) # constant / id / missing / redundant / noise
|
|
245
|
+
research.save_pdf_report("reports") # all of the above in one PDF, with charts
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
`save_pdf_report` writes `feature_research_<date>.pdf`: overview and target balance, columns to drop and
|
|
249
|
+
possible leakage, the feature ranking, a permutation-importance chart (with the best noise column), data
|
|
250
|
+
quality, correlated pairs, VIF and a correlation heatmap. The charts are also available on their own:
|
|
251
|
+
`research.plot_importance()`, `research.plot_correlation()`.
|
|
252
|
+
|
|
253
|
+
A feature is called `noise` only when it beats the random noise columns on neither permutation importance
|
|
254
|
+
nor mutual information: permutation importance alone under-rates correlated features (shuffling one
|
|
255
|
+
costs nothing while its twin is still there).
|
|
256
|
+
|
|
257
|
+
The data is kept like in a run: `research.features` is a `Features` object and `research.targets` a
|
|
258
|
+
`Target` (label-encoded for classification, `research.targets.classes` maps back). A step applied there,
|
|
259
|
+
e.g. `research.features.standard_scaler()`, is used by every method after it.
|
|
260
|
+
|
|
261
|
+
Every method returns a DataFrame or Series. The statistical methods use the numeric columns only, with
|
|
262
|
+
missing values filled by the median (`research.skipped` lists the others). The default model is a random
|
|
263
|
+
forest; pass any scikit-learn estimator instead.
|
|
264
|
+
|
|
265
|
+
## Sound when training is done
|
|
266
|
+
|
|
267
|
+
Pass `sound=` to hear when a run has finished. The sound plays once, when the object goes out of
|
|
268
|
+
scope (the end of its `with` block, or when it is garbage-collected), and only if at least one
|
|
269
|
+
training run finished.
|
|
270
|
+
|
|
271
|
+
```python
|
|
272
|
+
from fitfleet import Classification, FilteringCriteriaClassification, Sound
|
|
273
|
+
|
|
274
|
+
with Classification(sound=Sound.success) as run: # or sound="success"
|
|
275
|
+
run.set(X, y, models)
|
|
276
|
+
run.k_folds(5)
|
|
277
|
+
best = run.filter_by(FilteringCriteriaClassification.accuracy)
|
|
278
|
+
# the success sound plays here
|
|
279
|
+
|
|
280
|
+
Classification.play_sound(Sound.chime) # play one any time (a static method)
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
| `Sound` | What it sounds like |
|
|
284
|
+
|-----------|------------------------------------------------------------------|
|
|
285
|
+
| `none` | silent (the default) |
|
|
286
|
+
| `ping` | one soft bell note |
|
|
287
|
+
| `double` | two rising notes, a happy "ding-ding" |
|
|
288
|
+
| `chime` | three bell notes ringing into a major chord |
|
|
289
|
+
| `success` | quick rising arpeggio ending on a bright chord: "it worked!" |
|
|
290
|
+
| `sparkle` | a fast twinkling run up high, playful |
|
|
291
|
+
| `alarm` | repeated high/low tone, hard to miss |
|
|
292
|
+
| `bell` | the terminal bell only |
|
|
293
|
+
|
|
294
|
+
The sounds are generated in code (soft bell-like tones), so there are no audio files or extra
|
|
295
|
+
dependencies. They play with `winsound` on Windows, `afplay` on macOS and `paplay` / `aplay` on Linux;
|
|
296
|
+
if none of these is available, the terminal bell rings instead. Playing a sound never raises an error.
|
|
297
|
+
|
|
298
|
+
In a config file: `[output] sound = "success"`. The older `ping = true` still works (same as
|
|
299
|
+
`sound = "ping"`).
|
|
300
|
+
|
|
301
|
+
## Use a trained model
|
|
302
|
+
|
|
303
|
+
```python
|
|
304
|
+
path = best[0].save_model("models") # a .joblib file in models/
|
|
305
|
+
model = fitfleet.load(path) # only load files you trust
|
|
306
|
+
model.predict(new_df) # picks the training columns by name; returns the original labels
|
|
307
|
+
model.predict_proba(new_df) # DataFrame, one column per class
|
|
308
|
+
model.metrics, model.params, model.feature_columns, model.class_names
|
|
309
|
+
```
|
|
310
|
+
|
|
311
|
+
The saved model includes the run's scaling, polynomial features and feature subset, so pass raw rows.
|
|
312
|
+
Models saved before 0.2 (a bare estimator) still load.
|
|
313
|
+
|
|
314
|
+
Renamed in 0.2: `fetchers` -> `features`, `fetcher_selection()` -> `feature_selection()`,
|
|
315
|
+
`brut_force_features()` -> `brute_force_features()`. The old names still work but warn. Imports moved from
|
|
316
|
+
`external_classes` / `utils` / `results` into the `fitfleet` package.
|
|
317
|
+
|
|
318
|
+
## Usage notes
|
|
319
|
+
|
|
320
|
+
- `train_test_split(ratio)`: `ratio` is the fraction of rows used for **training** (0.8 = 80% train).
|
|
321
|
+
- Scalers and polynomial features (`pipeline.features.standard_scaler()` etc.) are refitted on the
|
|
322
|
+
training rows of every split, so test rows never leak into preprocessing.
|
|
323
|
+
- Classification uses stratified splits. All models in a run are scored on the same folds.
|
|
324
|
+
- k-folds shuffles the rows, so without a seed each run uses different folds and scores can change a
|
|
325
|
+
little from run to run. Set `run.random_state = 0` (or `random_state` in `[run]`) for repeatable results.
|
|
326
|
+
- Results carry metrics and predictions only. `filter_by()` (or `refit(results)`) fits the chosen
|
|
327
|
+
models; the fitted model is a scikit-learn `Pipeline` that takes raw rows.
|
|
328
|
+
- The worker pool is reused across runs on the same object. Use `with Classification() as c:` or
|
|
329
|
+
call `c.close()` to stop the workers.
|
|
330
|
+
|
|
331
|
+
## Tests
|
|
332
|
+
|
|
333
|
+
```
|
|
334
|
+
pytest # unit tests (load tests are skipped)
|
|
335
|
+
pytest --run-load -m load -s # CPU load tests, a few minutes, prints a CPU timeline
|
|
336
|
+
python -m benchmarks.bench # timing / memory benchmark
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
## License
|
|
341
|
+
|
|
342
|
+
MIT - see [LICENSE](LICENSE).
|