linearboost 0.1.9__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- linearboost-0.2.1/.github/social-preview.png +0 -0
- linearboost-0.2.1/.github/workflows/ci.yml +39 -0
- linearboost-0.2.1/.gitignore +142 -0
- linearboost-0.2.1/.pre-commit-config.yaml +25 -0
- linearboost-0.2.1/LICENSE +21 -0
- {linearboost-0.1.9 → linearboost-0.2.1}/PKG-INFO +41 -161
- {linearboost-0.1.9 → linearboost-0.2.1}/README.md +36 -157
- linearboost-0.2.1/notebooks/demo_linearboost_usage.ipynb +1699 -0
- linearboost-0.2.1/notebooks/optuna_sefrboost_demo.ipynb +186 -0
- {linearboost-0.1.9 → linearboost-0.2.1}/pyproject.toml +3 -2
- {linearboost-0.1.9 → linearboost-0.2.1}/requirements.txt +2 -1
- linearboost-0.2.1/src/linearboost/__init__.py +12 -0
- {linearboost-0.1.9 → linearboost-0.2.1}/src/linearboost/linear_boost.py +7 -4
- linearboost-0.2.1/src/linearboost/sefr_boost.py +679 -0
- linearboost-0.2.1/tests/__init__.py +0 -0
- linearboost-0.2.1/tests/_utils.py +19 -0
- linearboost-0.2.1/tests/test_linearboost.py +860 -0
- linearboost-0.2.1/tests/test_sefr.py +535 -0
- {linearboost-0.1.9 → linearboost-0.2.1}/uv.lock +63 -27
- linearboost-0.1.9/LICENSE.txt +0 -29
- linearboost-0.1.9/src/linearboost/__init__.py +0 -9
- {linearboost-0.1.9 → linearboost-0.2.1}/SECURITY.md +0 -0
- {linearboost-0.1.9 → linearboost-0.2.1}/src/linearboost/_utils.py +0 -0
- {linearboost-0.1.9 → linearboost-0.2.1/src/linearboost}/licenses/BSD-3-Clause.txt +0 -0
- {linearboost-0.1.9 → linearboost-0.2.1}/src/linearboost/sefr.py +0 -0
|
Binary file
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
branches: [ main ]
|
|
6
|
+
workflow_call:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
test:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
strategy:
|
|
16
|
+
matrix:
|
|
17
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
18
|
+
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v4
|
|
21
|
+
- name: Set up Python
|
|
22
|
+
uses: actions/setup-python@v5
|
|
23
|
+
with:
|
|
24
|
+
python-version: ${{ matrix.python-version }}
|
|
25
|
+
|
|
26
|
+
- name: Set up uv
|
|
27
|
+
uses: astral-sh/setup-uv@v5
|
|
28
|
+
|
|
29
|
+
- name: Install dependencies
|
|
30
|
+
run: uv sync
|
|
31
|
+
|
|
32
|
+
- name: Run ruff linter
|
|
33
|
+
run: uv run ruff check --output-format=github .
|
|
34
|
+
|
|
35
|
+
- name: Run ruff format check
|
|
36
|
+
run: uv run ruff format --check --diff .
|
|
37
|
+
|
|
38
|
+
- name: Run tests
|
|
39
|
+
run: uv run pytest -vv
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
pip-wheel-metadata/
|
|
24
|
+
share/python-wheels/
|
|
25
|
+
*.egg-info/
|
|
26
|
+
.installed.cfg
|
|
27
|
+
*.egg
|
|
28
|
+
MANIFEST
|
|
29
|
+
|
|
30
|
+
# PyInstaller
|
|
31
|
+
# Usually these files are written by a python script from a template
|
|
32
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
33
|
+
*.manifest
|
|
34
|
+
*.spec
|
|
35
|
+
|
|
36
|
+
# Installer logs
|
|
37
|
+
pip-log.txt
|
|
38
|
+
pip-delete-this-directory.txt
|
|
39
|
+
|
|
40
|
+
# Unit test / coverage reports
|
|
41
|
+
htmlcov/
|
|
42
|
+
.tox/
|
|
43
|
+
.nox/
|
|
44
|
+
.coverage
|
|
45
|
+
.coverage.*
|
|
46
|
+
.cache
|
|
47
|
+
nosetests.xml
|
|
48
|
+
coverage.xml
|
|
49
|
+
*.cover
|
|
50
|
+
*.py,cover
|
|
51
|
+
.hypothesis/
|
|
52
|
+
.pytest_cache/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
target/
|
|
76
|
+
|
|
77
|
+
# Jupyter Notebook
|
|
78
|
+
.ipynb_checkpoints
|
|
79
|
+
|
|
80
|
+
# IPython
|
|
81
|
+
profile_default/
|
|
82
|
+
ipython_config.py
|
|
83
|
+
|
|
84
|
+
# pyenv
|
|
85
|
+
.python-version
|
|
86
|
+
|
|
87
|
+
# pipenv
|
|
88
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
89
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
90
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
91
|
+
# install all needed dependencies.
|
|
92
|
+
#Pipfile.lock
|
|
93
|
+
|
|
94
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
|
95
|
+
__pypackages__/
|
|
96
|
+
|
|
97
|
+
# Celery stuff
|
|
98
|
+
celerybeat-schedule
|
|
99
|
+
celerybeat.pid
|
|
100
|
+
|
|
101
|
+
# SageMath parsed files
|
|
102
|
+
*.sage.py
|
|
103
|
+
|
|
104
|
+
# Environments
|
|
105
|
+
.env
|
|
106
|
+
.venv
|
|
107
|
+
env/
|
|
108
|
+
venv/
|
|
109
|
+
ENV/
|
|
110
|
+
env.bak/
|
|
111
|
+
venv.bak/
|
|
112
|
+
benchmark/
|
|
113
|
+
|
|
114
|
+
# Spyder project settings
|
|
115
|
+
.spyderproject
|
|
116
|
+
.spyproject
|
|
117
|
+
|
|
118
|
+
# Rope project settings
|
|
119
|
+
.ropeproject
|
|
120
|
+
|
|
121
|
+
# mkdocs documentation
|
|
122
|
+
/site
|
|
123
|
+
|
|
124
|
+
# mypy
|
|
125
|
+
.mypy_cache/
|
|
126
|
+
.dmypy.json
|
|
127
|
+
dmypy.json
|
|
128
|
+
|
|
129
|
+
# Pyre type checker
|
|
130
|
+
.pyre/
|
|
131
|
+
|
|
132
|
+
#PyCharm
|
|
133
|
+
.idea/
|
|
134
|
+
catboost_info/catboost_training.json
|
|
135
|
+
catboost_info/learn/events.out.tfevents
|
|
136
|
+
catboost_info/learn_error.tsv
|
|
137
|
+
catboost_info/time_left.tsv
|
|
138
|
+
notebooks/catboost_info/catboost_training.json
|
|
139
|
+
notebooks/catboost_info/learn/events.out.tfevents
|
|
140
|
+
notebooks/catboost_info/learn_error.tsv
|
|
141
|
+
notebooks/catboost_info/time_left.tsv
|
|
142
|
+
.DS_Store
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
default_language_version:
|
|
2
|
+
python: python3.10
|
|
3
|
+
|
|
4
|
+
repos:
|
|
5
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
6
|
+
rev: v5.0.0
|
|
7
|
+
hooks:
|
|
8
|
+
- id: check-added-large-files
|
|
9
|
+
- id: check-toml
|
|
10
|
+
- id: check-yaml
|
|
11
|
+
- id: end-of-file-fixer
|
|
12
|
+
- id: trailing-whitespace
|
|
13
|
+
|
|
14
|
+
- repo: https://github.com/pycqa/isort
|
|
15
|
+
rev: 6.0.1
|
|
16
|
+
hooks:
|
|
17
|
+
- id: isort
|
|
18
|
+
|
|
19
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
20
|
+
rev: v0.11.12
|
|
21
|
+
hooks:
|
|
22
|
+
- id: ruff
|
|
23
|
+
args:
|
|
24
|
+
- --fix
|
|
25
|
+
- id: ruff-format
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) Hamidreza Keshavarz, Reza Rawassizadeh
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: linearboost
|
|
3
|
-
Version: 0.1
|
|
4
|
-
Summary: LinearBoost
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: LinearBoost: a boosting classifier built on the linear classifier SEFR.
|
|
5
5
|
Project-URL: Homepage, https://github.com/LinearBoost/linearboost-classifier
|
|
6
6
|
Project-URL: Documentation, https://linearboost.readthedocs.io
|
|
7
7
|
Project-URL: Repository, https://github.com/LinearBoost/linearboost-classifier
|
|
8
8
|
Project-URL: Issues, https://github.com/LinearBoost/linearboost-classifier/issues
|
|
9
9
|
Author-email: Hamidreza Keshavarz <hamid9@outlook.com>, Reza Rawassizadeh <rezar@bu.edu>
|
|
10
|
-
License-File: LICENSE
|
|
10
|
+
License-File: LICENSE
|
|
11
11
|
Keywords: adaboost,boost,boosting,classification,classifier,linear
|
|
12
12
|
Classifier: Intended Audience :: Developers
|
|
13
13
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -20,18 +20,23 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
|
20
20
|
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
21
21
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
22
|
Requires-Python: <3.14,>=3.10
|
|
23
|
+
Requires-Dist: packaging>=20.0
|
|
23
24
|
Requires-Dist: scikit-learn>=1.2.2
|
|
24
25
|
Requires-Dist: typing-extensions>=4.1.0; python_full_version < '3.11'
|
|
25
26
|
Description-Content-Type: text/markdown
|
|
26
27
|
|
|
27
|
-
# LinearBoost
|
|
28
|
+
# LinearBoost and SEFRBoost Classifiers
|
|
28
29
|
|
|
29
|
-

|
|
30
31
|
[](https://pypi.org/project/linearboost/)
|
|
31
32
|

|
|
32
33
|
[](https://pepy.tech/projects/linearboost)
|
|
33
34
|
|
|
34
|
-
|
|
35
|
+
**Current release: v0.2.0.** This version adds **SEFRBoost**—gradient boosting with linear SEFR splits at tree nodes—via `SEFRBoostClassifier` and `SEFRBoostRegressor` (`from linearboost import …` or `linearboost.sefr_boost`). It is pronounced /sɛfr-buːst/. **LinearBoost security issues have been updated** in this release; upgrade from earlier versions to stay patched.
|
|
36
|
+
|
|
37
|
+
## 🧪 Quickstart demos
|
|
38
|
+
|
|
39
|
+
### LinearBoost
|
|
35
40
|
|
|
36
41
|
Want to see how LinearBoost works in practice?
|
|
37
42
|
|
|
@@ -42,19 +47,38 @@ This Jupyter notebook shows how to:
|
|
|
42
47
|
- Train `LinearBoostClassifier`
|
|
43
48
|
- Evaluate using F1 score and cross-validation
|
|
44
49
|
|
|
45
|
-
|
|
50
|
+
### SEFRBoost
|
|
51
|
+
|
|
52
|
+
- **[`optuna_sefrboost_demo.ipynb`](notebooks/optuna_sefrboost_demo.ipynb)** — Hyperparameter search for **`SEFRBoostClassifier`** with **[Optuna](https://optuna.org/)** on sklearn’s Breast Cancer Wisconsin data: default baseline, then **5-fold stratified CV** optimizing F1 (install `optuna` in addition to `linearboost` and `scikit-learn`).
|
|
53
|
+
|
|
54
|
+
LinearBoost is a boosting classifier built on the linear classifier SEFR: it trains SEFR classifiers in sequence with AdaBoost-style sample reweighting (SAMME or SAMME.R), optionally on kernel features. Training and prediction are fast, and memory use is low.
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## 🚀 New in Version 0.2.0
|
|
59
|
+
|
|
60
|
+
### SEFRBoost
|
|
46
61
|
|
|
47
|
-
|
|
62
|
+
SEFRBoost provides binary classification and regression with shallow trees whose internal nodes use **SEFR** hyperplane splits on pseudo-residuals—an oblique-split GBDT-style alternative that lives alongside `LinearBoostClassifier`.
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
from linearboost import SEFRBoostClassifier, SEFRBoostRegressor
|
|
66
|
+
# or: from linearboost.sefr_boost import SEFRBoostClassifier, SEFRBoostRegressor
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Hands-on examples: [`notebooks/optuna_sefrboost_demo.ipynb`](notebooks/optuna_sefrboost_demo.ipynb) (Optuna tuning).
|
|
70
|
+
|
|
71
|
+
### Security updates
|
|
72
|
+
|
|
73
|
+
LinearBoost-related security issues are addressed in v0.2.0. Upgrade from older releases.
|
|
74
|
+
|
|
75
|
+
---
|
|
48
76
|
|
|
49
|
-
|
|
50
|
-
- Surpassed LightGBM on five datasets
|
|
51
|
-
- Achieved up to **98% faster runtime** compared to both algorithms
|
|
77
|
+
## 🚀 New in Version 0.1.9
|
|
52
78
|
|
|
53
|
-
|
|
79
|
+
### Security Updates
|
|
54
80
|
|
|
55
|
-
|
|
56
|
-
- Exceptional Speed: Blazing fast training and inference times
|
|
57
|
-
- Resource Efficient: Low memory usage, ideal for large datasets
|
|
81
|
+
Version 0.1.9 included security updates. For the latest fixes, use **v0.2.0** or newer.
|
|
58
82
|
|
|
59
83
|
---
|
|
60
84
|
|
|
@@ -225,7 +249,7 @@ The documentation is available at https://linearboost.readthedocs.io/.
|
|
|
225
249
|
|
|
226
250
|
## Recommended Parameters for LinearBoost
|
|
227
251
|
|
|
228
|
-
|
|
252
|
+
Suggested starting values for the main parameters:
|
|
229
253
|
|
|
230
254
|
- **`n_estimators`**:
|
|
231
255
|
A range of 10 to 200 is suggested, with higher values potentially improving performance at the cost of longer training times. When using `early_stopping=True`, you can set a higher value (e.g., 500) and let training stop automatically.
|
|
@@ -278,150 +302,6 @@ The following parameters yielded optimal results during testing. All results are
|
|
|
278
302
|
|
|
279
303
|
These parameters should serve as a solid starting point for most datasets. For fine-tuning, consider using hyperparameter optimization tools like [Optuna](https://optuna.org/).
|
|
280
304
|
|
|
281
|
-
## Results
|
|
282
|
-
|
|
283
|
-
All of the results are reported based on 10-fold Cross-Validation. The weighted F1 score is reported, i.e. f1_score(y_valid, y_pred, average = 'weighted').
|
|
284
|
-
|
|
285
|
-
## Performance Comparison: F1 Scores Across Datasets
|
|
286
|
-
|
|
287
|
-
The following table presents the F1 scores of LinearBoost in comparison with XGBoost, CatBoost, and LightGBM across seven standard benchmark datasets. **Each result is obtained by running Optuna with 200 trials** to find the best hyperparameters for each algorithm and dataset, ensuring a fair and robust comparison.
|
|
288
|
-
|
|
289
|
-
| Dataset | XGBoost | CatBoost | LightGBM | LinearBoost |
|
|
290
|
-
|--------------------------------------|----------|----------|----------|-------------|
|
|
291
|
-
| Breast Cancer Wisconsin (Diagnostic) | 0.9767 | 0.9859 | 0.9771 | 0.9822 |
|
|
292
|
-
| Heart Disease | 0.8502 | 0.8529 | 0.8467 | 0.8507 |
|
|
293
|
-
| Pima Indians Diabetes Database | 0.7719 | 0.7776 | 0.7816 | 0.7753 |
|
|
294
|
-
| Banknote Authentication | 0.9985 | 1.0000 | 0.9993 | 1.0000 |
|
|
295
|
-
| Haberman's Survival | 0.7193 | 0.7427 | 0.7257 | 0.7485 |
|
|
296
|
-
| Loan Status Prediction | 0.8281 | 0.8495 | 0.8277 | 0.8387 |
|
|
297
|
-
| PCMAC | 0.9310 | 0.9351 | 0.9361 | 0.9331 |
|
|
298
|
-
|
|
299
|
-
### Experiment Details
|
|
300
|
-
- **Hyperparameter Optimization**:
|
|
301
|
-
- Each algorithm was tuned using **Optuna**, a powerful hyperparameter optimization framework.
|
|
302
|
-
- **200 trials** were conducted for each algorithm-dataset pair to identify the optimal hyperparameters.
|
|
303
|
-
- **Consistency**: This rigorous approach ensures fair comparison by evaluating each algorithm under its best-performing configuration.
|
|
304
|
-
|
|
305
|
-
### Key Highlights
|
|
306
|
-
- **LinearBoost** achieves competitive or superior F1 scores compared to the state-of-the-art algorithms.
|
|
307
|
-
- **Haberman's Survival**: LinearBoost achieves the highest F1 score (**0.7485**), outperforming all other algorithms.
|
|
308
|
-
- **Banknote Authentication**: LinearBoost matches the perfect F1 score of **1** achieved by CatBoost.
|
|
309
|
-
- LinearBoost demonstrates consistent performance across diverse datasets, making it a robust and efficient choice for classification tasks.
|
|
310
|
-
|
|
311
|
-
## Runtime Comparison: Time to Reach Best F1 Score
|
|
312
|
-
|
|
313
|
-
The following table shows the runtime (in seconds) required by LinearBoost, XGBoost, CatBoost, and LightGBM to achieve their best F1 scores. **Each result is obtained by running Optuna with 200 trials** to optimize the hyperparameters for each algorithm and dataset.
|
|
314
|
-
|
|
315
|
-
| Dataset | XGBoost | CatBoost | LightGBM | LinearBoost |
|
|
316
|
-
|--------------------------------------|----------|----------|----------|-------------|
|
|
317
|
-
| Breast Cancer Wisconsin (Diagnostic) | 3.22 | 9.68 | 4.52 | 0.30 |
|
|
318
|
-
| Heart Disease | 1.13 | 0.60 | 0.51 | 0.49 |
|
|
319
|
-
| Pima Indians Diabetes Database | 6.86 | 3.50 | 2.52 | 0.16 |
|
|
320
|
-
| Banknote Authentication | 0.46 | 4.26 | 5.54 | 0.33 |
|
|
321
|
-
| Haberman's Survival | 4.41 | 8.28 | 5.72 | 0.11 |
|
|
322
|
-
| Loan Status Prediction | 0.83 | 97.89 | 28.41 | 0.44 |
|
|
323
|
-
| PCMAC | 150.33 | 83.52 | 42.23 | 75.06 |
|
|
324
|
-
|
|
325
|
-
### Experiment Details
|
|
326
|
-
- **Hyperparameter Optimization**:
|
|
327
|
-
- Each algorithm was tuned using **Optuna** with **200 trials** per algorithm-dataset pair.
|
|
328
|
-
- The runtime includes the time to reach the best F1 score using the optimized hyperparameters.
|
|
329
|
-
- **Fair Comparison**: All algorithms were evaluated under their best configurations to ensure consistency.
|
|
330
|
-
|
|
331
|
-
### Key Highlights
|
|
332
|
-
- **LinearBoost** demonstrates exceptional runtime efficiency while achieving competitive F1 scores:
|
|
333
|
-
- **Breast Cancer Wisconsin (Diagnostic)**: LinearBoost achieves the best F1 score in just **0.30 seconds**, compared to **3.22 seconds** for XGBoost and **9.68 seconds** for CatBoost.
|
|
334
|
-
- **Loan Status Prediction**: LinearBoost runs in **0.44 seconds**, outperforming LightGBM (**28.41 seconds**) and CatBoost (**97.89 seconds**).
|
|
335
|
-
- Across most datasets, LinearBoost reduces runtime by up to **98%** compared to XGBoost and LightGBM while maintaining competitive performance.
|
|
336
|
-
|
|
337
|
-
### Tuned Hyperparameters
|
|
338
|
-
|
|
339
|
-
#### XGBoost
|
|
340
|
-
```python
|
|
341
|
-
params = {
|
|
342
|
-
'objective': 'binary:logistic',
|
|
343
|
-
'use_label_encoder': False,
|
|
344
|
-
'n_estimators': trial.suggest_int('n_estimators', 20, 1000),
|
|
345
|
-
'max_depth': trial.suggest_int('max_depth', 1, 20),
|
|
346
|
-
'learning_rate': trial.suggest_uniform('learning_rate', 0.01, 0.7),
|
|
347
|
-
'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),
|
|
348
|
-
'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),
|
|
349
|
-
'subsample': trial.suggest_float('subsample', 0.5, 1.0),
|
|
350
|
-
'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),
|
|
351
|
-
'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 1.0),
|
|
352
|
-
'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 1.0),
|
|
353
|
-
'enable_categorical': True,
|
|
354
|
-
'eval_metric': 'logloss'
|
|
355
|
-
}
|
|
356
|
-
```
|
|
357
|
-
|
|
358
|
-
#### CatBoost
|
|
359
|
-
```python
|
|
360
|
-
params = {
|
|
361
|
-
'iterations': trial.suggest_int('iterations', 50, 500),
|
|
362
|
-
'depth': trial.suggest_int('depth', 1, 16),
|
|
363
|
-
'learning_rate': trial.suggest_loguniform('learning_rate', 1e-3, 0.5),
|
|
364
|
-
'l2_leaf_reg': trial.suggest_loguniform('l2_leaf_reg', 1e-8, 10.0),
|
|
365
|
-
'random_strength': trial.suggest_loguniform('random_strength', 1e-8, 10.0),
|
|
366
|
-
'bagging_temperature': trial.suggest_loguniform('bagging_temperature', 1e-1, 10.0),
|
|
367
|
-
'border_count': trial.suggest_int('border_count', 32, 255),
|
|
368
|
-
'grow_policy': trial.suggest_categorical('grow_policy', ['SymmetricTree', 'Depthwise', 'Lossguide']),
|
|
369
|
-
'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 1, 100),
|
|
370
|
-
'rsm': trial.suggest_uniform('rsm', 0.1, 1.0),
|
|
371
|
-
'loss_function': 'Logloss',
|
|
372
|
-
'eval_metric': 'F1',
|
|
373
|
-
'cat_features': categorical_cols
|
|
374
|
-
}
|
|
375
|
-
```
|
|
376
|
-
|
|
377
|
-
#### LightGBM
|
|
378
|
-
```python
|
|
379
|
-
params = {
|
|
380
|
-
'objective': 'binary',
|
|
381
|
-
'metric': 'binary_logloss',
|
|
382
|
-
'boosting_type': trial.suggest_categorical('boosting_type', ['gbdt', 'dart', 'goss']),
|
|
383
|
-
'num_leaves': trial.suggest_int('num_leaves', 2, 256),
|
|
384
|
-
'learning_rate': trial.suggest_loguniform('learning_rate', 1e-3, 0.1),
|
|
385
|
-
'n_estimators': trial.suggest_int('n_estimators', 20, 1000),
|
|
386
|
-
'max_depth': trial.suggest_int('max_depth', 1, 20),
|
|
387
|
-
'min_child_samples': trial.suggest_int('min_child_samples', 1, 100),
|
|
388
|
-
'subsample': trial.suggest_uniform('subsample', 0.5, 1.0),
|
|
389
|
-
'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.5, 1.0),
|
|
390
|
-
'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 10.0),
|
|
391
|
-
'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 10.0),
|
|
392
|
-
'min_split_gain': trial.suggest_loguniform('min_split_gain', 1e-8, 1.0),
|
|
393
|
-
'cat_smooth': trial.suggest_int('cat_smooth', 1, 100),
|
|
394
|
-
'cat_l2': trial.suggest_loguniform('cat_l2', 1e-8, 10.0),
|
|
395
|
-
'verbosity': -1
|
|
396
|
-
}
|
|
397
|
-
```
|
|
398
|
-
|
|
399
|
-
#### LinearBoost
|
|
400
|
-
```python
|
|
401
|
-
params = {
|
|
402
|
-
'n_estimators': trial.suggest_int('n_estimators', 10, 500),
|
|
403
|
-
'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 1),
|
|
404
|
-
'algorithm': trial.suggest_categorical('algorithm', ['SAMME', 'SAMME.R']),
|
|
405
|
-
'scaler': trial.suggest_categorical('scaler', ['minmax', 'robust', 'quantile-uniform', 'quantile-normal']),
|
|
406
|
-
'kernel': trial.suggest_categorical('kernel', ['linear', 'rbf', 'poly']),
|
|
407
|
-
'boosting_type': trial.suggest_categorical('boosting_type', ['adaboost', 'gradient']),
|
|
408
|
-
'class_weight': trial.suggest_categorical('class_weight', [None, 'balanced']),
|
|
409
|
-
'subsample': trial.suggest_float('subsample', 0.6, 1.0),
|
|
410
|
-
'shrinkage': trial.suggest_float('shrinkage', 0.7, 1.0),
|
|
411
|
-
'early_stopping': True,
|
|
412
|
-
'n_iter_no_change': 5,
|
|
413
|
-
}
|
|
414
|
-
```
|
|
415
|
-
|
|
416
|
-
### Why LinearBoost?
|
|
417
|
-
LinearBoost's combination of **runtime efficiency** and **high accuracy** makes it a powerful choice for real-world machine learning tasks, particularly in resource-constrained or real-time applications.
|
|
418
|
-
|
|
419
|
-
### 📰 Featured in:
|
|
420
|
-
- [LightGBM Alternatives: A Comprehensive Comparison](https://nightwatcherai.com/blog/lightgbm-alternatives)
|
|
421
|
-
_by Jordan Cole, March 11, 2025_
|
|
422
|
-
*Discusses how LinearBoost outperforms traditional boosting frameworks in terms of speed while maintaining accuracy.*
|
|
423
|
-
|
|
424
|
-
|
|
425
305
|
## Future Developments
|
|
426
306
|
|
|
427
307
|
These are not yet supported in this current version, but are in the future plans:
|