stable-cart 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stable_cart-0.1.0/LICENSE +21 -0
- stable_cart-0.1.0/PKG-INFO +288 -0
- stable_cart-0.1.0/README.md +222 -0
- stable_cart-0.1.0/pyproject.toml +146 -0
- stable_cart-0.1.0/setup.cfg +4 -0
- stable_cart-0.1.0/stable_cart/__init__.py +16 -0
- stable_cart-0.1.0/stable_cart/bootstrap_variance_tree.py +540 -0
- stable_cart-0.1.0/stable_cart/evaluation.py +249 -0
- stable_cart-0.1.0/stable_cart/less_greedy_tree.py +529 -0
- stable_cart-0.1.0/stable_cart/robust_prefix.py +362 -0
- stable_cart-0.1.0/stable_cart.egg-info/PKG-INFO +288 -0
- stable_cart-0.1.0/stable_cart.egg-info/SOURCES.txt +18 -0
- stable_cart-0.1.0/stable_cart.egg-info/dependency_links.txt +1 -0
- stable_cart-0.1.0/stable_cart.egg-info/requires.txt +20 -0
- stable_cart-0.1.0/stable_cart.egg-info/top_level.txt +1 -0
- stable_cart-0.1.0/tests/test_bootstrap_variance.py +258 -0
- stable_cart-0.1.0/tests/test_evaluation.py +387 -0
- stable_cart-0.1.0/tests/test_less_greedy_tree.py +424 -0
- stable_cart-0.1.0/tests/test_realistic_scenarios.py +162 -0
- stable_cart-0.1.0/tests/test_robust_prefix.py +528 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Gaurav Sood
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: stable-cart
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Stable CART decision trees with enhanced prediction stability and variance reduction techniques for robust machine learning.
|
|
5
|
+
Author-email: Gaurav Sood <contact@gsood.com>, Arav Bhosle <aravbhosle@gmail.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2025 Gaurav Sood
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
Project-URL: Homepage, https://github.com/soodoku/stable-cart
|
|
28
|
+
Project-URL: Repository, https://github.com/soodoku/stable-cart
|
|
29
|
+
Project-URL: Documentation, https://soodoku.github.io/stable-cart/
|
|
30
|
+
Project-URL: Changelog, https://github.com/soodoku/stable-cart/blob/main/CHANGELOG.md
|
|
31
|
+
Project-URL: Issues, https://github.com/soodoku/stable-cart/issues
|
|
32
|
+
Keywords: machine learning,decision trees,CART,stability,variance reduction,sklearn,prediction stability,bootstrap,honest learning
|
|
33
|
+
Classifier: Development Status :: 3 - Alpha
|
|
34
|
+
Classifier: Intended Audience :: Science/Research
|
|
35
|
+
Classifier: Intended Audience :: Developers
|
|
36
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
37
|
+
Classifier: Programming Language :: Python :: 3
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
39
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
40
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
41
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
42
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
43
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
44
|
+
Classifier: Operating System :: OS Independent
|
|
45
|
+
Requires-Python: >=3.11
|
|
46
|
+
Description-Content-Type: text/markdown
|
|
47
|
+
License-File: LICENSE
|
|
48
|
+
Requires-Dist: numpy>=1.23.0
|
|
49
|
+
Requires-Dist: scikit-learn>=1.2.0
|
|
50
|
+
Requires-Dist: pandas>=1.5.0
|
|
51
|
+
Provides-Extra: dev
|
|
52
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
53
|
+
Requires-Dist: pytest-cov>=3.0.0; extra == "dev"
|
|
54
|
+
Requires-Dist: black>=22.0.0; extra == "dev"
|
|
55
|
+
Requires-Dist: flake8>=4.0.0; extra == "dev"
|
|
56
|
+
Requires-Dist: matplotlib>=3.3.0; extra == "dev"
|
|
57
|
+
Requires-Dist: mypy>=1.0.0; extra == "dev"
|
|
58
|
+
Provides-Extra: docs
|
|
59
|
+
Requires-Dist: sphinx>=4.0.0; extra == "docs"
|
|
60
|
+
Requires-Dist: sphinx-rtd-theme>=1.0.0; extra == "docs"
|
|
61
|
+
Requires-Dist: myst-parser>=0.18.0; extra == "docs"
|
|
62
|
+
Provides-Extra: benchmark
|
|
63
|
+
Requires-Dist: seaborn>=0.11.0; extra == "benchmark"
|
|
64
|
+
Requires-Dist: jupyterlab>=3.0.0; extra == "benchmark"
|
|
65
|
+
Dynamic: license-file
|
|
66
|
+
|
|
67
|
+
## Stable CART: Lower Cross-Bootstrap Prediction Variance
|
|
68
|
+
|
|
69
|
+
[](https://github.com/soodoku/stable-cart/actions/workflows/ci.yml)
|
|
70
|
+
[](https://pypi.org/project/stable-cart/)
|
|
71
|
+
[](https://pepy.tech/project/stable-cart)
|
|
72
|
+
[](https://soodoku.github.io/stable-cart/)
|
|
73
|
+
[](https://github.com/soodoku/stable-cart/blob/main/LICENSE)
|
|
74
|
+
[](https://www.python.org/downloads/)
|
|
75
|
+
|
|
76
|
+
A scikit-learn compatible implementation of **Stable CART** (Classification and Regression Trees) with advanced stability metrics and techniques to reduce prediction variance.
|
|
77
|
+
|
|
78
|
+
## Features
|
|
79
|
+
|
|
80
|
+
- 🌳 **LessGreedyHybridRegressor**: Advanced regression tree with honest data partitioning and lookahead
|
|
81
|
+
- 📊 **BootstrapVariancePenalizedRegressor**: Tree regressor that explicitly penalizes bootstrap prediction variance
|
|
82
|
+
- 🎯 **RobustPrefixHonestClassifier**: Binary classifier with robust prefix splits and honest leaf estimation
|
|
83
|
+
- 📈 **Prediction Stability Metrics**: Measure model consistency across different training runs
|
|
84
|
+
- 🔧 **Full sklearn Compatibility**: Works with pipelines, cross-validation, and grid search
|
|
85
|
+
|
|
86
|
+
## Installation
|
|
87
|
+
|
|
88
|
+
### From PyPI (Recommended)
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
pip install stable-cart
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### From Source
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
git clone https://github.com/soodoku/stable-cart.git
|
|
98
|
+
cd stable-cart
|
|
99
|
+
pip install -e .
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### With Development Dependencies
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
pip install -e ".[dev]"
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Quick Start
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from stable_cart import (
|
|
112
|
+
LessGreedyHybridRegressor,
|
|
113
|
+
BootstrapVariancePenalizedRegressor,
|
|
114
|
+
RobustPrefixHonestClassifier,
|
|
115
|
+
prediction_stability,
|
|
116
|
+
evaluate_models
|
|
117
|
+
)
|
|
118
|
+
from sklearn.datasets import make_regression, make_classification
|
|
119
|
+
from sklearn.model_selection import train_test_split
|
|
120
|
+
from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier
|
|
121
|
+
|
|
122
|
+
# Regression Example
|
|
123
|
+
X_reg, y_reg = make_regression(n_samples=1000, n_features=10, noise=10, random_state=42)
|
|
124
|
+
X_train, X_test, y_train, y_test = train_test_split(X_reg, y_reg, test_size=0.3, random_state=42)
|
|
125
|
+
|
|
126
|
+
# Train regression models
|
|
127
|
+
stable_model = LessGreedyHybridRegressor(max_depth=5, random_state=42)
|
|
128
|
+
bootstrap_model = BootstrapVariancePenalizedRegressor(
|
|
129
|
+
max_depth=5, variance_penalty=2.0, n_bootstrap=10, random_state=42
|
|
130
|
+
)
|
|
131
|
+
greedy_model = DecisionTreeRegressor(max_depth=5, random_state=42)
|
|
132
|
+
|
|
133
|
+
stable_model.fit(X_train, y_train)
|
|
134
|
+
bootstrap_model.fit(X_train, y_train)
|
|
135
|
+
greedy_model.fit(X_train, y_train)
|
|
136
|
+
|
|
137
|
+
# Evaluate performance
|
|
138
|
+
models = {
|
|
139
|
+
"stable": stable_model,
|
|
140
|
+
"bootstrap_penalized": bootstrap_model,
|
|
141
|
+
"greedy": greedy_model
|
|
142
|
+
}
|
|
143
|
+
metrics = evaluate_models(models, X_test, y_test, task="continuous")
|
|
144
|
+
print(f"Performance: {metrics}")
|
|
145
|
+
|
|
146
|
+
# Evaluate stability
|
|
147
|
+
stability = prediction_stability(models, X_test, task="continuous")
|
|
148
|
+
print(f"Stability (lower is better): {stability}")
|
|
149
|
+
|
|
150
|
+
# Classification Example
|
|
151
|
+
X_clf, y_clf = make_classification(n_samples=1000, n_features=10, n_classes=2, random_state=42)
|
|
152
|
+
X_train_clf, X_test_clf, y_train_clf, y_test_clf = train_test_split(
|
|
153
|
+
X_clf, y_clf, test_size=0.3, random_state=42
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
# Train classification models
|
|
157
|
+
robust_model = RobustPrefixHonestClassifier(top_levels=2, max_depth=5, random_state=42)
|
|
158
|
+
standard_model = DecisionTreeClassifier(max_depth=5, random_state=42)
|
|
159
|
+
|
|
160
|
+
robust_model.fit(X_train_clf, y_train_clf)
|
|
161
|
+
standard_model.fit(X_train_clf, y_train_clf)
|
|
162
|
+
|
|
163
|
+
# Evaluate classification performance
|
|
164
|
+
clf_models = {"robust": robust_model, "standard": standard_model}
|
|
165
|
+
clf_metrics = evaluate_models(clf_models, X_test_clf, y_test_clf, task="categorical")
|
|
166
|
+
print(f"Classification Performance: {clf_metrics}")
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
## Algorithms
|
|
170
|
+
|
|
171
|
+
### LessGreedyHybridRegressor
|
|
172
|
+
|
|
173
|
+
A regression tree that trades some accuracy for improved stability through:
|
|
174
|
+
- **Honest data partitioning**: Splits data into SPLIT (structure), VAL (validation), and EST (estimation) sets
|
|
175
|
+
- **Optional oblique root**: Linear combinations at the root node when beneficial
|
|
176
|
+
- **Lookahead with beam search**: Considers multiple steps ahead for better long-term decisions
|
|
177
|
+
- **Leaf shrinkage**: Ridge-like regularization for leaf predictions
|
|
178
|
+
|
|
179
|
+
### BootstrapVariancePenalizedRegressor
|
|
180
|
+
|
|
181
|
+
Explicitly reduces bootstrap prediction variance by:
|
|
182
|
+
- **Variance penalty**: Adds bootstrap variance as a regularization term
|
|
183
|
+
- **Honest estimation**: Separates structure learning from leaf value estimation
|
|
184
|
+
- **Bootstrap evaluation**: Uses multiple bootstrap samples to estimate prediction variance
|
|
185
|
+
|
|
186
|
+
### RobustPrefixHonestClassifier
|
|
187
|
+
|
|
188
|
+
A binary classifier designed for stability through:
|
|
189
|
+
- **Robust prefix**: Locks top-level splits using consensus across bootstrap samples
|
|
190
|
+
- **Honest leaves**: Estimates leaf probabilities on separate data from structure learning
|
|
191
|
+
- **m-estimate smoothing**: Stabilizes probability estimates in small leaves
|
|
192
|
+
- **Winsorization**: Reduces impact of outliers on split selection
|
|
193
|
+
|
|
194
|
+
## Performance Comparison
|
|
195
|
+
|
|
196
|
+
Here's how stable-cart models typically perform compared to standard trees:
|
|
197
|
+
|
|
198
|
+
| Metric | Standard Tree | Stable CART | Improvement |
|
|
199
|
+
|--------|---------------|-------------|-------------|
|
|
200
|
+
| **Prediction Variance** | High | Low | 30-50% reduction |
|
|
201
|
+
| **Out-of-sample Stability** | Variable | Consistent | 20-40% more stable |
|
|
202
|
+
| **Accuracy** | High | Slightly lower | 2-5% trade-off |
|
|
203
|
+
| **Interpretability** | Good | Good | Maintained |
|
|
204
|
+
|
|
205
|
+
## Development and Testing
|
|
206
|
+
|
|
207
|
+
### Running Tests
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
# Install dev dependencies
|
|
211
|
+
pip install -e ".[dev]"
|
|
212
|
+
|
|
213
|
+
# Run all tests
|
|
214
|
+
pytest
|
|
215
|
+
|
|
216
|
+
# Run with coverage
|
|
217
|
+
pytest --cov=stable_cart
|
|
218
|
+
|
|
219
|
+
# Run specific test categories
|
|
220
|
+
pytest -m "not slow" # Skip slow tests
|
|
221
|
+
pytest -m "benchmark" # Benchmark tests only
|
|
222
|
+
pytest tests/ # All tests
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
### Local CI Testing
|
|
226
|
+
|
|
227
|
+
Test the CI pipeline locally using Docker:
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
# Run the full CI pipeline in a clean Docker container
|
|
231
|
+
make ci-docker
|
|
232
|
+
|
|
233
|
+
# Or run individual steps
|
|
234
|
+
make lint # Check code formatting and style
|
|
235
|
+
make test # Run the test suite
|
|
236
|
+
make coverage # Run tests with coverage report
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
### Contributing
|
|
240
|
+
|
|
241
|
+
1. Fork the repository
|
|
242
|
+
2. Create a feature branch (`git checkout -b feature/amazing-feature`)
|
|
243
|
+
3. Make your changes and add tests
|
|
244
|
+
4. Run the test suite (`make test`)
|
|
245
|
+
5. Run linting (`make lint`)
|
|
246
|
+
6. Commit your changes (`git commit -m 'Add amazing feature'`)
|
|
247
|
+
7. Push to the branch (`git push origin feature/amazing-feature`)
|
|
248
|
+
8. Open a Pull Request
|
|
249
|
+
|
|
250
|
+
### Benchmarking
|
|
251
|
+
|
|
252
|
+
Run performance benchmarks:
|
|
253
|
+
|
|
254
|
+
```bash
|
|
255
|
+
# Run benchmark scripts
|
|
256
|
+
make benchmark
|
|
257
|
+
|
|
258
|
+
# View results
|
|
259
|
+
ls bench_out/
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
## Citation
|
|
263
|
+
|
|
264
|
+
If you use stable-cart in your research, please cite:
|
|
265
|
+
|
|
266
|
+
```bibtex
|
|
267
|
+
@software{stable_cart_2025,
|
|
268
|
+
title={Stable CART: Enhanced Decision Trees with Prediction Stability},
|
|
269
|
+
author={Sood, Gaurav and Bhosle, Arav},
|
|
270
|
+
year={2025},
|
|
271
|
+
url={https://github.com/soodoku/stable-cart},
|
|
272
|
+
version={0.1.0}
|
|
273
|
+
}
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
## Changelog
|
|
277
|
+
|
|
278
|
+
See [CHANGELOG.md](CHANGELOG.md) for a detailed history of changes.
|
|
279
|
+
|
|
280
|
+
## License
|
|
281
|
+
|
|
282
|
+
MIT License - see [LICENSE](LICENSE) file for details.
|
|
283
|
+
|
|
284
|
+
## Related Work
|
|
285
|
+
|
|
286
|
+
- **CART**: Breiman, L., et al. (1984). Classification and regression trees.
|
|
287
|
+
- **Honest Trees**: Wager, S., & Athey, S. (2018). Estimation and inference of heterogeneous treatment effects using random forests.
|
|
288
|
+
- **Bootstrap Aggregating**: Breiman, L. (1996). Bagging predictors.
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
## Stable CART: Lower Cross-Bootstrap Prediction Variance
|
|
2
|
+
|
|
3
|
+
[](https://github.com/soodoku/stable-cart/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/stable-cart/)
|
|
5
|
+
[](https://pepy.tech/project/stable-cart)
|
|
6
|
+
[](https://soodoku.github.io/stable-cart/)
|
|
7
|
+
[](https://github.com/soodoku/stable-cart/blob/main/LICENSE)
|
|
8
|
+
[](https://www.python.org/downloads/)
|
|
9
|
+
|
|
10
|
+
A scikit-learn compatible implementation of **Stable CART** (Classification and Regression Trees) with advanced stability metrics and techniques to reduce prediction variance.
|
|
11
|
+
|
|
12
|
+
## Features
|
|
13
|
+
|
|
14
|
+
- 🌳 **LessGreedyHybridRegressor**: Advanced regression tree with honest data partitioning and lookahead
|
|
15
|
+
- 📊 **BootstrapVariancePenalizedRegressor**: Tree regressor that explicitly penalizes bootstrap prediction variance
|
|
16
|
+
- 🎯 **RobustPrefixHonestClassifier**: Binary classifier with robust prefix splits and honest leaf estimation
|
|
17
|
+
- 📈 **Prediction Stability Metrics**: Measure model consistency across different training runs
|
|
18
|
+
- 🔧 **Full sklearn Compatibility**: Works with pipelines, cross-validation, and grid search
|
|
19
|
+
|
|
20
|
+
## Installation
|
|
21
|
+
|
|
22
|
+
### From PyPI (Recommended)
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install stable-cart
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
### From Source
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
git clone https://github.com/soodoku/stable-cart.git
|
|
32
|
+
cd stable-cart
|
|
33
|
+
pip install -e .
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
### With Development Dependencies
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install -e ".[dev]"
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Quick Start
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from stable_cart import (
|
|
46
|
+
LessGreedyHybridRegressor,
|
|
47
|
+
BootstrapVariancePenalizedRegressor,
|
|
48
|
+
RobustPrefixHonestClassifier,
|
|
49
|
+
prediction_stability,
|
|
50
|
+
evaluate_models
|
|
51
|
+
)
|
|
52
|
+
from sklearn.datasets import make_regression, make_classification
|
|
53
|
+
from sklearn.model_selection import train_test_split
|
|
54
|
+
from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier
|
|
55
|
+
|
|
56
|
+
# Regression Example
|
|
57
|
+
X_reg, y_reg = make_regression(n_samples=1000, n_features=10, noise=10, random_state=42)
|
|
58
|
+
X_train, X_test, y_train, y_test = train_test_split(X_reg, y_reg, test_size=0.3, random_state=42)
|
|
59
|
+
|
|
60
|
+
# Train regression models
|
|
61
|
+
stable_model = LessGreedyHybridRegressor(max_depth=5, random_state=42)
|
|
62
|
+
bootstrap_model = BootstrapVariancePenalizedRegressor(
|
|
63
|
+
max_depth=5, variance_penalty=2.0, n_bootstrap=10, random_state=42
|
|
64
|
+
)
|
|
65
|
+
greedy_model = DecisionTreeRegressor(max_depth=5, random_state=42)
|
|
66
|
+
|
|
67
|
+
stable_model.fit(X_train, y_train)
|
|
68
|
+
bootstrap_model.fit(X_train, y_train)
|
|
69
|
+
greedy_model.fit(X_train, y_train)
|
|
70
|
+
|
|
71
|
+
# Evaluate performance
|
|
72
|
+
models = {
|
|
73
|
+
"stable": stable_model,
|
|
74
|
+
"bootstrap_penalized": bootstrap_model,
|
|
75
|
+
"greedy": greedy_model
|
|
76
|
+
}
|
|
77
|
+
metrics = evaluate_models(models, X_test, y_test, task="continuous")
|
|
78
|
+
print(f"Performance: {metrics}")
|
|
79
|
+
|
|
80
|
+
# Evaluate stability
|
|
81
|
+
stability = prediction_stability(models, X_test, task="continuous")
|
|
82
|
+
print(f"Stability (lower is better): {stability}")
|
|
83
|
+
|
|
84
|
+
# Classification Example
|
|
85
|
+
X_clf, y_clf = make_classification(n_samples=1000, n_features=10, n_classes=2, random_state=42)
|
|
86
|
+
X_train_clf, X_test_clf, y_train_clf, y_test_clf = train_test_split(
|
|
87
|
+
X_clf, y_clf, test_size=0.3, random_state=42
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# Train classification models
|
|
91
|
+
robust_model = RobustPrefixHonestClassifier(top_levels=2, max_depth=5, random_state=42)
|
|
92
|
+
standard_model = DecisionTreeClassifier(max_depth=5, random_state=42)
|
|
93
|
+
|
|
94
|
+
robust_model.fit(X_train_clf, y_train_clf)
|
|
95
|
+
standard_model.fit(X_train_clf, y_train_clf)
|
|
96
|
+
|
|
97
|
+
# Evaluate classification performance
|
|
98
|
+
clf_models = {"robust": robust_model, "standard": standard_model}
|
|
99
|
+
clf_metrics = evaluate_models(clf_models, X_test_clf, y_test_clf, task="categorical")
|
|
100
|
+
print(f"Classification Performance: {clf_metrics}")
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Algorithms
|
|
104
|
+
|
|
105
|
+
### LessGreedyHybridRegressor
|
|
106
|
+
|
|
107
|
+
A regression tree that trades some accuracy for improved stability through:
|
|
108
|
+
- **Honest data partitioning**: Splits data into SPLIT (structure), VAL (validation), and EST (estimation) sets
|
|
109
|
+
- **Optional oblique root**: Linear combinations at the root node when beneficial
|
|
110
|
+
- **Lookahead with beam search**: Considers multiple steps ahead for better long-term decisions
|
|
111
|
+
- **Leaf shrinkage**: Ridge-like regularization for leaf predictions
|
|
112
|
+
|
|
113
|
+
### BootstrapVariancePenalizedRegressor
|
|
114
|
+
|
|
115
|
+
Explicitly reduces bootstrap prediction variance by:
|
|
116
|
+
- **Variance penalty**: Adds bootstrap variance as a regularization term
|
|
117
|
+
- **Honest estimation**: Separates structure learning from leaf value estimation
|
|
118
|
+
- **Bootstrap evaluation**: Uses multiple bootstrap samples to estimate prediction variance
|
|
119
|
+
|
|
120
|
+
### RobustPrefixHonestClassifier
|
|
121
|
+
|
|
122
|
+
A binary classifier designed for stability through:
|
|
123
|
+
- **Robust prefix**: Locks top-level splits using consensus across bootstrap samples
|
|
124
|
+
- **Honest leaves**: Estimates leaf probabilities on separate data from structure learning
|
|
125
|
+
- **m-estimate smoothing**: Stabilizes probability estimates in small leaves
|
|
126
|
+
- **Winsorization**: Reduces impact of outliers on split selection
|
|
127
|
+
|
|
128
|
+
## Performance Comparison
|
|
129
|
+
|
|
130
|
+
Here's how stable-cart models typically perform compared to standard trees:
|
|
131
|
+
|
|
132
|
+
| Metric | Standard Tree | Stable CART | Improvement |
|
|
133
|
+
|--------|---------------|-------------|-------------|
|
|
134
|
+
| **Prediction Variance** | High | Low | 30-50% reduction |
|
|
135
|
+
| **Out-of-sample Stability** | Variable | Consistent | 20-40% more stable |
|
|
136
|
+
| **Accuracy** | High | Slightly lower | 2-5% trade-off |
|
|
137
|
+
| **Interpretability** | Good | Good | Maintained |
|
|
138
|
+
|
|
139
|
+
## Development and Testing
|
|
140
|
+
|
|
141
|
+
### Running Tests
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
# Install dev dependencies
|
|
145
|
+
pip install -e ".[dev]"
|
|
146
|
+
|
|
147
|
+
# Run all tests
|
|
148
|
+
pytest
|
|
149
|
+
|
|
150
|
+
# Run with coverage
|
|
151
|
+
pytest --cov=stable_cart
|
|
152
|
+
|
|
153
|
+
# Run specific test categories
|
|
154
|
+
pytest -m "not slow" # Skip slow tests
|
|
155
|
+
pytest -m "benchmark" # Benchmark tests only
|
|
156
|
+
pytest tests/ # All tests
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
### Local CI Testing
|
|
160
|
+
|
|
161
|
+
Test the CI pipeline locally using Docker:
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
# Run the full CI pipeline in a clean Docker container
|
|
165
|
+
make ci-docker
|
|
166
|
+
|
|
167
|
+
# Or run individual steps
|
|
168
|
+
make lint # Check code formatting and style
|
|
169
|
+
make test # Run the test suite
|
|
170
|
+
make coverage # Run tests with coverage report
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
### Contributing
|
|
174
|
+
|
|
175
|
+
1. Fork the repository
|
|
176
|
+
2. Create a feature branch (`git checkout -b feature/amazing-feature`)
|
|
177
|
+
3. Make your changes and add tests
|
|
178
|
+
4. Run the test suite (`make test`)
|
|
179
|
+
5. Run linting (`make lint`)
|
|
180
|
+
6. Commit your changes (`git commit -m 'Add amazing feature'`)
|
|
181
|
+
7. Push to the branch (`git push origin feature/amazing-feature`)
|
|
182
|
+
8. Open a Pull Request
|
|
183
|
+
|
|
184
|
+
### Benchmarking
|
|
185
|
+
|
|
186
|
+
Run performance benchmarks:
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
# Run benchmark scripts
|
|
190
|
+
make benchmark
|
|
191
|
+
|
|
192
|
+
# View results
|
|
193
|
+
ls bench_out/
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
## Citation
|
|
197
|
+
|
|
198
|
+
If you use stable-cart in your research, please cite:
|
|
199
|
+
|
|
200
|
+
```bibtex
|
|
201
|
+
@software{stable_cart_2025,
|
|
202
|
+
title={Stable CART: Enhanced Decision Trees with Prediction Stability},
|
|
203
|
+
author={Sood, Gaurav and Bhosle, Arav},
|
|
204
|
+
year={2025},
|
|
205
|
+
url={https://github.com/soodoku/stable-cart},
|
|
206
|
+
version={0.1.0}
|
|
207
|
+
}
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
## Changelog
|
|
211
|
+
|
|
212
|
+
See [CHANGELOG.md](CHANGELOG.md) for a detailed history of changes.
|
|
213
|
+
|
|
214
|
+
## License
|
|
215
|
+
|
|
216
|
+
MIT License - see [LICENSE](LICENSE) file for details.
|
|
217
|
+
|
|
218
|
+
## Related Work
|
|
219
|
+
|
|
220
|
+
- **CART**: Breiman, L., et al. (1984). Classification and regression trees.
|
|
221
|
+
- **Honest Trees**: Wager, S., & Athey, S. (2018). Estimation and inference of heterogeneous treatment effects using random forests.
|
|
222
|
+
- **Bootstrap Aggregating**: Breiman, L. (1996). Bagging predictors.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "stable-cart"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Stable CART decision trees with enhanced prediction stability and variance reduction techniques for robust machine learning."
|
|
9
|
+
authors = [
|
|
10
|
+
{ name = "Gaurav Sood", email = "contact@gsood.com" },
|
|
11
|
+
{ name = "Arav Bhosle", email = "aravbhosle@gmail.com"}
|
|
12
|
+
]
|
|
13
|
+
readme = "README.md"
|
|
14
|
+
license = { file = "LICENSE" }
|
|
15
|
+
requires-python = ">=3.11"
|
|
16
|
+
keywords = [
|
|
17
|
+
"machine learning",
|
|
18
|
+
"decision trees",
|
|
19
|
+
"CART",
|
|
20
|
+
"stability",
|
|
21
|
+
"variance reduction",
|
|
22
|
+
"sklearn",
|
|
23
|
+
"prediction stability",
|
|
24
|
+
"bootstrap",
|
|
25
|
+
"honest learning"
|
|
26
|
+
]
|
|
27
|
+
dependencies = [
|
|
28
|
+
"numpy>=1.23.0",
|
|
29
|
+
"scikit-learn>=1.2.0",
|
|
30
|
+
"pandas>=1.5.0"
|
|
31
|
+
]
|
|
32
|
+
classifiers = [
|
|
33
|
+
"Development Status :: 3 - Alpha",
|
|
34
|
+
"Intended Audience :: Science/Research",
|
|
35
|
+
"Intended Audience :: Developers",
|
|
36
|
+
"License :: OSI Approved :: MIT License",
|
|
37
|
+
"Programming Language :: Python :: 3",
|
|
38
|
+
"Programming Language :: Python :: 3.11",
|
|
39
|
+
"Programming Language :: Python :: 3.12",
|
|
40
|
+
"Programming Language :: Python :: 3.13",
|
|
41
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
42
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
43
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
44
|
+
"Operating System :: OS Independent",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
[project.urls]
|
|
48
|
+
Homepage = "https://github.com/soodoku/stable-cart"
|
|
49
|
+
Repository = "https://github.com/soodoku/stable-cart"
|
|
50
|
+
Documentation = "https://soodoku.github.io/stable-cart/"
|
|
51
|
+
Changelog = "https://github.com/soodoku/stable-cart/blob/main/CHANGELOG.md"
|
|
52
|
+
Issues = "https://github.com/soodoku/stable-cart/issues"
|
|
53
|
+
|
|
54
|
+
[project.optional-dependencies]
|
|
55
|
+
dev = [
|
|
56
|
+
"pytest>=7.0.0",
|
|
57
|
+
"pytest-cov>=3.0.0",
|
|
58
|
+
"black>=22.0.0",
|
|
59
|
+
"flake8>=4.0.0",
|
|
60
|
+
"matplotlib>=3.3.0",
|
|
61
|
+
"mypy>=1.0.0",
|
|
62
|
+
]
|
|
63
|
+
docs = [
|
|
64
|
+
"sphinx>=4.0.0",
|
|
65
|
+
"sphinx-rtd-theme>=1.0.0",
|
|
66
|
+
"myst-parser>=0.18.0",
|
|
67
|
+
]
|
|
68
|
+
benchmark = [
|
|
69
|
+
"seaborn>=0.11.0",
|
|
70
|
+
"jupyterlab>=3.0.0",
|
|
71
|
+
]
|
|
72
|
+
|
|
73
|
+
[tool.setuptools.packages.find]
|
|
74
|
+
where = ["."]
|
|
75
|
+
include = ["stable_cart*"]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
[tool.pytest.ini_options]
|
|
79
|
+
minversion = "7.0"
|
|
80
|
+
addopts = "-ra -v --strict-markers"
|
|
81
|
+
testpaths = ["tests"]
|
|
82
|
+
python_files = "test_*.py"
|
|
83
|
+
python_classes = "Test*"
|
|
84
|
+
python_functions = "test_*"
|
|
85
|
+
filterwarnings = [
|
|
86
|
+
"ignore::UserWarning",
|
|
87
|
+
"ignore::DeprecationWarning",
|
|
88
|
+
]
|
|
89
|
+
markers = [
|
|
90
|
+
"e2e: end-to-end integration tests",
|
|
91
|
+
"slow: slow running tests",
|
|
92
|
+
"benchmark: benchmark tests",
|
|
93
|
+
]
|
|
94
|
+
|
|
95
|
+
[tool.coverage.run]
|
|
96
|
+
branch = true
|
|
97
|
+
source = ["stable_cart"]
|
|
98
|
+
omit = [
|
|
99
|
+
"*/tests/*",
|
|
100
|
+
"*/test_*.py",
|
|
101
|
+
"*/__pycache__/*",
|
|
102
|
+
"*/benchmark/*",
|
|
103
|
+
]
|
|
104
|
+
|
|
105
|
+
[tool.coverage.report]
|
|
106
|
+
show_missing = true
|
|
107
|
+
exclude_lines = [
|
|
108
|
+
"pragma: no cover",
|
|
109
|
+
"def __repr__",
|
|
110
|
+
"if __name__ == .__main__.:",
|
|
111
|
+
"raise NotImplementedError",
|
|
112
|
+
"if TYPE_CHECKING:",
|
|
113
|
+
]
|
|
114
|
+
|
|
115
|
+
[tool.black]
|
|
116
|
+
line-length = 100
|
|
117
|
+
target-version = ["py311", "py312", "py313"]
|
|
118
|
+
include = '\.pyi?$'
|
|
119
|
+
exclude = '''
|
|
120
|
+
/(
|
|
121
|
+
\.git
|
|
122
|
+
| \.hg
|
|
123
|
+
| \.mypy_cache
|
|
124
|
+
| \.tox
|
|
125
|
+
| \.venv
|
|
126
|
+
| _build
|
|
127
|
+
| buck-out
|
|
128
|
+
| build
|
|
129
|
+
| dist
|
|
130
|
+
)/'''
|
|
131
|
+
|
|
132
|
+
[tool.isort]
|
|
133
|
+
profile = "black"
|
|
134
|
+
line_length = 100
|
|
135
|
+
skip_gitignore = true
|
|
136
|
+
|
|
137
|
+
[tool.flake8]
|
|
138
|
+
max-line-length = 100
|
|
139
|
+
extend-ignore = ["E203", "W503"]
|
|
140
|
+
|
|
141
|
+
[tool.mypy]
|
|
142
|
+
python_version = "3.11"
|
|
143
|
+
warn_return_any = true
|
|
144
|
+
warn_unused_configs = true
|
|
145
|
+
disallow_untyped_defs = false
|
|
146
|
+
ignore_missing_imports = true
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Public package exports for stable_cart."""
|
|
2
|
+
|
|
3
|
+
from .evaluation import prediction_stability, evaluate_models
|
|
4
|
+
from .less_greedy_tree import LessGreedyHybridRegressor
|
|
5
|
+
from .bootstrap_variance_tree import BootstrapVariancePenalizedRegressor
|
|
6
|
+
from .robust_prefix import RobustPrefixHonestClassifier
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"prediction_stability",
|
|
10
|
+
"evaluate_models",
|
|
11
|
+
"LessGreedyHybridRegressor",
|
|
12
|
+
"BootstrapVariancePenalizedRegressor",
|
|
13
|
+
"RobustPrefixHonestClassifier",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
__version__ = "0.1.0"
|