raman-bench 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raman_bench-0.1.0/LICENSE +21 -0
- raman_bench-0.1.0/PKG-INFO +535 -0
- raman_bench-0.1.0/README.md +440 -0
- raman_bench-0.1.0/pyproject.toml +144 -0
- raman_bench-0.1.0/setup.cfg +4 -0
- raman_bench-0.1.0/src/raman_bench/__init__.py +76 -0
- raman_bench-0.1.0/src/raman_bench/benchmark.py +444 -0
- raman_bench-0.1.0/src/raman_bench/cli.py +171 -0
- raman_bench-0.1.0/src/raman_bench/config.py +113 -0
- raman_bench-0.1.0/src/raman_bench/custom_models/__init__.py +8 -0
- raman_bench-0.1.0/src/raman_bench/custom_models/base.py +1 -0
- raman_bench-0.1.0/src/raman_bench/custom_models/coatnet.py +1 -0
- raman_bench-0.1.0/src/raman_bench/custom_models/deepcnn.py +1 -0
- raman_bench-0.1.0/src/raman_bench/custom_models/fcresnext.py +1 -0
- raman_bench-0.1.0/src/raman_bench/custom_models/pls.py +1 -0
- raman_bench-0.1.0/src/raman_bench/custom_models/ramanformer.py +1 -0
- raman_bench-0.1.0/src/raman_bench/custom_models/ramannet.py +1 -0
- raman_bench-0.1.0/src/raman_bench/custom_models/ramantransformer.py +1 -0
- raman_bench-0.1.0/src/raman_bench/custom_models/rezeronet.py +1 -0
- raman_bench-0.1.0/src/raman_bench/custom_models/sanet.py +1 -0
- raman_bench-0.1.0/src/raman_bench/custom_models/sktime_models.py +1 -0
- raman_bench-0.1.0/src/raman_bench/data/__init__.py +0 -0
- raman_bench-0.1.0/src/raman_bench/data/precomputed/__init__.py +0 -0
- raman_bench-0.1.0/src/raman_bench/data/precomputed/classification_metrics.csv +1828 -0
- raman_bench-0.1.0/src/raman_bench/data/precomputed/dataset_stats.json +3829 -0
- raman_bench-0.1.0/src/raman_bench/data/precomputed/datasets.csv +78 -0
- raman_bench-0.1.0/src/raman_bench/data/precomputed/leaderboard_clf.csv +29 -0
- raman_bench-0.1.0/src/raman_bench/data/precomputed/leaderboard_overall.csv +29 -0
- raman_bench-0.1.0/src/raman_bench/data/precomputed/leaderboard_reg.csv +27 -0
- raman_bench-0.1.0/src/raman_bench/data/precomputed/regression_metrics.csv +10450 -0
- raman_bench-0.1.0/src/raman_bench/data/precomputed/score_params.json +614 -0
- raman_bench-0.1.0/src/raman_bench/evaluation.py +253 -0
- raman_bench-0.1.0/src/raman_bench/leaderboard.py +603 -0
- raman_bench-0.1.0/src/raman_bench/logging_utils.py +44 -0
- raman_bench-0.1.0/src/raman_bench/metrics/__init__.py +11 -0
- raman_bench-0.1.0/src/raman_bench/metrics/classification.py +101 -0
- raman_bench-0.1.0/src/raman_bench/metrics/regression.py +65 -0
- raman_bench-0.1.0/src/raman_bench/metrics/utils.py +82 -0
- raman_bench-0.1.0/src/raman_bench/model.py +419 -0
- raman_bench-0.1.0/src/raman_bench/models/__init__.py +27 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/__init__.py +63 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/base.py +291 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/coatnet.py +198 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/deepcnn.py +128 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/fcresnext.py +150 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/pls.py +68 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/ramanformer.py +180 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/ramannet.py +144 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/ramantransformer.py +153 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/rezeronet.py +211 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/sanet.py +187 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/sktime_models.py +120 -0
- raman_bench-0.1.0/src/raman_bench/models/custom/tabular_foundation.py +205 -0
- raman_bench-0.1.0/src/raman_bench/predictions.py +563 -0
- raman_bench-0.1.0/src/raman_bench/preprocessing/__init__.py +46 -0
- raman_bench-0.1.0/src/raman_bench/preprocessing/mixin.py +461 -0
- raman_bench-0.1.0/src/raman_bench/preprocessing/raman_preprocessing.py +460 -0
- raman_bench-0.1.0/src/raman_bench/preprocessing/wrapped_models.py +437 -0
- raman_bench-0.1.0/src/raman_bench/seeds.py +25 -0
- raman_bench-0.1.0/src/raman_bench.egg-info/PKG-INFO +535 -0
- raman_bench-0.1.0/src/raman_bench.egg-info/SOURCES.txt +67 -0
- raman_bench-0.1.0/src/raman_bench.egg-info/dependency_links.txt +1 -0
- raman_bench-0.1.0/src/raman_bench.egg-info/entry_points.txt +2 -0
- raman_bench-0.1.0/src/raman_bench.egg-info/requires.txt +48 -0
- raman_bench-0.1.0/src/raman_bench.egg-info/top_level.txt +1 -0
- raman_bench-0.1.0/tests/test_config.py +55 -0
- raman_bench-0.1.0/tests/test_leaderboard.py +73 -0
- raman_bench-0.1.0/tests/test_metrics.py +73 -0
- raman_bench-0.1.0/tests/test_preprocessing.py +59 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 KI-Werkstatt
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,535 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: raman-bench
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A large-scale benchmark for machine learning on Raman spectroscopy data
|
|
5
|
+
Author-email: Mario Koddenbrock <mario.koddenbrock@htw-berlin.de>, Christoph Lange <christoph.lange@tu-berlin.de>
|
|
6
|
+
Maintainer-email: Mario Koddenbrock <mario.koddenbrock@htw-berlin.de>
|
|
7
|
+
License: MIT License
|
|
8
|
+
|
|
9
|
+
Copyright (c) 2026 KI-Werkstatt
|
|
10
|
+
|
|
11
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
in the Software without restriction, including without limitation the rights
|
|
14
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
furnished to do so, subject to the following conditions:
|
|
17
|
+
|
|
18
|
+
The above copyright notice and this permission notice shall be included in all
|
|
19
|
+
copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
27
|
+
SOFTWARE.
|
|
28
|
+
|
|
29
|
+
Project-URL: Homepage, https://github.com/ml-lab-htw/RamanBench
|
|
30
|
+
Project-URL: Documentation, https://ramanbench.readthedocs.io
|
|
31
|
+
Project-URL: Repository, https://github.com/ml-lab-htw/RamanBench
|
|
32
|
+
Project-URL: Bug Tracker, https://github.com/ml-lab-htw/RamanBench/issues
|
|
33
|
+
Project-URL: Leaderboard, https://huggingface.co/spaces/HTW-KI-Werkstatt/RamanBench
|
|
34
|
+
Project-URL: raman-data, https://github.com/ml-lab-htw/raman_data
|
|
35
|
+
Project-URL: Paper, https://arxiv.org/abs/2605.02003
|
|
36
|
+
Keywords: raman,spectroscopy,benchmark,machine-learning,deep-learning,autogluon,tabular,chemistry,biology,material-science
|
|
37
|
+
Classifier: Development Status :: 3 - Alpha
|
|
38
|
+
Classifier: Intended Audience :: Science/Research
|
|
39
|
+
Classifier: Intended Audience :: Developers
|
|
40
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
41
|
+
Classifier: Operating System :: OS Independent
|
|
42
|
+
Classifier: Programming Language :: Python :: 3
|
|
43
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
44
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
45
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
46
|
+
Classifier: Topic :: Scientific/Engineering :: Chemistry
|
|
47
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
48
|
+
Classifier: Topic :: Scientific/Engineering :: Physics
|
|
49
|
+
Requires-Python: >=3.11
|
|
50
|
+
Description-Content-Type: text/markdown
|
|
51
|
+
License-File: LICENSE
|
|
52
|
+
Requires-Dist: numpy<2.0,>=1.21
|
|
53
|
+
Requires-Dist: pandas<3.0,>=1.4
|
|
54
|
+
Requires-Dist: scikit-learn<2.0,>=1.0
|
|
55
|
+
Requires-Dist: matplotlib<4.0,>=3.5
|
|
56
|
+
Requires-Dist: seaborn>=0.12
|
|
57
|
+
Requires-Dist: tqdm<5.0,>=4.64
|
|
58
|
+
Requires-Dist: raman-data>=1.0.0
|
|
59
|
+
Provides-Extra: autogluon
|
|
60
|
+
Requires-Dist: autogluon.common>=1.5; extra == "autogluon"
|
|
61
|
+
Requires-Dist: autogluon.core>=1.5; extra == "autogluon"
|
|
62
|
+
Requires-Dist: autogluon.features>=1.5; extra == "autogluon"
|
|
63
|
+
Requires-Dist: autogluon.tabular>=1.5; extra == "autogluon"
|
|
64
|
+
Provides-Extra: models
|
|
65
|
+
Requires-Dist: torch<3.0,>=2.0; extra == "models"
|
|
66
|
+
Requires-Dist: ramanspy<0.3.0,>=0.2.10; extra == "models"
|
|
67
|
+
Requires-Dist: imodels>=2.0.4; extra == "models"
|
|
68
|
+
Requires-Dist: tabpfn>=6.3.2; extra == "models"
|
|
69
|
+
Requires-Dist: pytabkit>=1.7.3; extra == "models"
|
|
70
|
+
Requires-Dist: tabdpt>=1.1.12; extra == "models"
|
|
71
|
+
Requires-Dist: sktime>=0.40.0; extra == "models"
|
|
72
|
+
Requires-Dist: numba>=0.63.0; extra == "models"
|
|
73
|
+
Provides-Extra: full
|
|
74
|
+
Requires-Dist: raman-bench[autogluon]; extra == "full"
|
|
75
|
+
Requires-Dist: raman-bench[models]; extra == "full"
|
|
76
|
+
Requires-Dist: adjustText>=1.2.0; extra == "full"
|
|
77
|
+
Requires-Dist: pynvml>=11.0; extra == "full"
|
|
78
|
+
Provides-Extra: dev
|
|
79
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
80
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
81
|
+
Requires-Dist: black>=23.0; extra == "dev"
|
|
82
|
+
Requires-Dist: isort>=5.12; extra == "dev"
|
|
83
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
84
|
+
Requires-Dist: pre-commit>=3.0; extra == "dev"
|
|
85
|
+
Provides-Extra: docs
|
|
86
|
+
Requires-Dist: sphinx>=7.0; extra == "docs"
|
|
87
|
+
Requires-Dist: sphinx-rtd-theme>=2.0; extra == "docs"
|
|
88
|
+
Requires-Dist: myst-parser>=2.0; extra == "docs"
|
|
89
|
+
Requires-Dist: nbsphinx>=0.9; extra == "docs"
|
|
90
|
+
Requires-Dist: sphinx-autodoc-typehints>=1.25; extra == "docs"
|
|
91
|
+
Provides-Extra: notebooks
|
|
92
|
+
Requires-Dist: jupyter>=1.0; extra == "notebooks"
|
|
93
|
+
Requires-Dist: ipykernel>=6.0; extra == "notebooks"
|
|
94
|
+
Dynamic: license-file
|
|
95
|
+
|
|
96
|
+
# RamanBench
|
|
97
|
+
|
|
98
|
+
[](https://pypi.org/project/raman-bench/)
|
|
99
|
+
[](https://www.python.org)
|
|
100
|
+
[](https://github.com/ml-lab-htw/RamanBench/actions/workflows/ci.yml)
|
|
101
|
+
[](LICENSE)
|
|
102
|
+
[](https://arxiv.org/abs/2605.02003)
|
|
103
|
+
[](https://huggingface.co/spaces/HTW-KI-Werkstatt/RamanBench)
|
|
104
|
+
|
|
105
|
+
**A large-scale benchmark for machine learning on Raman spectroscopy data.**
|
|
106
|
+
|
|
107
|
+
> 74 datasets · 163 prediction targets · 28 baseline models · 4 application domains
|
|
108
|
+
|
|
109
|
+
RamanBench provides a reproducible evaluation protocol and a curated collection
|
|
110
|
+
of public Raman spectroscopy datasets spanning Material Science, Biological,
|
|
111
|
+
Medical, and Chemical applications. Researchers can rank new models against
|
|
112
|
+
28 pre-evaluated baselines — from classical PLS to tabular foundation models
|
|
113
|
+
and Raman-specific deep learning architectures — without re-running all experiments.
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
## Ecosystem
|
|
118
|
+
|
|
119
|
+
```
|
|
120
|
+
raman-data ──▶ raman-bench ──▶ Live Leaderboard
|
|
121
|
+
(datasets) (this package) HuggingFace Space
|
|
122
|
+
PyPI / GitHub PyPI / GitHub
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
| Resource | Link |
|
|
126
|
+
|---------------------------------|----------------------------------------------------------------------------------------------------|
|
|
127
|
+
| **raman-data** (dataset loader) | [GitHub](https://github.com/ml-lab-htw/raman_data) · [PyPI](https://pypi.org/project/raman-data/) |
|
|
128
|
+
| **raman-bench** (this package) | [GitHub](https://github.com/ml-lab-htw/RamanBench) · [PyPI](https://pypi.org/project/raman-bench/) |
|
|
129
|
+
| **Live Leaderboard** | [huggingface.co/spaces/HTW-KI-Werkstatt/RamanBench](https://huggingface.co/spaces/HTW-KI-Werkstatt/RamanBench) |
|
|
130
|
+
| **Paper** | [arXiv:2605.02003](https://arxiv.org/abs/2605.02003) |
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
## Installation
|
|
135
|
+
|
|
136
|
+
### Option 1 — Datasets + leaderboard (recommended starting point)
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
pip install raman-bench
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
This gives you:
|
|
143
|
+
|
|
144
|
+
- **All 74 datasets** with standardised train/test splits via `raman-data`
|
|
145
|
+
- **Precomputed results** for 28 baseline models (bundled CSVs, no internet needed)
|
|
146
|
+
- **Leaderboard API** — rank, plot, and compare against baselines
|
|
147
|
+
- **Evaluation API** — `lb.evaluate_and_add(model)` works with *any* sklearn-compatible model
|
|
148
|
+
|
|
149
|
+
You can use any ML library you already have installed — scikit-learn, LightGBM,
|
|
150
|
+
XGBoost, PyTorch, JAX, or anything else — against a large-scale, curated data
|
|
151
|
+
foundation without installing a single additional dependency.
|
|
152
|
+
|
|
153
|
+
### Option 2 — With all built-in models
|
|
154
|
+
|
|
155
|
+
Adds all Raman-specific architectures and standalone tabular foundation models,
|
|
156
|
+
all with a standard `fit(X, y)` / `predict(X)` interface:
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
pip install "raman-bench[models]"
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
This installs `torch`, `tabpfn`, `pytabkit`, `tabdpt`, `sktime`, and
|
|
163
|
+
`ramanspy` on top of the core package. **No AutoGluon required.**
|
|
164
|
+
|
|
165
|
+
### Option 3 — Full benchmark reproducibility (AutoGluon fork)
|
|
166
|
+
|
|
167
|
+
The paper's benchmark runs all models through AutoGluon's automated
|
|
168
|
+
preprocessing and HPO pipeline. The fork addresses two limitations of
|
|
169
|
+
standard AutoGluon 1.5:
|
|
170
|
+
|
|
171
|
+
1. **Feature cap** — AutoGluon caps tabular foundation models (TabPFN v2,
|
|
172
|
+
TabICL, TabDPT, MITRA) at 500 features; Raman spectra typically have
|
|
173
|
+
500–4000 wavenumber points. The fork removes this cap.
|
|
174
|
+
2. **TabICL v2 regression** — AutoGluon 1.5 ships TabICL v1, which supports
|
|
175
|
+
classification only. The fork upgrades to TabICL v2, adding regression
|
|
176
|
+
support. This limitation is expected to be resolved in AutoGluon 1.6.
|
|
177
|
+
|
|
178
|
+
A [patched fork](https://github.com/ml-lab-htw/autogluon) incorporates both fixes.
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
git clone https://github.com/ml-lab-htw/RamanBench.git
|
|
182
|
+
cd RamanBench
|
|
183
|
+
pip install -r requirements-autogluon-fork.txt
|
|
184
|
+
pip install -e ".[models]"
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
> **The fork is only needed to reproduce the exact paper benchmark.**
|
|
188
|
+
> Options 1 and 2 work with a standard `pip install` and give full access to
|
|
189
|
+
> all datasets, splits, and built-in models.
|
|
190
|
+
|
|
191
|
+
---
|
|
192
|
+
|
|
193
|
+
## Quick Start
|
|
194
|
+
|
|
195
|
+
### Load a dataset (Option 1 — core install only)
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
from raman_data import raman_data
|
|
199
|
+
|
|
200
|
+
ds = raman_data("amino_acids_glycine")
|
|
201
|
+
print(ds.spectra.shape) # (n_samples, n_wavenumbers)
|
|
202
|
+
print(ds.targets.shape) # (n_samples,)
|
|
203
|
+
print(ds.raman_shifts[:5]) # wavenumber axis in cm⁻¹
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
All 74 datasets are available this way. Each comes with a fixed train/test
|
|
207
|
+
split so results are directly comparable to the precomputed baselines.
|
|
208
|
+
|
|
209
|
+
### Evaluate your model against 28 baselines (Option 1)
|
|
210
|
+
|
|
211
|
+
Any scikit-learn–compatible estimator works:
|
|
212
|
+
|
|
213
|
+
```python
|
|
214
|
+
from raman_bench import Leaderboard
|
|
215
|
+
from sklearn.cross_decomposition import PLSRegression
|
|
216
|
+
|
|
217
|
+
lb = Leaderboard.from_precomputed() # loads bundled v0.1 results
|
|
218
|
+
|
|
219
|
+
# Evaluates on all 74 datasets (3 seeds) and inserts into the ranking
|
|
220
|
+
results = lb.evaluate_and_add(
|
|
221
|
+
model_name="My-PLS-10",
|
|
222
|
+
model=PLSRegression(n_components=10),
|
|
223
|
+
)
|
|
224
|
+
print(lb.rank())
|
|
225
|
+
lb.plot()
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
Bring any library — LightGBM, XGBoost, a PyTorch model, a JAX model — and it
|
|
229
|
+
will be scored on the same protocol as the 28 precomputed baselines.
|
|
230
|
+
|
|
231
|
+
### Explore the precomputed leaderboard (Option 1)
|
|
232
|
+
|
|
233
|
+
```python
|
|
234
|
+
from raman_bench import Leaderboard
|
|
235
|
+
|
|
236
|
+
lb = Leaderboard.from_precomputed()
|
|
237
|
+
print(lb.rank()) # ranked DataFrame
|
|
238
|
+
lb.plot() # horizontal bar chart
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
### Use a built-in Raman model directly
|
|
242
|
+
|
|
243
|
+
All built-in models expose a standard sklearn `fit` / `predict` API:
|
|
244
|
+
|
|
245
|
+
```python
|
|
246
|
+
import numpy as np
|
|
247
|
+
from raman_bench.models.custom import DeepCNNModel, TabPFNModel, RocketModel
|
|
248
|
+
|
|
249
|
+
X = np.random.randn(200, 512).astype("float32") # 200 spectra, 512 wavenumbers
|
|
250
|
+
y = np.random.randn(200) # regression targets
|
|
251
|
+
|
|
252
|
+
# Raman-specific deep learning model
|
|
253
|
+
model = DeepCNNModel(n_epochs=50)
|
|
254
|
+
model.fit(X, y)
|
|
255
|
+
predictions = model.predict(X)
|
|
256
|
+
|
|
257
|
+
# Tabular foundation model (no feature-count limit)
|
|
258
|
+
tfm = TabPFNModel()
|
|
259
|
+
tfm.fit(X, y)
|
|
260
|
+
predictions = tfm.predict(X)
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
### Run the full benchmark pipeline (fork required)
|
|
264
|
+
|
|
265
|
+
```bash
|
|
266
|
+
# Pre-cache all dataset splits (optional, speeds up the run)
|
|
267
|
+
python scripts/prepare_datasets.py --config configs/benchmark_v0.1.json
|
|
268
|
+
|
|
269
|
+
# Run predictions → metrics
|
|
270
|
+
raman-bench run --config configs/benchmark_v0.1.json
|
|
271
|
+
|
|
272
|
+
# Run individual steps
|
|
273
|
+
raman-bench run --config configs/benchmark_v0.1.json --step predictions
|
|
274
|
+
raman-bench run --config configs/benchmark_v0.1.json --step metrics
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
### Notebooks
|
|
278
|
+
|
|
279
|
+
| Notebook | Description |
|
|
280
|
+
|---|---|
|
|
281
|
+
| [`01_quick_start.ipynb`](notebooks/01_quick_start.ipynb) | Load a dataset, explore the precomputed leaderboard, plot rankings |
|
|
282
|
+
| [`02_benchmark_new_model.ipynb`](notebooks/02_benchmark_new_model.ipynb) | Evaluate your own model and add it to the leaderboard |
|
|
283
|
+
| [`03_explore_results.ipynb`](notebooks/03_explore_results.ipynb) | Deep dive into per-dataset and per-domain results |
|
|
284
|
+
| [`04_contribute_dataset.ipynb`](notebooks/04_contribute_dataset.ipynb) | Step-by-step guide to contributing a new dataset |
|
|
285
|
+
|
|
286
|
+
---
|
|
287
|
+
|
|
288
|
+
## Models
|
|
289
|
+
|
|
290
|
+
### Paper baselines (28 models)
|
|
291
|
+
|
|
292
|
+
All results in the paper were produced through the AutoGluon pipeline (Option 3 install).
|
|
293
|
+
|
|
294
|
+
| Category | Models |
|
|
295
|
+
|
|
296
|
+
| Category | Models |
|
|
297
|
+
|---|---|
|
|
298
|
+
| Classical spectroscopy | PLS, KNN, LR |
|
|
299
|
+
| Tree ensembles | GBM (LightGBM), XGB, CatBoost, RF, XT |
|
|
300
|
+
| Tabular deep learning | NN_TORCH, FastAI, RealMLP |
|
|
301
|
+
| Tabular foundation models | TabPFN v2, TabPFN v2.5, TabM, TabDPT, TabICL, MITRA |
|
|
302
|
+
| Time-series classifiers | ROCKET, Arsenal |
|
|
303
|
+
| Raman-specific DL | DeepCNN, RamanNet, SANet, RamanFormer, RamanTransformer, ReZeroNet, FC-ResNeXt, CoAtNet |
|
|
304
|
+
| AutoGluon ensemble | AUTOGLUON |
|
|
305
|
+
|
|
306
|
+
### Standalone sklearn wrappers (`raman-bench[models]`)
|
|
307
|
+
|
|
308
|
+
`raman-bench[models]` provides sklearn-compatible (`fit` / `predict`) wrappers
|
|
309
|
+
for many of the same algorithm families, usable directly without AutoGluon or
|
|
310
|
+
the fork. These are **not** the exact pipeline configurations from the paper
|
|
311
|
+
(no AutoGluon preprocessing or HPO), but they use the same underlying
|
|
312
|
+
algorithms and are well-suited for building and evaluating new models.
|
|
313
|
+
|
|
314
|
+
| Class | Algorithm | Requires |
|
|
315
|
+
|---|---|---|
|
|
316
|
+
| `PLSModel` | Partial Least Squares | — |
|
|
317
|
+
| `DeepCNNModel` | Raman-specific CNN | `torch` |
|
|
318
|
+
| `RamanNetModel` | Raman-specific CNN | `torch` |
|
|
319
|
+
| `SANetModel` | Spectral attention net | `torch` |
|
|
320
|
+
| `RamanFormerModel` | Raman transformer | `torch` |
|
|
321
|
+
| `RamanTransformerModel` | Raman transformer | `torch` |
|
|
322
|
+
| `ReZeroNetModel` | ReZero CNN | `torch` |
|
|
323
|
+
| `FCResNeXtModel` | FC-ResNeXt | `torch` |
|
|
324
|
+
| `CoAtNetModel` | Conv + attention | `torch` |
|
|
325
|
+
| `RocketModel` | ROCKET classifier | `sktime` |
|
|
326
|
+
| `ArsenalModel` | Arsenal classifier | `sktime` |
|
|
327
|
+
| `TabPFNModel` | TabPFN v2 | `tabpfn` |
|
|
328
|
+
| `RealMLPModel` | RealMLP-TD | `pytabkit` |
|
|
329
|
+
| `TabMModel` | TabM-D | `pytabkit` |
|
|
330
|
+
| `TabDPTModel` | TabDPT | `tabdpt` |
|
|
331
|
+
|
|
332
|
+
All classes support classification and regression and auto-detect the task from
|
|
333
|
+
`y`. All package dependencies are included in `raman-bench[models]`.
|
|
334
|
+
|
|
335
|
+
---
|
|
336
|
+
|
|
337
|
+
## Benchmark Composition
|
|
338
|
+
|
|
339
|
+
### Datasets
|
|
340
|
+
|
|
341
|
+
74 public Raman spectroscopy datasets from four application domains:
|
|
342
|
+
|
|
343
|
+
| Domain | Datasets | Task | Sources |
|
|
344
|
+
|---|---|---|---|
|
|
345
|
+
| Chemical | 37 | Regression | Zenodo, HuggingFace |
|
|
346
|
+
| Medical | 11 | Classification | Kaggle, Zenodo |
|
|
347
|
+
| Biological | 8 | Regression | HuggingFace, Zenodo |
|
|
348
|
+
| Material Science | 4 | Classification | RRUFF, Zenodo |
|
|
349
|
+
|
|
350
|
+
All datasets are accessible via `pip install raman-data`:
|
|
351
|
+
|
|
352
|
+
```python
|
|
353
|
+
from raman_data import raman_data
|
|
354
|
+
|
|
355
|
+
dataset = raman_data("amino_acids_glycine")
|
|
356
|
+
X = dataset.spectra # (n_samples, n_wavenumbers)
|
|
357
|
+
y = dataset.targets # regression targets or class labels
|
|
358
|
+
w = dataset.raman_shifts # wavenumber axis in cm⁻¹
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
**Dataset catalog:** [raman-data on GitHub](https://github.com/ml-lab-htw/raman_data)
|
|
362
|
+
|
|
363
|
+
---
|
|
364
|
+
|
|
365
|
+
## Ranking Protocol
|
|
366
|
+
|
|
367
|
+
Models are evaluated under three complementary metrics:
|
|
368
|
+
|
|
369
|
+
| Metric | Description |
|
|
370
|
+
|---|---|
|
|
371
|
+
| **Elo** | Pairwise win-rate Elo calibrated to RF = 1000 (200-round bootstrap) |
|
|
372
|
+
| **Score** | Normalised per-dataset score: best model = 1, median model = 0 |
|
|
373
|
+
| **Avg Rank** | Average rank across all datasets and targets |
|
|
374
|
+
| **Improvability** | % gap to the best model, averaged across datasets |
|
|
375
|
+
|
|
376
|
+
See the [live leaderboard](https://huggingface.co/spaces/HTW-KI-Werkstatt/RamanBench) for
|
|
377
|
+
interactive filtering by model category, task type, and dataset domain.
|
|
378
|
+
|
|
379
|
+
---
|
|
380
|
+
|
|
381
|
+
## Repository Structure
|
|
382
|
+
|
|
383
|
+
```
|
|
384
|
+
RamanBench/
|
|
385
|
+
├── src/raman_bench/
|
|
386
|
+
│ ├── leaderboard.py # Leaderboard + model evaluation API
|
|
387
|
+
│ ├── benchmark.py # Dataset loading and cross-validation
|
|
388
|
+
│ ├── predictions.py # Prediction generation (benchmark step 1)
|
|
389
|
+
│ ├── evaluation.py # Metric computation (benchmark step 2)
|
|
390
|
+
│ ├── model.py # AutoGluon pipeline wrapper (fork required)
|
|
391
|
+
│ ├── config.py # JSON config loader
|
|
392
|
+
│ ├── models/custom/ # All built-in Raman models (sklearn API)
|
|
393
|
+
│ │ ├── base.py # BaseRamanEstimator (shared training loop)
|
|
394
|
+
│ │ ├── deepcnn.py # DeepCNNModel
|
|
395
|
+
│ │ ├── ramannet.py # RamanNetModel
|
|
396
|
+
│ │ ├── sanet.py # SANetModel
|
|
397
|
+
│ │ ├── ramanformer.py # RamanFormerModel
|
|
398
|
+
│ │ ├── ramantransformer.py # RamanTransformerModel
|
|
399
|
+
│ │ ├── rezeronet.py # ReZeroNetModel
|
|
400
|
+
│ │ ├── fcresnext.py # FCResNeXtModel
|
|
401
|
+
│ │ ├── coatnet.py # CoAtNetModel
|
|
402
|
+
│ │ ├── pls.py # PLSModel
|
|
403
|
+
│ │ ├── sktime_models.py # RocketModel, ArsenalModel
|
|
404
|
+
│ │ └── tabular_foundation.py # TabPFNModel, RealMLPModel, TabMModel, TabDPTModel
|
|
405
|
+
│ └── preprocessing/
|
|
406
|
+
│ ├── mixin.py # RamanPreprocessingMixin (AutoGluon HPO)
|
|
407
|
+
│ └── wrapped_models.py # Prep_* classes + SklearnAutoGluonBridge
|
|
408
|
+
├── configs/ # Benchmark configuration files
|
|
409
|
+
├── data/precomputed/ # Bundled v0.1 results
|
|
410
|
+
├── notebooks/ # Example Jupyter notebooks
|
|
411
|
+
├── scripts/ # CLI scripts
|
|
412
|
+
└── tests/ # pytest test suite
|
|
413
|
+
```
|
|
414
|
+
|
|
415
|
+
### Architecture: two paths, one set of model classes
|
|
416
|
+
|
|
417
|
+
Custom models are implemented once as plain scikit-learn `BaseEstimator`
|
|
418
|
+
subclasses. The same classes are used in both usage modes:
|
|
419
|
+
|
|
420
|
+
```
|
|
421
|
+
Custom model (e.g. DeepCNNModel)
|
|
422
|
+
BaseEstimator — no AutoGluon dependency
|
|
423
|
+
fit(X, y) / predict(X)
|
|
424
|
+
│
|
|
425
|
+
├─── Standalone path (pip install "raman-bench[models]")
|
|
426
|
+
│ CUSTOM_MODELS["DEEPCNN"] → DeepCNNModel().fit(X, y)
|
|
427
|
+
│
|
|
428
|
+
└─── AutoGluon pipeline path (fork required)
|
|
429
|
+
SklearnAutoGluonBridge._fit() → DeepCNNModel(**params).fit(X_np, y_np)
|
|
430
|
+
Prep_DEEPCNN(_RamanDLBase, _DeepCNNBridge)
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
`SklearnAutoGluonBridge` (in `preprocessing/wrapped_models.py`) is the only
|
|
434
|
+
file that imports AutoGluon. All model source files are AutoGluon-free.
|
|
435
|
+
|
|
436
|
+
---
|
|
437
|
+
|
|
438
|
+
## Contributing
|
|
439
|
+
|
|
440
|
+
We welcome contributions of new models and datasets!
|
|
441
|
+
|
|
442
|
+
### Adding a New Model
|
|
443
|
+
|
|
444
|
+
The simplest way to add a model is to implement it as a scikit-learn–compatible
|
|
445
|
+
estimator and submit a pull request. No AutoGluon knowledge is required.
|
|
446
|
+
|
|
447
|
+
1. Create `src/raman_bench/models/custom/my_model.py`:
|
|
448
|
+
|
|
449
|
+
```python
|
|
450
|
+
import numpy as np
|
|
451
|
+
from sklearn.base import BaseEstimator
|
|
452
|
+
|
|
453
|
+
class MyModel(BaseEstimator):
|
|
454
|
+
|
|
455
|
+
def __init__(self, n_components=10, lr=1e-3):
|
|
456
|
+
self.n_components = n_components
|
|
457
|
+
self.lr = lr
|
|
458
|
+
|
|
459
|
+
def fit(self, X, y):
|
|
460
|
+
# X: np.ndarray (n_samples, n_features)
|
|
461
|
+
# y: np.ndarray — float → regression, int/str → classification
|
|
462
|
+
...
|
|
463
|
+
return self
|
|
464
|
+
|
|
465
|
+
def predict(self, X):
|
|
466
|
+
... # return np.ndarray (n_samples,)
|
|
467
|
+
|
|
468
|
+
def predict_proba(self, X):
|
|
469
|
+
... # classification only, return (n_samples, n_classes)
|
|
470
|
+
```
|
|
471
|
+
|
|
472
|
+
For PyTorch-based models, inherit from `BaseRamanEstimator` in
|
|
473
|
+
`models/custom/base.py` which provides a complete training loop with early
|
|
474
|
+
stopping, cosine LR schedule, mixed-class augmentation, and batched inference.
|
|
475
|
+
|
|
476
|
+
2. Register in `src/raman_bench/models/custom/__init__.py`:
|
|
477
|
+
|
|
478
|
+
```python
|
|
479
|
+
from raman_bench.models.custom.my_model import MyModel
|
|
480
|
+
|
|
481
|
+
CUSTOM_MODELS["MYMODEL"] = MyModel
|
|
482
|
+
```
|
|
483
|
+
|
|
484
|
+
3. Add tests in `tests/models/test_my_model.py` following the patterns in
|
|
485
|
+
`tests/models/test_sanet.py`.
|
|
486
|
+
|
|
487
|
+
4. Open a pull request — CI will run the full test suite automatically.
|
|
488
|
+
|
|
489
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the full guide, including how to
|
|
490
|
+
optionally wire your model into the AutoGluon benchmark pipeline for full
|
|
491
|
+
reproducibility.
|
|
492
|
+
|
|
493
|
+
### Adding a New Dataset
|
|
494
|
+
|
|
495
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md#adding-a-new-dataset) and
|
|
496
|
+
[NEW_DATASETS.md](NEW_DATASETS.md) for detailed instructions and examples.
|
|
497
|
+
|
|
498
|
+
Quick summary:
|
|
499
|
+
1. Upload your dataset to HuggingFace Datasets or Zenodo under CC BY 4.0.
|
|
500
|
+
2. Add a loader to the [raman-data](https://github.com/ml-lab-htw/raman_data)
|
|
501
|
+
package (open a PR there).
|
|
502
|
+
3. Open an issue here linking to the raman-data PR.
|
|
503
|
+
|
|
504
|
+
The [live leaderboard](https://huggingface.co/spaces/HTW-KI-Werkstatt/RamanBench)
|
|
505
|
+
also has a "How to Contribute" section with step-by-step instructions.
|
|
506
|
+
|
|
507
|
+
---
|
|
508
|
+
|
|
509
|
+
## Citation
|
|
510
|
+
|
|
511
|
+
If you use RamanBench in your research, please cite:
|
|
512
|
+
|
|
513
|
+
```bibtex
|
|
514
|
+
@misc{koddenbrock2026ramanbench,
|
|
515
|
+
title = {RamanBench: A Large-Scale Benchmark for Machine Learning on Raman Spectroscopy},
|
|
516
|
+
author = {Koddenbrock, Mario and Lange, Christoph and Legner, Robin and Jaeger, Martin
|
|
517
|
+
and K{\"o}gler, Martin and Cruz Bournazou, Mariano N. and Neubauer, Peter
|
|
518
|
+
and Bie{\ss}mann, Felix and Rodner, Erik},
|
|
519
|
+
year = {2026},
|
|
520
|
+
eprint = {2605.02003},
|
|
521
|
+
archivePrefix = {arXiv},
|
|
522
|
+
primaryClass = {cs.LG},
|
|
523
|
+
url = {https://arxiv.org/abs/2605.02003}
|
|
524
|
+
}
|
|
525
|
+
```
|
|
526
|
+
|
|
527
|
+
---
|
|
528
|
+
|
|
529
|
+
## License
|
|
530
|
+
|
|
531
|
+
MIT — see [LICENSE](LICENSE).
|
|
532
|
+
|
|
533
|
+
Dataset licenses vary; see the [dataset catalog](https://huggingface.co/spaces/HTW-KI-Werkstatt/RamanBench)
|
|
534
|
+
or [raman-data](https://github.com/ml-lab-htw/raman_data) for per-dataset license information.
|
|
535
|
+
Most datasets are released under CC BY 4.0.
|