specsrbench 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. specsrbench-0.1.1/.gitignore +49 -0
  2. specsrbench-0.1.1/LICENSE +21 -0
  3. specsrbench-0.1.1/PKG-INFO +289 -0
  4. specsrbench-0.1.1/README.md +216 -0
  5. specsrbench-0.1.1/docs/GUARDS.md +106 -0
  6. specsrbench-0.1.1/docs/_static/.gitkeep +0 -0
  7. specsrbench-0.1.1/docs/api/index.md +32 -0
  8. specsrbench-0.1.1/docs/conf.py +85 -0
  9. specsrbench-0.1.1/docs/guides/figures.md +76 -0
  10. specsrbench-0.1.1/docs/guides/guards.md +5 -0
  11. specsrbench-0.1.1/docs/guides/installation.md +54 -0
  12. specsrbench-0.1.1/docs/guides/rebuilding.md +90 -0
  13. specsrbench-0.1.1/docs/index.md +80 -0
  14. specsrbench-0.1.1/pyproject.toml +116 -0
  15. specsrbench-0.1.1/requirements.txt +36 -0
  16. specsrbench-0.1.1/scripts/make_classical_figure.py +502 -0
  17. specsrbench-0.1.1/scripts/make_classical_toy_figure.py +416 -0
  18. specsrbench-0.1.1/src/specsrbench/__init__.py +22 -0
  19. specsrbench-0.1.1/src/specsrbench/build/__init__.py +72 -0
  20. specsrbench-0.1.1/src/specsrbench/build/classical_cache.py +327 -0
  21. specsrbench-0.1.1/src/specsrbench/build/lines.py +285 -0
  22. specsrbench-0.1.1/src/specsrbench/build/lsf.py +166 -0
  23. specsrbench-0.1.1/src/specsrbench/build/predictions.py +147 -0
  24. specsrbench-0.1.1/src/specsrbench/build/sets.py +195 -0
  25. specsrbench-0.1.1/src/specsrbench/build/tune.py +445 -0
  26. specsrbench-0.1.1/src/specsrbench/classical.py +338 -0
  27. specsrbench-0.1.1/src/specsrbench/cli.py +102 -0
  28. specsrbench-0.1.1/src/specsrbench/data.py +199 -0
  29. specsrbench-0.1.1/src/specsrbench/figures/__init__.py +72 -0
  30. specsrbench-0.1.1/src/specsrbench/figures/fig1_toy_methods.py +363 -0
  31. specsrbench-0.1.1/src/specsrbench/figures/fig2_qualitative.py +152 -0
  32. specsrbench-0.1.1/src/specsrbench/figures/fig3_residual_maps.py +154 -0
  33. specsrbench-0.1.1/src/specsrbench/figures/fig4_mae_summary.py +151 -0
  34. specsrbench-0.1.1/src/specsrbench/figures/fig5_per_line_snr.py +137 -0
  35. specsrbench-0.1.1/src/specsrbench/figures/fig6_redshift_mae.py +87 -0
  36. specsrbench-0.1.1/src/specsrbench/methods.py +103 -0
  37. specsrbench-0.1.1/src/specsrbench/metrics.py +120 -0
  38. specsrbench-0.1.1/src/specsrbench/paths.py +87 -0
  39. specsrbench-0.1.1/src/specsrbench/style.py +54 -0
  40. specsrbench-0.1.1/tests/conftest.py +120 -0
  41. specsrbench-0.1.1/tests/test_algorithms.py +227 -0
  42. specsrbench-0.1.1/tests/test_cache_integrity.py +191 -0
  43. specsrbench-0.1.1/tests/test_figures.py +203 -0
  44. specsrbench-0.1.1/tests/test_invariants.py +200 -0
  45. specsrbench-0.1.1/tests/test_metrics.py +95 -0
  46. specsrbench-0.1.1/tests/test_paper_consistency.py +330 -0
  47. specsrbench-0.1.1/tests/test_paper_prose.py +323 -0
@@ -0,0 +1,49 @@
1
+ # The public specsrbench tree ships code only. Everything the figures read is
2
+ # derived data that `specsrbench build all` regenerates from JADES DR4 and the
3
+ # published weights, so none of it belongs in git.
4
+ #
5
+ # This is a separate file from the working .gitignore rather than a filtered
6
+ # copy of it: that one exists mostly to record which large directories are
7
+ # committed *on purpose* in the private repository, and none of them exist
8
+ # here. See scripts/make_public_release.sh.
9
+ #
10
+ # Every directory rule below is anchored with a leading slash, and that is not
11
+ # cosmetic: the build backend honours this file when it decides what goes into
12
+ # the wheel. An unanchored `build/` or `figures/` also matches
13
+ # `src/specsrbench/build/` and `src/specsrbench/figures/`, shipping a package
14
+ # with two of its subpackages missing -- which installs cleanly and then fails
15
+ # on the first import.
16
+
17
+ # Derived arrays. `specsrbench build all` writes these.
18
+ /cache/
19
+ /cache_logR/
20
+ /cache_logR_tuned/
21
+ *.npy
22
+ *.npz
23
+
24
+ # Model weights come from the Hugging Face Hub, never from git.
25
+ *.pth
26
+ *.pt
27
+ *.ckpt
28
+
29
+ # Built figures. The point of the package is that these regenerate.
30
+ /figures/
31
+
32
+ # Sphinx: _build is output, api/generated/ is autosummary output that
33
+ # `autosummary_generate = True` recreates on every build.
34
+ /docs/_build/
35
+ /docs/api/generated/
36
+
37
+ # Python
38
+ __pycache__/
39
+ *.py[cod]
40
+ .venv/
41
+ /venv/
42
+ .pytest_cache/
43
+ .ruff_cache/
44
+ /build/
45
+ /dist/
46
+ *.egg-info/
47
+
48
+ # OS
49
+ .DS_Store
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Aryana Haghjoo
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,289 @@
1
+ Metadata-Version: 2.5
2
+ Name: specsrbench
3
+ Version: 0.1.1
4
+ Summary: Benchmarking deep learning against classical deconvolution for galaxy spectral super-resolution
5
+ Project-URL: Homepage, https://github.com/aryana-haghjoo/specsr-benchmark
6
+ Project-URL: Repository, https://github.com/aryana-haghjoo/specsr-benchmark
7
+ Project-URL: Issues, https://github.com/aryana-haghjoo/specsr-benchmark/issues
8
+ Project-URL: Companion package, https://github.com/aryana-haghjoo/specsr
9
+ Project-URL: Model weights, https://huggingface.co/aryana-haghjoo/specsr
10
+ Project-URL: Paper 1, https://arxiv.org/abs/2603.18357
11
+ Project-URL: Documentation, https://aryana-haghjoo.github.io/specsr-benchmark/
12
+ Author-email: Aryana Haghjoo <ahagh010@ucr.edu>
13
+ License: MIT License
14
+
15
+ Copyright (c) 2026 Aryana Haghjoo
16
+
17
+ Permission is hereby granted, free of charge, to any person obtaining a copy
18
+ of this software and associated documentation files (the "Software"), to deal
19
+ in the Software without restriction, including without limitation the rights
20
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
21
+ copies of the Software, and to permit persons to whom the Software is
22
+ furnished to do so, subject to the following conditions:
23
+
24
+ The above copyright notice and this permission notice shall be included in all
25
+ copies or substantial portions of the Software.
26
+
27
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
28
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
29
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
30
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
31
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
32
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33
+ SOFTWARE.
34
+ License-File: LICENSE
35
+ Keywords: astronomy,benchmark,deconvolution,jwst,nirspec,spectroscopy,super-resolution
36
+ Classifier: Development Status :: 3 - Alpha
37
+ Classifier: Intended Audience :: Science/Research
38
+ Classifier: License :: OSI Approved :: MIT License
39
+ Classifier: Programming Language :: Python :: 3
40
+ Classifier: Programming Language :: Python :: 3.10
41
+ Classifier: Programming Language :: Python :: 3.11
42
+ Classifier: Programming Language :: Python :: 3.12
43
+ Classifier: Topic :: Scientific/Engineering :: Astronomy
44
+ Classifier: Topic :: Scientific/Engineering :: Image Processing
45
+ Requires-Python: >=3.10
46
+ Requires-Dist: matplotlib>=3.7
47
+ Requires-Dist: numpy<2,>=1.24
48
+ Requires-Dist: pandas>=2.0
49
+ Requires-Dist: pywavelets>=1.4
50
+ Requires-Dist: scikit-image>=0.22
51
+ Requires-Dist: scipy>=1.10
52
+ Requires-Dist: tqdm>=4.65
53
+ Provides-Extra: all
54
+ Requires-Dist: astropy<8,>=5.3; extra == 'all'
55
+ Requires-Dist: huggingface-hub>=0.23; extra == 'all'
56
+ Requires-Dist: specsr>=1.0; extra == 'all'
57
+ Requires-Dist: torch>=2.0; extra == 'all'
58
+ Provides-Extra: dev
59
+ Requires-Dist: pytest-cov>=4.1; extra == 'dev'
60
+ Requires-Dist: pytest>=7.4; extra == 'dev'
61
+ Requires-Dist: ruff>=0.5; extra == 'dev'
62
+ Provides-Extra: docs
63
+ Requires-Dist: furo; extra == 'docs'
64
+ Requires-Dist: myst-parser; extra == 'docs'
65
+ Requires-Dist: sphinx>=7.2; extra == 'docs'
66
+ Provides-Extra: lsf
67
+ Requires-Dist: astropy<8,>=5.3; extra == 'lsf'
68
+ Provides-Extra: ml
69
+ Requires-Dist: huggingface-hub>=0.23; extra == 'ml'
70
+ Requires-Dist: specsr>=1.0; extra == 'ml'
71
+ Requires-Dist: torch>=2.0; extra == 'ml'
72
+ Description-Content-Type: text/markdown
73
+
74
+ # specsrbench
75
+
76
+ [![tests](https://github.com/aryana-haghjoo/specsr-benchmark/actions/workflows/tests.yml/badge.svg)](https://github.com/aryana-haghjoo/specsr-benchmark/actions/workflows/tests.yml)
77
+ [![docs](https://github.com/aryana-haghjoo/specsr-benchmark/actions/workflows/docs.yml/badge.svg)](https://aryana-haghjoo.github.io/specsr-benchmark/)
78
+ [![python](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue)](https://github.com/aryana-haghjoo/specsr-benchmark)
79
+ [![license](https://img.shields.io/badge/license-MIT-green)](LICENSE)
80
+
81
+ **Benchmarking deep learning against classical deconvolution for galaxy
82
+ spectral super-resolution.**
83
+
84
+ Seven classical deconvolution methods — cubic interpolation, Wiener, Tikhonov,
85
+ Wiener + total variation, Richardson–Lucy, wavelet-sparse FISTA, and a
86
+ redshift-informed matched filter — scored against the SR2 deep-learning
87
+ pipeline of [Haghjoo et al. 2026](https://arxiv.org/abs/2603.18357) on
88
+ JWST/NIRSpec prism spectra from JADES.
89
+
90
+ The deep-learning side is a separate package,
91
+ [`specsr`](https://github.com/aryana-haghjoo/specsr), whose weights live
92
+ [on the Hub](https://huggingface.co/aryana-haghjoo/specsr). This package holds
93
+ the classical baselines, the metrics, the tuning, and the figures.
94
+
95
+ ## Install
96
+
97
+ Not on PyPI yet. Install the wheel from the
98
+ [latest release](https://github.com/aryana-haghjoo/specsr-benchmark/releases/latest):
99
+
100
+ ```bash
101
+ pip install https://github.com/aryana-haghjoo/specsr-benchmark/releases/download/v0.1.1/specsrbench-0.1.1-py3-none-any.whl
102
+ ```
103
+
104
+ or from source:
105
+
106
+ ```bash
107
+ git clone https://github.com/aryana-haghjoo/specsr-benchmark
108
+ cd specsr-benchmark
109
+ pip install -e . # the baselines and the figures
110
+ pip install -e '.[all]' # + the ML arm and the LSF derivation
111
+ ```
112
+
113
+ `[ml]` pulls in torch and `specsr` (needed by `build predictions` and by
114
+ figure 1's toy CNN); `[lsf]` pulls in astropy (needed only to re-derive the
115
+ instrument LSF). Neither is required to draw the other five figures.
116
+
117
+ ---
118
+
119
+ ## What it is for
120
+
121
+ The short version of the result: **on this data, at the pixel level, no method
122
+ beats cubic interpolation.**
123
+
124
+ SR2 leads the raw mean-absolute-error table by 30%, and does it by producing a
125
+ spectrum at 0.54 of the reference's amplitude — absolute error against a noisy
126
+ reference falls when you shrink toward zero, whatever the reconstruction
127
+ quality. On a scale-free metric all nine methods land within 1.2% of each
128
+ other and SR2 ranks eighth of nine. What does survive is sharper and more
129
+ interesting than a leaderboard: SR2 exceeds the *reference's own* line signal
130
+ to noise on all four diagnostic lines while recovering only 36–53% of true line
131
+ amplitudes, at false-detection rates of 0.30 (Hβ) and 0.44 ([O II]) against
132
+ ≤ 0.09 for any classical method.
133
+
134
+ Getting there took four separate metric traps, each of which had to be closed
135
+ before the benchmark meant anything. They are documented in
136
+ [`docs/GUARDS.md`](docs/GUARDS.md), and they are the part of this repository
137
+ most likely to be useful to someone benchmarking something else.
138
+
139
+ ## Quick start
140
+
141
+ ```bash
142
+ specsrbench paths # where it will look for inputs
143
+ specsrbench figures all # rebuild all six paper figures
144
+ specsrbench figures 4 # or just one
145
+ ```
146
+
147
+ The figures read a cache of derived arrays. If you do not have one, build it
148
+ (next section) or point at one you do have:
149
+
150
+ ```bash
151
+ export SPECSRBENCH_CACHE=/path/to/cache_logR_tuned
152
+ ```
153
+
154
+ | figure | what it shows | command |
155
+ |---|---|---|
156
+ | 1 | every method on a 1D toy where truth is known | `specsrbench figures toy` |
157
+ | 2 | one held-out galaxy, all methods, [O III] inset | `specsrbench figures qualitative` |
158
+ | 3 | residual maps over all 572 spectra vs redshift | `specsrbench figures residuals` |
159
+ | 4 | global fidelity: MAE, uncertainty-normalised, RMSE | `specsrbench figures mae` |
160
+ | 5 | per-line S/N, detection, false detection, width bias | `specsrbench figures per-line-snr` |
161
+ | 6 | error against redshift, in equal-count bins | `specsrbench figures redshift` |
162
+
163
+ Figure 1 trains a small CNN inline and needs `specsrbench[ml]`; the other five
164
+ are pure numpy and matplotlib.
165
+
166
+ ## Rebuilding everything from the data
167
+
168
+ No large files are distributed with this package. The chain runs from the raw
169
+ [JADES DR4](https://jades-survey.github.io/) products and the published model
170
+ weights, in six stages:
171
+
172
+ ```bash
173
+ specsrbench build predictions # specsr chain over the held-out split (Hub weights)
174
+ specsrbench build sets # eval / tune / calib sets, galaxy-disjoint
175
+ specsrbench build lsf --jades-root <JADES DR4> # measure the instrument LSF
176
+ specsrbench build tune # search the classical parameters under four guards
177
+ specsrbench build classical # the classical reconstructions
178
+ specsrbench build lines # Gaussian line fits, S/N, the summary table
179
+ ```
180
+
181
+ `specsrbench build all` runs them in order. `--dry-run` on any stage prints
182
+ what it would read and write without doing it.
183
+
184
+ What each stage needs:
185
+
186
+ | stage | needs | why |
187
+ |---|---|---|
188
+ | `predictions` | `specsr`, torch, network | runs SR1 → ZHead → SR2, weights from the Hub |
189
+ | `sets` | the paired dataset | draws the three galaxy-disjoint sets |
190
+ | `lsf` | raw JADES DR4, `astropy` | measures the prism→grating kernel |
191
+ | `tune`, `classical`, `lines` | nothing beyond the above | pure numpy/scipy |
192
+
193
+ The paired dataset (3.5 GB) is built by `specsr` from the JADES tree and is not
194
+ redistributable here:
195
+
196
+ ```bash
197
+ specsr build --jades-root <JADES DR4 tree> --out paired_DR4_logR.npz
198
+ ```
199
+
200
+ ### The one thing to get right
201
+
202
+ **Match the checkpoint to the wavelength grid by reading its provenance, never
203
+ by its filename.** More than one file called `best_sr2.pth` exists, trained on
204
+ different grids. Loading the wrong one does not fail — it produces plausible
205
+ spectra that are wrong. `specsrbench build predictions` pins the Hub revision
206
+ and records what it loaded in the output's `provenance` field.
207
+
208
+ ## Why the LSF is measured and not read
209
+
210
+ The evaluation products carry a `sigma_pix` array that was widely used as the
211
+ deconvolution kernel and **does not describe the data**: it is roughly constant
212
+ in nanometres, where a spectrograph's line-spread function is fixed in detector
213
+ *pixels*. Measured against the paired spectra it is up to 2.3× too broad at
214
+ 5 µm. With it, Wiener, Tikhonov and TV merge line pairs that their own input
215
+ still resolves.
216
+
217
+ `specsrbench build lsf` measures the real one from the instrument: the prism
218
+ dispersion from the raw x1d products, the effective kernel width from
219
+ σ_eff² = σ_LR² − σ_HR² on fits to the four diagnostic lines, and the ratio of
220
+ the two. It comes out constant to a few per cent across a factor of three in
221
+ wavelength, which is the check that it is an instrumental LSF and not a curve
222
+ fitted to noise. The sets this package builds deliberately do **not** carry a
223
+ `sigma_pix`, so the kernel can only come from measuring it.
224
+
225
+ ## Tuning is guarded, in four ways
226
+
227
+ Mean absolute error is not a safe objective, and no single guard catches what
228
+ the others do:
229
+
230
+ | trap | what it rewards | guard |
231
+ |---|---|---|
232
+ | smoothing | erasing every line incurs no line-shaped residual | median line S/N ≥ 0.9 × no-deconvolution baseline |
233
+ | shrinkage | scaling toward zero lowers error against a noisy reference | output std within [0.90, 1.15] of the target's |
234
+ | blurring | a unit-gain Wiener filter with snr ≤ 1 can only broaden | median FWHM bias ≤ the baseline's |
235
+ | merging | two close lines smeared into one peak look like one good line | resolvable [O III] pairs kept resolved ≥ baseline |
236
+
237
+ Each is blind to the others: line S/N is amplitude over sideband noise, so a
238
+ global rescale leaves it unchanged; the amplitude guard is width-invariant; and
239
+ all three of those are single-line statistics, so a Gaussian fitted to a
240
+ blended doublet passes them. The fourth is measured on **real** spectra — a
241
+ synthetic pair was tried and discarded as far too easy to hold apart.
242
+
243
+ `specsrbench build tune` applies all four inside the search, and
244
+ `tests/test_invariants.py` asserts them against whatever cache is present. A
245
+ method is allowed to fail guard 4 only if `classical_params.json` records that
246
+ no setting passes it; that is Tikhonov's case, and the paper says so.
247
+
248
+ ## Tests
249
+
250
+ ```bash
251
+ pytest # the full suite
252
+ pytest -m "not slow" # skip the end-to-end figure builds
253
+ ```
254
+
255
+ The suite runs without any data — tests that need a cache skip cleanly. With
256
+ one present it checks the cache's structural invariants, the four guards, and
257
+ rebuilds a slice of each classical cache from the recorded parameters to
258
+ confirm it reproduces.
259
+
260
+ ## Layout
261
+
262
+ ```
263
+ src/specsrbench/
264
+ paths.py where inputs and outputs are found
265
+ methods.py the nine reconstructions, their names and cache keys
266
+ metrics.py MAE, scale-free MAE, amplitude ratio, the guards
267
+ classical.py the seven classical deconvolvers, on the log R=4000 grid
268
+ data.py the cache loader every figure shares
269
+ figures/ one module per paper figure
270
+ build/ the six pipeline stages
271
+ scripts/ talk figures, which are not paper figures
272
+ tests/
273
+ ```
274
+
275
+ ## Documentation
276
+
277
+ Full documentation, including the API reference, is at
278
+ **<https://aryana-haghjoo.github.io/specsr-benchmark/>**.
279
+
280
+ ## Citing
281
+
282
+ Paper 2 is in preparation. Until it appears, cite the archived software release
283
+ and [Haghjoo et al. 2026](https://arxiv.org/abs/2603.18357) for the models.
284
+ `CITATION.cff` carries the machine-readable metadata; GitHub's "Cite this
285
+ repository" button reads it.
286
+
287
+ ## License
288
+
289
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,216 @@
1
+ # specsrbench
2
+
3
+ [![tests](https://github.com/aryana-haghjoo/specsr-benchmark/actions/workflows/tests.yml/badge.svg)](https://github.com/aryana-haghjoo/specsr-benchmark/actions/workflows/tests.yml)
4
+ [![docs](https://github.com/aryana-haghjoo/specsr-benchmark/actions/workflows/docs.yml/badge.svg)](https://aryana-haghjoo.github.io/specsr-benchmark/)
5
+ [![python](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue)](https://github.com/aryana-haghjoo/specsr-benchmark)
6
+ [![license](https://img.shields.io/badge/license-MIT-green)](LICENSE)
7
+
8
+ **Benchmarking deep learning against classical deconvolution for galaxy
9
+ spectral super-resolution.**
10
+
11
+ Seven classical deconvolution methods — cubic interpolation, Wiener, Tikhonov,
12
+ Wiener + total variation, Richardson–Lucy, wavelet-sparse FISTA, and a
13
+ redshift-informed matched filter — scored against the SR2 deep-learning
14
+ pipeline of [Haghjoo et al. 2026](https://arxiv.org/abs/2603.18357) on
15
+ JWST/NIRSpec prism spectra from JADES.
16
+
17
+ The deep-learning side is a separate package,
18
+ [`specsr`](https://github.com/aryana-haghjoo/specsr), whose weights live
19
+ [on the Hub](https://huggingface.co/aryana-haghjoo/specsr). This package holds
20
+ the classical baselines, the metrics, the tuning, and the figures.
21
+
22
+ ## Install
23
+
24
+ Not on PyPI yet. Install the wheel from the
25
+ [latest release](https://github.com/aryana-haghjoo/specsr-benchmark/releases/latest):
26
+
27
+ ```bash
28
+ pip install https://github.com/aryana-haghjoo/specsr-benchmark/releases/download/v0.1.1/specsrbench-0.1.1-py3-none-any.whl
29
+ ```
30
+
31
+ or from source:
32
+
33
+ ```bash
34
+ git clone https://github.com/aryana-haghjoo/specsr-benchmark
35
+ cd specsr-benchmark
36
+ pip install -e . # the baselines and the figures
37
+ pip install -e '.[all]' # + the ML arm and the LSF derivation
38
+ ```
39
+
40
+ `[ml]` pulls in torch and `specsr` (needed by `build predictions` and by
41
+ figure 1's toy CNN); `[lsf]` pulls in astropy (needed only to re-derive the
42
+ instrument LSF). Neither is required to draw the other five figures.
43
+
44
+ ---
45
+
46
+ ## What it is for
47
+
48
+ The short version of the result: **on this data, at the pixel level, no method
49
+ beats cubic interpolation.**
50
+
51
+ SR2 leads the raw mean-absolute-error table by 30%, and does it by producing a
52
+ spectrum at 0.54 of the reference's amplitude — absolute error against a noisy
53
+ reference falls when you shrink toward zero, whatever the reconstruction
54
+ quality. On a scale-free metric all nine methods land within 1.2% of each
55
+ other and SR2 ranks eighth of nine. What does survive is sharper and more
56
+ interesting than a leaderboard: SR2 exceeds the *reference's own* line signal
57
+ to noise on all four diagnostic lines while recovering only 36–53% of true line
58
+ amplitudes, at false-detection rates of 0.30 (Hβ) and 0.44 ([O II]) against
59
+ ≤ 0.09 for any classical method.
60
+
61
+ Getting there took four separate metric traps, each of which had to be closed
62
+ before the benchmark meant anything. They are documented in
63
+ [`docs/GUARDS.md`](docs/GUARDS.md), and they are the part of this repository
64
+ most likely to be useful to someone benchmarking something else.
65
+
66
+ ## Quick start
67
+
68
+ ```bash
69
+ specsrbench paths # where it will look for inputs
70
+ specsrbench figures all # rebuild all six paper figures
71
+ specsrbench figures 4 # or just one
72
+ ```
73
+
74
+ The figures read a cache of derived arrays. If you do not have one, build it
75
+ (next section) or point at one you do have:
76
+
77
+ ```bash
78
+ export SPECSRBENCH_CACHE=/path/to/cache_logR_tuned
79
+ ```
80
+
81
+ | figure | what it shows | command |
82
+ |---|---|---|
83
+ | 1 | every method on a 1D toy where truth is known | `specsrbench figures toy` |
84
+ | 2 | one held-out galaxy, all methods, [O III] inset | `specsrbench figures qualitative` |
85
+ | 3 | residual maps over all 572 spectra vs redshift | `specsrbench figures residuals` |
86
+ | 4 | global fidelity: MAE, uncertainty-normalised, RMSE | `specsrbench figures mae` |
87
+ | 5 | per-line S/N, detection, false detection, width bias | `specsrbench figures per-line-snr` |
88
+ | 6 | error against redshift, in equal-count bins | `specsrbench figures redshift` |
89
+
90
+ Figure 1 trains a small CNN inline and needs `specsrbench[ml]`; the other five
91
+ are pure numpy and matplotlib.
92
+
93
+ ## Rebuilding everything from the data
94
+
95
+ No large files are distributed with this package. The chain runs from the raw
96
+ [JADES DR4](https://jades-survey.github.io/) products and the published model
97
+ weights, in six stages:
98
+
99
+ ```bash
100
+ specsrbench build predictions # specsr chain over the held-out split (Hub weights)
101
+ specsrbench build sets # eval / tune / calib sets, galaxy-disjoint
102
+ specsrbench build lsf --jades-root <JADES DR4> # measure the instrument LSF
103
+ specsrbench build tune # search the classical parameters under four guards
104
+ specsrbench build classical # the classical reconstructions
105
+ specsrbench build lines # Gaussian line fits, S/N, the summary table
106
+ ```
107
+
108
+ `specsrbench build all` runs them in order. `--dry-run` on any stage prints
109
+ what it would read and write without doing it.
110
+
111
+ What each stage needs:
112
+
113
+ | stage | needs | why |
114
+ |---|---|---|
115
+ | `predictions` | `specsr`, torch, network | runs SR1 → ZHead → SR2, weights from the Hub |
116
+ | `sets` | the paired dataset | draws the three galaxy-disjoint sets |
117
+ | `lsf` | raw JADES DR4, `astropy` | measures the prism→grating kernel |
118
+ | `tune`, `classical`, `lines` | nothing beyond the above | pure numpy/scipy |
119
+
120
+ The paired dataset (3.5 GB) is built by `specsr` from the JADES tree and is not
121
+ redistributable here:
122
+
123
+ ```bash
124
+ specsr build --jades-root <JADES DR4 tree> --out paired_DR4_logR.npz
125
+ ```
126
+
127
+ ### The one thing to get right
128
+
129
+ **Match the checkpoint to the wavelength grid by reading its provenance, never
130
+ by its filename.** More than one file called `best_sr2.pth` exists, trained on
131
+ different grids. Loading the wrong one does not fail — it produces plausible
132
+ spectra that are wrong. `specsrbench build predictions` pins the Hub revision
133
+ and records what it loaded in the output's `provenance` field.
134
+
135
+ ## Why the LSF is measured and not read
136
+
137
+ The evaluation products carry a `sigma_pix` array that was widely used as the
138
+ deconvolution kernel and **does not describe the data**: it is roughly constant
139
+ in nanometres, where a spectrograph's line-spread function is fixed in detector
140
+ *pixels*. Measured against the paired spectra it is up to 2.3× too broad at
141
+ 5 µm. With it, Wiener, Tikhonov and TV merge line pairs that their own input
142
+ still resolves.
143
+
144
+ `specsrbench build lsf` measures the real one from the instrument: the prism
145
+ dispersion from the raw x1d products, the effective kernel width from
146
+ σ_eff² = σ_LR² − σ_HR² on fits to the four diagnostic lines, and the ratio of
147
+ the two. It comes out constant to a few per cent across a factor of three in
148
+ wavelength, which is the check that it is an instrumental LSF and not a curve
149
+ fitted to noise. The sets this package builds deliberately do **not** carry a
150
+ `sigma_pix`, so the kernel can only come from measuring it.
151
+
152
+ ## Tuning is guarded, in four ways
153
+
154
+ Mean absolute error is not a safe objective, and no single guard catches what
155
+ the others do:
156
+
157
+ | trap | what it rewards | guard |
158
+ |---|---|---|
159
+ | smoothing | erasing every line incurs no line-shaped residual | median line S/N ≥ 0.9 × no-deconvolution baseline |
160
+ | shrinkage | scaling toward zero lowers error against a noisy reference | output std within [0.90, 1.15] of the target's |
161
+ | blurring | a unit-gain Wiener filter with snr ≤ 1 can only broaden | median FWHM bias ≤ the baseline's |
162
+ | merging | two close lines smeared into one peak look like one good line | resolvable [O III] pairs kept resolved ≥ baseline |
163
+
164
+ Each is blind to the others: line S/N is amplitude over sideband noise, so a
165
+ global rescale leaves it unchanged; the amplitude guard is width-invariant; and
166
+ all three of those are single-line statistics, so a Gaussian fitted to a
167
+ blended doublet passes them. The fourth is measured on **real** spectra — a
168
+ synthetic pair was tried and discarded as far too easy to hold apart.
169
+
170
+ `specsrbench build tune` applies all four inside the search, and
171
+ `tests/test_invariants.py` asserts them against whatever cache is present. A
172
+ method is allowed to fail guard 4 only if `classical_params.json` records that
173
+ no setting passes it; that is Tikhonov's case, and the paper says so.
174
+
175
+ ## Tests
176
+
177
+ ```bash
178
+ pytest # the full suite
179
+ pytest -m "not slow" # skip the end-to-end figure builds
180
+ ```
181
+
182
+ The suite runs without any data — tests that need a cache skip cleanly. With
183
+ one present it checks the cache's structural invariants, the four guards, and
184
+ rebuilds a slice of each classical cache from the recorded parameters to
185
+ confirm it reproduces.
186
+
187
+ ## Layout
188
+
189
+ ```
190
+ src/specsrbench/
191
+ paths.py where inputs and outputs are found
192
+ methods.py the nine reconstructions, their names and cache keys
193
+ metrics.py MAE, scale-free MAE, amplitude ratio, the guards
194
+ classical.py the seven classical deconvolvers, on the log R=4000 grid
195
+ data.py the cache loader every figure shares
196
+ figures/ one module per paper figure
197
+ build/ the six pipeline stages
198
+ scripts/ talk figures, which are not paper figures
199
+ tests/
200
+ ```
201
+
202
+ ## Documentation
203
+
204
+ Full documentation, including the API reference, is at
205
+ **<https://aryana-haghjoo.github.io/specsr-benchmark/>**.
206
+
207
+ ## Citing
208
+
209
+ Paper 2 is in preparation. Until it appears, cite the archived software release
210
+ and [Haghjoo et al. 2026](https://arxiv.org/abs/2603.18357) for the models.
211
+ `CITATION.cff` carries the machine-readable metadata; GitHub's "Cite this
212
+ repository" button reads it.
213
+
214
+ ## License
215
+
216
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,106 @@
1
+ # Four ways mean absolute error lies
2
+
3
+ Every classical method in this benchmark has free parameters, and they have to
4
+ be chosen somehow. The obvious objective — minimise mean absolute error against
5
+ the high-resolution reference — is wrong in four separate ways. Each was found
6
+ by shipping it, and each cost a full rebuild.
7
+
8
+ They are written down here because none of them is specific to spectra. Any
9
+ benchmark that tunes a restoration method against a noisy reference by an
10
+ error norm has all four available to it.
11
+
12
+ The rule that came out of it: **a number is only safe if a test recomputes
13
+ it.**
14
+
15
+ ---
16
+
17
+ ## 1. Smoothing
18
+
19
+ **What it rewards.** A filter that erases every emission line incurs no
20
+ line-shaped residual. Against a noisy reference, the residual of a smooth
21
+ output is *smaller* than the residual of a correct one, because the correct one
22
+ also reproduces the noise it is scored against.
23
+
24
+ **Guard.** Median line signal-to-noise ≥ 0.9 × the no-deconvolution baseline.
25
+
26
+ **Where it was found.** The first tuning pass drove every regularisation
27
+ strength to its maximum and produced spectra with no lines in them at all,
28
+ scoring better than the input they came from.
29
+
30
+ ## 2. Shrinkage
31
+
32
+ **What it rewards.** Scaling the whole output toward zero. If the reference is
33
+ `signal + noise` and the estimate is `k × signal`, then for `k < 1` the
34
+ absolute error falls for any noise level, regardless of whether the estimate
35
+ resolved anything.
36
+
37
+ **Guard.** Output standard deviation within [0.90, 1.15] of the target's.
38
+
39
+ **Why guard 1 cannot see it.** Line S/N is amplitude over sideband noise. A
40
+ global rescale multiplies both and leaves the ratio exactly unchanged. Every
41
+ scale-free diagnostic is blind to shrinkage by construction — which is also
42
+ why the headline comparison in this benchmark is reported on a scale-free
43
+ metric *and* an amplitude ratio, never on MAE alone.
44
+
45
+ **Where it was found.** The Wiener filter's MMSE form `W = H/(H² + 1/snr)` has
46
+ DC gain `1/(1 + 1/snr)` — 0.91 at snr = 10, 0.44 at snr = 0.8. Tuning it
47
+ against MAE drove `snr` toward zero. The filter is now normalised to unit gain
48
+ at zero frequency, which keeps its shape and removes its ability to buy MAE
49
+ with a rescale. The deep-learning model is subject to exactly the same effect
50
+ and is not immune to it: SR2's output sits at 0.54 of the reference's scale.
51
+
52
+ ## 3. Blurring
53
+
54
+ **What it rewards.** Broadening lines rather than sharpening them. A unit-gain
55
+ Wiener filter with `snr ≤ 1` has its maximum at zero frequency, so it cannot
56
+ amplify any frequency — it can only smooth — while still scoring well.
57
+
58
+ **Guard.** Median line FWHM bias no larger than the baseline's.
59
+
60
+ **Why guard 2 cannot see it.** The amplitude guard is width-invariant: a line
61
+ can be twice as wide at the same standard deviation.
62
+
63
+ ## 4. Merging
64
+
65
+ **What it rewards.** Smearing two close lines into one peak. This is the one
66
+ that matters physically — resolving blended doublets is most of the reason to
67
+ deconvolve a spectrum at all.
68
+
69
+ **Guard.** The fraction of resolvable [O III] pairs kept resolved must be at
70
+ least the no-deconvolution baseline's, measured on real spectra from a
71
+ galaxy-disjoint calibration set.
72
+
73
+ **Why guards 1–3 cannot see it.** All three are single-line Gaussian-fit
74
+ statistics, and a Gaussian fitted to a *blended* doublet has much the same
75
+ amplitude, signal-to-noise and width as one fitted to a separated pair. All
76
+ three pass a filter that merges the doublet.
77
+
78
+ **A synthetic pair does not work.** One was tried first and discarded: a
79
+ synthetic doublet is far easier to hold apart than a real one, and it passed
80
+ settings that merge in real data. The guard has to be measured on real spectra.
81
+
82
+ **Where it was found.** With the kernel that was in use at the time, Wiener,
83
+ Tikhonov and Wiener + TV merged line pairs that plain cubic interpolation still
84
+ resolves — 0% pair survival for all three, against 43% for doing nothing.
85
+
86
+ ---
87
+
88
+ ## What the guards do not fix
89
+
90
+ A guard rules a setting out; it does not make a method good. Tikhonov fails
91
+ guard 4 at every setting searched — nothing passes it and the amplitude guard
92
+ together. It is retained at its published value rather than dropped, because
93
+ dropping it would be a larger claim than the evidence supports: the failure is
94
+ of this segmented implementation on this grid, not of Tikhonov regularisation.
95
+ `classical_params.json` records the failure in its `failed_guards` field, and
96
+ the paper says so.
97
+
98
+ ## Where they live
99
+
100
+ `specsrbench build tune` applies all four *inside* the search. They used to be
101
+ applied by hand afterwards, which is how a Richardson–Lucy setting of
102
+ `n_iter=1` — one iteration, i.e. barely deconvolving at all — came to be
103
+ shipped.
104
+
105
+ `tests/test_invariants.py` asserts all four against whatever cache is present,
106
+ so a rebuild that violates one fails the suite rather than reaching a figure.
File without changes