svc-processing 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pipeline/__init__.py +0 -0
- pipeline/cli.py +92 -0
- pipeline/processor.py +582 -0
- pipeline/resampler.py +495 -0
- pipeline/run_config.py +303 -0
- pipeline/runner.py +216 -0
- pipeline/sig_processor.py +258 -0
- svc_processing-0.1.1.dist-info/METADATA +250 -0
- svc_processing-0.1.1.dist-info/RECORD +13 -0
- svc_processing-0.1.1.dist-info/WHEEL +5 -0
- svc_processing-0.1.1.dist-info/entry_points.txt +2 -0
- svc_processing-0.1.1.dist-info/licenses/LICENSE +674 -0
- svc_processing-0.1.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class SigFileProcessor:
|
|
11
|
+
"""Process .sig files with configurable correction types."""
|
|
12
|
+
|
|
13
|
+
DEFAULT_CORRECTION_TYPES = {
|
|
14
|
+
'bronze': "2520.4",
|
|
15
|
+
'silver': "2517.9"
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
DEFAULT_INSTRUMENT_NUMBERS = {
|
|
19
|
+
'bronze': "2212118",
|
|
20
|
+
'silver': "1202103"
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
@staticmethod
|
|
24
|
+
def parse_correction_types_file(config_path: str | Path) -> dict[str, str]:
|
|
25
|
+
"""Parse a sensor-calibration JSON file into a correction_type -> end_line mapping.
|
|
26
|
+
|
|
27
|
+
Pure — does not touch ``DEFAULT_CORRECTION_TYPES``. Used by
|
|
28
|
+
:meth:`load_default_correction_types` and by callers (such as
|
|
29
|
+
``RunConfig``) that want a resolved calibration table for a single
|
|
30
|
+
instance without changing the class-wide defaults.
|
|
31
|
+
"""
|
|
32
|
+
path_obj = Path(config_path).expanduser().resolve()
|
|
33
|
+
if not path_obj.exists():
|
|
34
|
+
raise FileNotFoundError(f"Correction-types config file not found: {path_obj}")
|
|
35
|
+
|
|
36
|
+
with path_obj.open("r") as f:
|
|
37
|
+
data = json.load(f)
|
|
38
|
+
|
|
39
|
+
if not isinstance(data, dict):
|
|
40
|
+
raise ValueError(
|
|
41
|
+
"Correction-types config JSON must be an object mapping correction_type -> end_line string."
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
normalized: dict[str, str] = {}
|
|
45
|
+
for key, value in data.items():
|
|
46
|
+
if key is None:
|
|
47
|
+
continue
|
|
48
|
+
correction_type = str(key).strip().lower()
|
|
49
|
+
if not correction_type:
|
|
50
|
+
continue
|
|
51
|
+
end_line_value = str(value).strip()
|
|
52
|
+
if not end_line_value:
|
|
53
|
+
raise ValueError(f"End-line value for correction type '{correction_type}' is empty in {path_obj}")
|
|
54
|
+
normalized[correction_type] = end_line_value
|
|
55
|
+
|
|
56
|
+
if not normalized:
|
|
57
|
+
raise ValueError(f"No correction types found in {path_obj}")
|
|
58
|
+
|
|
59
|
+
return normalized
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def load_default_correction_types(cls, config_path: str | Path) -> dict[str, str]:
|
|
63
|
+
"""Load correction-type end-line values from a JSON file and update DEFAULT_CORRECTION_TYPES."""
|
|
64
|
+
cls.DEFAULT_CORRECTION_TYPES = cls.parse_correction_types_file(config_path)
|
|
65
|
+
return cls.DEFAULT_CORRECTION_TYPES
|
|
66
|
+
|
|
67
|
+
def __init__(self, correction_value: str | None = None, instrument_number: str | None = None,
|
|
68
|
+
correction_type: str | None = None, correction_config: dict | None = None,
|
|
69
|
+
correction_types: dict[str, str] | None = None,
|
|
70
|
+
instrument_numbers: dict[str, str] | None = None,
|
|
71
|
+
logger: logging.Logger | None = None):
|
|
72
|
+
self._logger = logger
|
|
73
|
+
# Instance-level calibration tables, defaulting to the class-wide
|
|
74
|
+
# defaults. Passing these explicitly lets a caller (e.g. RunConfig)
|
|
75
|
+
# resolve per-run calibration without mutating DEFAULT_CORRECTION_TYPES /
|
|
76
|
+
# DEFAULT_INSTRUMENT_NUMBERS, which are shared, process-wide state.
|
|
77
|
+
self._correction_types = correction_types if correction_types is not None else self.DEFAULT_CORRECTION_TYPES
|
|
78
|
+
self._instrument_numbers = (
|
|
79
|
+
instrument_numbers if instrument_numbers is not None else self.DEFAULT_INSTRUMENT_NUMBERS
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
if correction_config:
|
|
83
|
+
self.end_line_value = correction_config.get('end_line')
|
|
84
|
+
self.instrument_number = correction_config.get('instrument_number')
|
|
85
|
+
self.correction_type = correction_config.get('name', 'custom')
|
|
86
|
+
|
|
87
|
+
elif correction_value is not None:
|
|
88
|
+
self.end_line_value = correction_value
|
|
89
|
+
self.instrument_number = instrument_number
|
|
90
|
+
self.correction_type = 'custom'
|
|
91
|
+
|
|
92
|
+
elif correction_type:
|
|
93
|
+
if correction_type not in self._correction_types:
|
|
94
|
+
raise ValueError(f"Invalid correction_type specified. Use 'bronze' or 'silver'. Got: {correction_type}")
|
|
95
|
+
|
|
96
|
+
self.correction_type = correction_type
|
|
97
|
+
self.end_line_value = self._correction_types[correction_type]
|
|
98
|
+
self.instrument_number = self._instrument_numbers[correction_type]
|
|
99
|
+
|
|
100
|
+
else:
|
|
101
|
+
raise ValueError("Must provide either correction_value, correction_type, or correction_config")
|
|
102
|
+
|
|
103
|
+
if not self.end_line_value or self.end_line_value.strip() == "":
|
|
104
|
+
raise ValueError("correction_value must be provided and cannot be empty")
|
|
105
|
+
|
|
106
|
+
def _log_info(self, message: str) -> None:
|
|
107
|
+
"""Route info-level messages through a logger if one was supplied, else print."""
|
|
108
|
+
if self._logger is not None:
|
|
109
|
+
self._logger.info(message)
|
|
110
|
+
else:
|
|
111
|
+
print(message)
|
|
112
|
+
|
|
113
|
+
def _log_error(self, message: str) -> None:
|
|
114
|
+
"""Route error-level messages through a logger if one was supplied, else print."""
|
|
115
|
+
if self._logger is not None:
|
|
116
|
+
self._logger.error(message)
|
|
117
|
+
else:
|
|
118
|
+
print(message)
|
|
119
|
+
|
|
120
|
+
def process_sig_files(self, input_folder: str, output_folder: str, verbose: bool = False) -> None:
|
|
121
|
+
"""Process all .sig files in input_folder and write truncated copies to output_folder."""
|
|
122
|
+
end_line_start = self.end_line_value
|
|
123
|
+
|
|
124
|
+
input_dir = Path(input_folder).expanduser().resolve()
|
|
125
|
+
output_dir = Path(output_folder).expanduser().resolve()
|
|
126
|
+
|
|
127
|
+
if not output_dir.exists():
|
|
128
|
+
output_dir.mkdir(parents=True)
|
|
129
|
+
if verbose:
|
|
130
|
+
self._log_info(f"Created output folder: {output_dir}")
|
|
131
|
+
|
|
132
|
+
processed_count = 0
|
|
133
|
+
|
|
134
|
+
for entry in sorted(input_dir.iterdir()):
|
|
135
|
+
if entry.suffix == '.sig':
|
|
136
|
+
output_file_path = output_dir / entry.name
|
|
137
|
+
|
|
138
|
+
try:
|
|
139
|
+
if verbose:
|
|
140
|
+
self._log_info(f"Processing: {entry.name}")
|
|
141
|
+
self._process_single_file(entry, output_file_path, end_line_start)
|
|
142
|
+
processed_count += 1
|
|
143
|
+
if verbose:
|
|
144
|
+
self._log_info(f"Processed: {entry.name}")
|
|
145
|
+
except Exception as e:
|
|
146
|
+
self._log_error(f"Error processing {entry.name}: {e}")
|
|
147
|
+
|
|
148
|
+
if verbose:
|
|
149
|
+
self._log_info(f"Processing complete. {processed_count} files processed.")
|
|
150
|
+
|
|
151
|
+
def _process_single_file(self, input_file_path: Path, output_file_path: Path, end_line_start: str) -> None:
|
|
152
|
+
with input_file_path.open('r') as input_file, output_file_path.open('w') as output_file:
|
|
153
|
+
target_found = True
|
|
154
|
+
for line in input_file:
|
|
155
|
+
if target_found:
|
|
156
|
+
output_file.write(line)
|
|
157
|
+
if line.startswith(end_line_start):
|
|
158
|
+
target_found = False
|
|
159
|
+
|
|
160
|
+
def get_supported_correction_types(self) -> list:
|
|
161
|
+
"""Correction-type names this instance knows about (e.g. 'bronze', 'silver')."""
|
|
162
|
+
return list(self._correction_types.keys())
|
|
163
|
+
|
|
164
|
+
def extract_instrument_from_file(self, file_path: str) -> str | None:
|
|
165
|
+
try:
|
|
166
|
+
with Path(file_path).expanduser().resolve().open('r') as f:
|
|
167
|
+
for line in f:
|
|
168
|
+
if line.startswith('instrument='):
|
|
169
|
+
return line.split('=', 1)[1].strip()
|
|
170
|
+
except (FileNotFoundError, IOError, IndexError):
|
|
171
|
+
pass
|
|
172
|
+
return None
|
|
173
|
+
|
|
174
|
+
def get_file_metadata(self, file_path: str) -> dict:
|
|
175
|
+
metadata = {}
|
|
176
|
+
try:
|
|
177
|
+
with Path(file_path).expanduser().resolve().open('r') as f:
|
|
178
|
+
for line in f:
|
|
179
|
+
if '=' in line and not line.startswith('data='):
|
|
180
|
+
parts = line.strip().split('=', 1)
|
|
181
|
+
if len(parts) == 2:
|
|
182
|
+
metadata[parts[0].strip()] = parts[1].strip()
|
|
183
|
+
except (FileNotFoundError, IOError):
|
|
184
|
+
pass
|
|
185
|
+
return metadata
|
|
186
|
+
|
|
187
|
+
def _extract_instrument_name(self, instrument: str) -> str:
|
|
188
|
+
if not instrument:
|
|
189
|
+
return "Unknown"
|
|
190
|
+
match = re.search(r'(\d{7})', instrument)
|
|
191
|
+
if match:
|
|
192
|
+
instrument_number = match.group(1)
|
|
193
|
+
if instrument_number == self._instrument_numbers['silver']:
|
|
194
|
+
return 'Silver'
|
|
195
|
+
elif instrument_number == self._instrument_numbers['bronze']:
|
|
196
|
+
return 'Bronze'
|
|
197
|
+
elif self.instrument_number and instrument_number == self.instrument_number:
|
|
198
|
+
return 'Custom'
|
|
199
|
+
return 'Unknown'
|
|
200
|
+
|
|
201
|
+
def check_instrument_consistency(self, folder_path: str) -> dict[str, Any]:
|
|
202
|
+
"""Check that all .sig files in folder_path share the same instrument."""
|
|
203
|
+
folder = Path(folder_path).expanduser().resolve()
|
|
204
|
+
|
|
205
|
+
if not folder.exists():
|
|
206
|
+
return {
|
|
207
|
+
'consistent': False,
|
|
208
|
+
'instrument': None,
|
|
209
|
+
'instrument_name': 'Unknown',
|
|
210
|
+
'files_by_instrument': {},
|
|
211
|
+
'total_files': 0,
|
|
212
|
+
'warnings': [f"Folder does not exist: {folder}"]
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
files_by_instrument = {}
|
|
216
|
+
warnings = []
|
|
217
|
+
total_files = 0
|
|
218
|
+
|
|
219
|
+
for entry in sorted(folder.iterdir()):
|
|
220
|
+
if entry.suffix == '.sig':
|
|
221
|
+
instrument = self.extract_instrument_from_file(str(entry))
|
|
222
|
+
total_files += 1
|
|
223
|
+
|
|
224
|
+
if instrument is None:
|
|
225
|
+
warnings.append(f"Could not extract instrument from: {entry.name}")
|
|
226
|
+
instrument = "Unknown"
|
|
227
|
+
|
|
228
|
+
if instrument not in files_by_instrument:
|
|
229
|
+
files_by_instrument[instrument] = []
|
|
230
|
+
files_by_instrument[instrument].append(entry.name)
|
|
231
|
+
|
|
232
|
+
unique_instruments = list(files_by_instrument.keys())
|
|
233
|
+
is_consistent = len(unique_instruments) == 1
|
|
234
|
+
|
|
235
|
+
if is_consistent:
|
|
236
|
+
instrument_value = unique_instruments[0]
|
|
237
|
+
if instrument_value == "Unknown":
|
|
238
|
+
warnings.append("All files have unknown instrument values")
|
|
239
|
+
instrument_name = "Unknown"
|
|
240
|
+
else:
|
|
241
|
+
instrument_name = self._extract_instrument_name(instrument_value)
|
|
242
|
+
else:
|
|
243
|
+
instrument_value = None
|
|
244
|
+
instrument_name = "Mixed"
|
|
245
|
+
for instrument, files in files_by_instrument.items():
|
|
246
|
+
if len(files) > 1:
|
|
247
|
+
warnings.append(f"Files with instrument '{instrument}': {', '.join(files)}")
|
|
248
|
+
else:
|
|
249
|
+
warnings.append(f"File with instrument '{instrument}': {files[0]}")
|
|
250
|
+
|
|
251
|
+
return {
|
|
252
|
+
'consistent': is_consistent,
|
|
253
|
+
'instrument': instrument_value,
|
|
254
|
+
'instrument_name': instrument_name,
|
|
255
|
+
'files_by_instrument': files_by_instrument,
|
|
256
|
+
'total_files': total_files,
|
|
257
|
+
'warnings': warnings
|
|
258
|
+
}
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: svc-processing
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Pure-Python SVC HR-1024i hyperspectral .sig processing pipeline
|
|
5
|
+
Author: Gold Lab (GrapeSPEC project), Cornell University
|
|
6
|
+
Author-email: Cole Regnier <nr466@cornell.edu>
|
|
7
|
+
License-Expression: GPL-3.0-only
|
|
8
|
+
Project-URL: Repository, https://github.com/regs08/SVCProcessingPipeline
|
|
9
|
+
Keywords: hyperspectral,spectroscopy,SVC,HR-1024i,remote-sensing,reflectance
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: numpy
|
|
20
|
+
Requires-Dist: pandas
|
|
21
|
+
Requires-Dist: scipy
|
|
22
|
+
Provides-Extra: demo
|
|
23
|
+
Requires-Dist: matplotlib; extra == "demo"
|
|
24
|
+
Requires-Dist: nbclient; extra == "demo"
|
|
25
|
+
Requires-Dist: nbconvert; extra == "demo"
|
|
26
|
+
Requires-Dist: nbformat; extra == "demo"
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pyflakes; extra == "dev"
|
|
29
|
+
Requires-Dist: pytest>=8.3.0; extra == "dev"
|
|
30
|
+
Requires-Dist: ruff; extra == "dev"
|
|
31
|
+
Requires-Dist: vulture; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# SIG Processing Pipeline
|
|
35
|
+
|
|
36
|
+
A pure-Python pipeline for processing SVC HR-1024i field hyperspectral `.sig` files. Reads raw multi-detector spectra, performs sensor stitching and radiometric correction, applies resolution-matched Gaussian smoothing, and resamples onto a uniform 1 nm grid from 400–2500 nm. Numerically verified against the legacy R/`spectrolab` reference to better than 1 × 10⁻⁶ absolute reflectance.
|
|
37
|
+
|
|
38
|
+
> **This README is the canonical entry point for both humans and LLMs.** Every directory in the repo has its own `README.md` describing its contents in detail. Follow the links below — do not assume anything that is not stated here or in the linked docs.
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## Repository map
|
|
43
|
+
|
|
44
|
+
| Path | Purpose | Read me first |
|
|
45
|
+
|---|---|---|
|
|
46
|
+
| [`TUTORIAL.md`](TUTORIAL.md) | Beginner walkthrough — starts in the Jupyter notebook (gentle path), with an advanced CLI track. | Start here if new |
|
|
47
|
+
| [`pyproject.toml`](pyproject.toml) | Package metadata and `svc-pipeline` console script. | This file ↓ |
|
|
48
|
+
| [`pipeline/`](pipeline/) | Core Python package: CLI, `SigFileProcessor`, `resample_spectra`, `SVCDataProcessor`. | [`pipeline/README.md`](pipeline/README.md) |
|
|
49
|
+
| [`tests/`](tests/) | Pytest suite, including the R-vs-Python parity test. | [`tests/README.md`](tests/README.md) |
|
|
50
|
+
| [`config/`](config/) | Run configs + instrument calibration JSONs. | [`config/README.md`](config/README.md) |
|
|
51
|
+
| [`docs/`](docs/) | Manuscript-grade methods, parity reports, LLM re-test prompt. | [`docs/README.md`](docs/README.md) |
|
|
52
|
+
| [`archived_r_scripts/`](archived_r_scripts/) | Frozen R/`spectrolab` reference (Pipeline A) — kept only for parity verification. | [`archived_r_scripts/README.md`](archived_r_scripts/README.md) |
|
|
53
|
+
| [`notebooks/`](notebooks/) | Demo and visualization notebooks (not on the production path). | [`notebooks/README.md`](notebooks/README.md) |
|
|
54
|
+
| [`naming_ids/`](naming_ids/) | Private CSV lookup tables for grouping scans into samples; only the schema README is tracked. | [`naming_ids/README.md`](naming_ids/README.md) |
|
|
55
|
+
| [`FOLDER_STRUCTURE.md`](FOLDER_STRUCTURE.md) | Authoritative tree + reading order. | — |
|
|
56
|
+
| [`ACKNOWLEDGMENTS.md`](ACKNOWLEDGMENTS.md) | Attribution, third-party licenses, and citation. | — |
|
|
57
|
+
| [`LICENSE`](LICENSE) | GNU General Public License v3.0. | — |
|
|
58
|
+
|
|
59
|
+
Generated outputs (gitignored): `pipeline_outputs/sig_processed/<run>/`, `pipeline_outputs/sig_resampled/<run>/`.
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## Quick start
|
|
64
|
+
|
|
65
|
+
> New to the pipeline? [`TUTORIAL.md`](TUTORIAL.md) starts with the interactive
|
|
66
|
+
> Jupyter notebook (the gentle path) and has an advanced track for the CLI below.
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
python3.11 -m venv .venv
|
|
70
|
+
source .venv/bin/activate
|
|
71
|
+
python -m pip install -e ".[dev,demo]"
|
|
72
|
+
|
|
73
|
+
# Edit config/config.json — replace "<PATH_TO_SIG_INPUT_ROOT>" with your data path.
|
|
74
|
+
svc-pipeline config.json
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Outputs land under `pipeline_outputs/` by default. See [`config/README.md`](config/README.md) for every supported key.
|
|
78
|
+
|
|
79
|
+
---
|
|
80
|
+
|
|
81
|
+
## Pipeline architecture
|
|
82
|
+
|
|
83
|
+
```
|
|
84
|
+
raw *.sig files (sig_input_dir)
|
|
85
|
+
│
|
|
86
|
+
▼
|
|
87
|
+
┌────────────────────────────────────┐
|
|
88
|
+
│ Stage 1: SigFileProcessor │ pipeline/sig_processor.py
|
|
89
|
+
│ - instrument-consistency check │
|
|
90
|
+
│ - truncate at calibration │
|
|
91
|
+
│ end-line wavelength │
|
|
92
|
+
│ - write summary CSV │
|
|
93
|
+
└────────────────────────────────────┘
|
|
94
|
+
│
|
|
95
|
+
▼ processed *.sig + *_processed_sig_summary.csv
|
|
96
|
+
┌────────────────────────────────────┐
|
|
97
|
+
│ Stage 2: resample_spectra() │ pipeline/resampler.py
|
|
98
|
+
│ - detect sensor segments │
|
|
99
|
+
│ - guess_splice_at + trim │
|
|
100
|
+
│ - match_sensors (iter = 1) │
|
|
101
|
+
│ - smooth_fwhm (k=3 kmeans) │
|
|
102
|
+
│ - Gaussian resample fwhm=10 │
|
|
103
|
+
│ onto 400–2500 nm @ 1 nm │
|
|
104
|
+
└────────────────────────────────────┘
|
|
105
|
+
│
|
|
106
|
+
▼ <run>_merged_spectra.csv
|
|
107
|
+
┌────────────────────────────────────┐
|
|
108
|
+
│ Stage 3 (post-hoc, optional): │ pipeline/processor.py
|
|
109
|
+
│ SVCDataProcessor / │ + notebooks/
|
|
110
|
+
│ SigSpectraAverager — group & │
|
|
111
|
+
│ average scans into samples. │
|
|
112
|
+
└────────────────────────────────────┘
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
The `svc-pipeline` console script glues Stages 1 and 2 together through
|
|
116
|
+
`pipeline.cli`; Stage 3 is invoked from notebooks against the Stage 2 output.
|
|
117
|
+
The pipeline is **single-pass and idempotent per input directory** — previous
|
|
118
|
+
processed `.sig` files in the target directory are deleted at the start of each
|
|
119
|
+
run.
|
|
120
|
+
|
|
121
|
+
For the formal algorithmic spec (every constant, every formula), read [`docs/supplementary_methods.md`](docs/supplementary_methods.md). That document — not this README — is the source of truth for *what* the code does.
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## Running the pipeline
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
svc-pipeline [config] [options]
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
| Argument | Meaning |
|
|
132
|
+
|---|---|
|
|
133
|
+
| `config` | Run-config JSON (positional, optional; default `config/config.json`). Bare names resolve under `config/`, so `config.json`, `config`, and `config/config.json` are equivalent. See [`config/README.md`](config/README.md) for schema. |
|
|
134
|
+
| `--input-dir <path>` | Override `sig_input_dir` and process only this directory. |
|
|
135
|
+
| `--step {1,2,all}` | `1` = process + summary CSV only; `2` = resample only (requires Stage 1 to have been run); `all` = both (default). |
|
|
136
|
+
| `--verbose` | Print INFO/DEBUG logs before and after each stage. |
|
|
137
|
+
|
|
138
|
+
### Expected output layout
|
|
139
|
+
|
|
140
|
+
```
|
|
141
|
+
pipeline_outputs/
|
|
142
|
+
├── sig_processed/<input_dir_name>/
|
|
143
|
+
│ ├── <truncated *.sig files>
|
|
144
|
+
│ └── <input_dir_name>_processed_sig_summary.csv
|
|
145
|
+
└── sig_resampled/<input_dir_name>/
|
|
146
|
+
└── <input_dir_name>_merged_spectra.csv # 2101 columns (400–2500 nm)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
`summary_csv_name` and `merged_csv_name` in the run config are suffixes; the real filenames are prefixed with the input directory name. Verify your config in [`config/README.md`](config/README.md).
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
## Verification (R-vs-Python parity)
|
|
154
|
+
|
|
155
|
+
The Python resampler is verified against the legacy R/`spectrolab` script ([`archived_r_scripts/merge_resample_sig.R`](archived_r_scripts/merge_resample_sig.R)). Acceptance threshold: **1 × 10⁻³ absolute reflectance** (0.1 %, well below the HR-1024i radiometric noise floor). Historical results:
|
|
156
|
+
|
|
157
|
+
| Dataset | Samples | Max abs diff | Mean abs diff |
|
|
158
|
+
|---|---|---|---|
|
|
159
|
+
| Silver instrument (Serial 1202103) | 66 | 1.10 × 10⁻⁶ | 4.0 × 10⁻⁸ |
|
|
160
|
+
| Bronze instrument (Serial 2212118), `a4any_sb_2025-cn_ch-svc-aviris_bottom` | 15 | 9.4 × 10⁻⁷ | 3.4 × 10⁻⁸ |
|
|
161
|
+
|
|
162
|
+
Run the parity test (skipped automatically without reference data):
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
pytest tests/test_resampler_parity.py \
|
|
166
|
+
--r-reference-csv=/path/to/r_output/merged_spectra.csv \
|
|
167
|
+
--r-input-dir=/path/to/processed_sig_files/
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
To re-run parity on a new dataset, hand the prompt at [`docs/parity_retest_prompt.md`](docs/parity_retest_prompt.md) to any capable coding LLM. Full details in [`tests/README.md`](tests/README.md) and [`docs/README.md`](docs/README.md).
|
|
171
|
+
|
|
172
|
+
---
|
|
173
|
+
|
|
174
|
+
## Public Python API
|
|
175
|
+
|
|
176
|
+
The most-used entry points (all importable from their concrete modules — `pipeline/__init__.py` is intentionally empty):
|
|
177
|
+
|
|
178
|
+
```python
|
|
179
|
+
from pipeline.sig_processor import SigFileProcessor # truncation + instrument inspection
|
|
180
|
+
from pipeline.resampler import process_sig_file, resample_spectra
|
|
181
|
+
from pipeline.processor import (
|
|
182
|
+
SVCDataProcessor, # chainable load/group/average
|
|
183
|
+
SigSpectraAverager, # facade — pass a DataFrame, get aggregated DataFrame back
|
|
184
|
+
GroupSpec, # GroupSpec.from_csv("naming_ids/<file>.csv")
|
|
185
|
+
find_spectra_by_name, # cross-DataFrame name search
|
|
186
|
+
)
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Detailed signatures and behavioural notes in [`pipeline/README.md`](pipeline/README.md).
|
|
190
|
+
|
|
191
|
+
---
|
|
192
|
+
|
|
193
|
+
## For LLMs working in this repo
|
|
194
|
+
|
|
195
|
+
1. **Read [`FOLDER_STRUCTURE.md`](FOLDER_STRUCTURE.md) first** — it lists every directory and the reading order.
|
|
196
|
+
2. **Before modifying [`pipeline/resampler.py`](pipeline/resampler.py), re-read [`docs/supplementary_methods.md`](docs/supplementary_methods.md).** The constants `_FWHM_NM`, `_SIGMA_NM`, `_INTERP_WVL`, `_FIXED_SENSOR`, `_BAND_MIN`, `_BAND_MAX`, and the algorithm steps are load-bearing for the parity claim. Changing any of them requires re-running the parity test and writing a new `docs/parity_<dataset>_<date>.md`.
|
|
197
|
+
3. **The R script at [`archived_r_scripts/merge_resample_sig.R`](archived_r_scripts/merge_resample_sig.R) is frozen.** Treat it as a behavioural reference, not as live code to edit.
|
|
198
|
+
4. **Never commit machine paths or private data.** Use the placeholders already present in [`config/config.json`](config/config.json) and the notebooks.
|
|
199
|
+
5. **Run `pytest` after any change to `pipeline/`.** The parity test will skip if reference data is unavailable; the rest of the suite still runs.
|
|
200
|
+
|
|
201
|
+
---
|
|
202
|
+
|
|
203
|
+
## Demo Notebook
|
|
204
|
+
|
|
205
|
+
The demo notebook uses an external 15-file `.sig` artifact because raw headers
|
|
206
|
+
contain GPS/location metadata. Prepare local demo data with:
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
python3 scripts/prepare_demo_data.py \
|
|
210
|
+
--source-dir data/a4any_sb_2025-cn_ch-svc-aviris_bottom
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
The notebook is config-driven: edit the settings cell (`DATA_FOLDER`,
|
|
214
|
+
`OUTPUT_FOLDER`, `INSTRUMENT`) to run it on your own data and instrument. See
|
|
215
|
+
[`notebooks/pipeline_demo/README.md`](notebooks/pipeline_demo/README.md).
|
|
216
|
+
|
|
217
|
+
---
|
|
218
|
+
|
|
219
|
+
## Requirements
|
|
220
|
+
|
|
221
|
+
Python 3.11 is the supported runtime. Runtime, demo, and development
|
|
222
|
+
dependencies are declared in [`pyproject.toml`](pyproject.toml).
|
|
223
|
+
|
|
224
|
+
- `numpy`, `scipy`, `pandas` — numerical core.
|
|
225
|
+
- `matplotlib` — demo notebook plotting.
|
|
226
|
+
- `pytest>=8.3.0` — test runner for local verification.
|
|
227
|
+
|
|
228
|
+
Install with `python -m pip install -e ".[dev,demo]"` for local development and
|
|
229
|
+
use `svc-pipeline = "pipeline.cli:main"` as the command-line entry point.
|
|
230
|
+
|
|
231
|
+
R is **not** required for the production pipeline. It is needed only to regenerate Pipeline A parity references; see [`archived_r_scripts/README.md`](archived_r_scripts/README.md).
|
|
232
|
+
|
|
233
|
+
---
|
|
234
|
+
|
|
235
|
+
## License
|
|
236
|
+
|
|
237
|
+
Released under the [GNU General Public License v3.0](LICENSE) © 2026 Cole
|
|
238
|
+
Regnier ([regs08](https://github.com/regs08), nr466@cornell.edu), Gold Lab
|
|
239
|
+
(GrapeSPEC project), Cornell University. The package uses the SPDX expression
|
|
240
|
+
`GPL-3.0-only`, chosen for compatibility with the GPL-3 `spectrolab` reference
|
|
241
|
+
implementation.
|
|
242
|
+
|
|
243
|
+
---
|
|
244
|
+
|
|
245
|
+
## Acknowledgments & Citation
|
|
246
|
+
|
|
247
|
+
This project is an independent reimplementation of the **spectrolab** algorithm
|
|
248
|
+
(no source copied) and builds on prior field-spectroscopy work. See
|
|
249
|
+
[`ACKNOWLEDGMENTS.md`](ACKNOWLEDGMENTS.md) for full attribution, third-party
|
|
250
|
+
licenses, and how to cite this software and spectrolab.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
pipeline/cli.py,sha256=jYlIKIVWuMWgHvQjpWuinz58xyqSOJwBXyDM1Nv5lBw,3185
|
|
3
|
+
pipeline/processor.py,sha256=-STrqx-jqY0ax0xQAZxkqGZ1AJEtsw0DkwzCAaKv1vA,23150
|
|
4
|
+
pipeline/resampler.py,sha256=6EAWs7gvxtM4hoLraZ-s1jiTt7Jl1nCxm23mR5kfR1w,19917
|
|
5
|
+
pipeline/run_config.py,sha256=P2wzjJ-6Kbx3pP37klQo-11r6tCVRCDQ-UGJo91eYqs,13207
|
|
6
|
+
pipeline/runner.py,sha256=mJtHZckn25rA6F_bDPu2sp5XMcRrr3IiMXS7Roholmk,8345
|
|
7
|
+
pipeline/sig_processor.py,sha256=EQZhQxAxnviCYFdEDPZ278OnyVVvl1MQlbFnat5EeqY,10739
|
|
8
|
+
svc_processing-0.1.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
9
|
+
svc_processing-0.1.1.dist-info/METADATA,sha256=8Cye5PSZMVU7xemjtj7PWGrwbEIYQB4GU1kTjUyBUkw,12356
|
|
10
|
+
svc_processing-0.1.1.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
11
|
+
svc_processing-0.1.1.dist-info/entry_points.txt,sha256=kBXIcL7fl8QtSFslLKnuB57jPn50rywWc-0uCNLZan8,51
|
|
12
|
+
svc_processing-0.1.1.dist-info/top_level.txt,sha256=Qdc1eKrvhKK_o9CPbdooOdDt7g3ZSXZDrNXHmUGl94Q,9
|
|
13
|
+
svc_processing-0.1.1.dist-info/RECORD,,
|